├── .python-version ├── bin └── go-httpbin.tar.gz ├── benchmarks ├── libs │ ├── __init__.py │ └── aiohttp_bench.py └── results │ ├── darwin │ ├── 0.2.2 │ │ ├── graphics │ │ │ ├── 04_http2_latest.png │ │ │ ├── 06_trends_latest.png │ │ │ ├── 07_proxy_latest.png │ │ │ ├── 08_heatmap_latest.png │ │ │ ├── 09_ranking_latest.png │ │ │ ├── 03_async_all_latest.png │ │ │ ├── 05_stability_latest.png │ │ │ ├── 01_sequential_all_latest.png │ │ │ └── 02_concurrent_all_latest.png │ │ └── benchmark.md │ └── 0.2.4 │ │ ├── graphics │ │ ├── 04_http2_latest.png │ │ ├── 06_trends_latest.png │ │ ├── 07_proxy_latest.png │ │ ├── 08_heatmap_latest.png │ │ ├── 09_ranking_latest.png │ │ ├── 03_async_all_latest.png │ │ ├── 05_stability_latest.png │ │ ├── 01_sequential_all_latest.png │ │ └── 02_concurrent_all_latest.png │ │ └── benchmark.md │ └── windows │ └── 0.2.4 │ ├── graphics │ ├── 04_http2_latest.png │ ├── 06_trends_latest.png │ ├── 07_proxy_latest.png │ ├── 08_heatmap_latest.png │ ├── 09_ranking_latest.png │ ├── 03_async_all_latest.png │ ├── 05_stability_latest.png │ ├── 01_sequential_all_latest.png │ └── 02_concurrent_all_latest.png │ └── benchmark.md ├── docs ├── requirements.txt ├── Makefile ├── make.bat ├── README.md └── source │ ├── conf.py │ ├── index.rst │ └── installation.rst ├── src ├── httpmorph │ ├── zlib.dll │ ├── nghttp2.dll │ └── __init__.py ├── core │ ├── internal │ │ ├── client.h │ │ ├── session.h │ │ ├── request.h │ │ ├── util.h │ │ ├── url.h │ │ ├── network.h │ │ ├── compression.h │ │ ├── core.h │ │ ├── cookies.h │ │ ├── response.h │ │ ├── proxy.h │ │ ├── http1.h │ │ ├── tls.h │ │ ├── http2_logic.h │ │ └── internal.h │ ├── http2_client.c │ ├── boringssl_wrapper.cc │ ├── http2_client.h │ ├── string_intern.h │ ├── url.c │ ├── util.c │ ├── string_intern.c │ ├── buffer_pool.h │ ├── request_builder.h │ ├── iocp_dispatcher.h │ ├── async_request_manager.h │ ├── proxy.c │ ├── request_builder.c │ ├── io_engine.h │ ├── cookies.c │ └── http2_session_manager.h ├── bindings │ └── _http2.pyx └── tls │ └── browser_profiles.h ├── .ruffignore ├── tests ├── __init__.py ├── test_basic.py └── conftest.py ├── docker ├── docker-compose.test.yml ├── docker-compose.benchmark.yml ├── Dockerfile.test ├── Dockerfile.benchmark └── run-benchmark.sh ├── hooks └── pre-commit ├── .readthedocs.yaml ├── examples ├── .env.example ├── http2_example.py ├── async_example.py └── advanced_features.py ├── .dockerignore ├── .env.example ├── pytest.ini ├── .gitignore ├── .ruff.toml ├── include └── boringssl_compat.h ├── LICENSE ├── scripts ├── setup_vendors.sh ├── dev-setup.sh ├── windows │ ├── test_build_local.sh │ └── test_windows_build_local.ps1 └── darwin │ ├── setup.py │ └── test_build_local.sh ├── Makefile ├── .github └── workflows │ ├── _config.yml │ ├── _build_linux_x86_64.yml │ ├── _build_linux_aarch64.yml │ ├── ci.yml │ ├── _build_macos.yml │ └── _build_windows.yml └── pyproject.toml /.python-version: -------------------------------------------------------------------------------- 1 | 3.11 2 | -------------------------------------------------------------------------------- /bin/go-httpbin.tar.gz: -------------------------------------------------------------------------------- 1 | Not Found -------------------------------------------------------------------------------- /benchmarks/libs/__init__.py: -------------------------------------------------------------------------------- 1 | # Benchmark library implementations 2 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | sphinx>=7.0 2 | sphinx-rtd-theme>=1.3.0 3 | myst-parser>=2.0.0 4 | -------------------------------------------------------------------------------- /src/httpmorph/zlib.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/src/httpmorph/zlib.dll -------------------------------------------------------------------------------- /src/httpmorph/nghttp2.dll: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/src/httpmorph/nghttp2.dll -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/04_http2_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/04_http2_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/06_trends_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/06_trends_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/07_proxy_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/07_proxy_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/08_heatmap_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/08_heatmap_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/09_ranking_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/09_ranking_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/04_http2_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/04_http2_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/06_trends_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/06_trends_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/07_proxy_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/07_proxy_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/08_heatmap_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/08_heatmap_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/09_ranking_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/09_ranking_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/04_http2_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/04_http2_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/06_trends_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/06_trends_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/07_proxy_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/07_proxy_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/03_async_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/03_async_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/05_stability_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/05_stability_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/03_async_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/03_async_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/05_stability_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/05_stability_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/08_heatmap_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/08_heatmap_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/09_ranking_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/09_ranking_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/03_async_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/03_async_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/05_stability_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/05_stability_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/01_sequential_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/01_sequential_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/graphics/02_concurrent_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.2/graphics/02_concurrent_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/01_sequential_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/01_sequential_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/graphics/02_concurrent_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/darwin/0.2.4/graphics/02_concurrent_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/01_sequential_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/01_sequential_all_latest.png -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/graphics/02_concurrent_all_latest.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arman-bd/httpmorph/HEAD/benchmarks/results/windows/0.2.4/graphics/02_concurrent_all_latest.png -------------------------------------------------------------------------------- /src/core/internal/client.h: -------------------------------------------------------------------------------- 1 | /** 2 | * client.h - HTTP client management 3 | */ 4 | 5 | #ifndef CLIENT_H 6 | #define CLIENT_H 7 | 8 | #include "internal.h" 9 | 10 | /* Client functions are defined in the public API (httpmorph.h) */ 11 | /* This header is for internal client-related utilities if needed */ 12 | 13 | #endif /* CLIENT_H */ 14 | -------------------------------------------------------------------------------- /src/core/internal/session.h: -------------------------------------------------------------------------------- 1 | /** 2 | * session.h - Session management 3 | */ 4 | 5 | #ifndef SESSION_H 6 | #define SESSION_H 7 | 8 | #include "internal.h" 9 | 10 | /* Session functions are defined in the public API (httpmorph.h) */ 11 | /* This header is for internal session-related utilities if needed */ 12 | 13 | #endif /* SESSION_H */ 14 | -------------------------------------------------------------------------------- /.ruffignore: -------------------------------------------------------------------------------- 1 | # Ruff ignore patterns 2 | 3 | # Third-party dependencies 4 | vendor/ 5 | 6 | # Build artifacts 7 | build/ 8 | dist/ 9 | *.egg-info/ 10 | __pycache__/ 11 | *.pyc 12 | *.pyo 13 | 14 | # Virtual environments 15 | .venv/ 16 | venv/ 17 | env/ 18 | 19 | # Compiled C extensions 20 | *.so 21 | *.pyd 22 | *.dll 23 | 24 | # IDE 25 | .vscode/ 26 | .idea/ 27 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | httpmorph test suite 3 | """ 4 | 5 | # Read version from package metadata (single source of truth: pyproject.toml) 6 | try: 7 | from importlib.metadata import version as _get_version 8 | except ImportError: 9 | # Python < 3.8 10 | from importlib_metadata import version as _get_version 11 | 12 | __version__ = _get_version("httpmorph") 13 | -------------------------------------------------------------------------------- /docker/docker-compose.test.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | test: 5 | build: 6 | context: . 7 | dockerfile: Dockerfile.test 8 | volumes: 9 | - .:/workspace 10 | - vendor-cache:/workspace/vendor 11 | environment: 12 | - PYTHONDONTWRITEBYTECODE=1 13 | - PYTHONUNBUFFERED=1 14 | command: bash 15 | 16 | volumes: 17 | vendor-cache: 18 | -------------------------------------------------------------------------------- /src/core/internal/request.h: -------------------------------------------------------------------------------- 1 | /** 2 | * request.h - HTTP request structures and operations 3 | */ 4 | 5 | #ifndef REQUEST_H 6 | #define REQUEST_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Convert HTTP method enum to string 12 | * 13 | * @param method HTTP method 14 | * @return Method string 15 | */ 16 | const char* httpmorph_method_to_string(httpmorph_method_t method); 17 | 18 | #endif /* REQUEST_H */ 19 | -------------------------------------------------------------------------------- /src/core/internal/util.h: -------------------------------------------------------------------------------- 1 | /** 2 | * util.h - Utility functions 3 | */ 4 | 5 | #ifndef UTIL_H 6 | #define UTIL_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Get current time in microseconds 12 | */ 13 | uint64_t httpmorph_get_time_us(void); 14 | 15 | /** 16 | * Base64 encode a binary buffer 17 | * Returns newly allocated string that must be freed by caller 18 | */ 19 | char* httpmorph_base64_encode(const char *input, size_t length); 20 | 21 | #endif /* UTIL_H */ 22 | -------------------------------------------------------------------------------- /hooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Pre-commit hook for httpmorph 3 | # Runs linting before allowing commits 4 | # 5 | # To install: cp hooks/pre-commit .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit 6 | 7 | set -e 8 | 9 | echo "🔍 Running code quality checks..." 10 | echo "" 11 | 12 | # Run linting 13 | echo "📋 Linting with ruff..." 14 | if ruff check src/ tests/; then 15 | echo "✅ Linting passed" 16 | else 17 | echo "❌ Linting failed - please fix errors" 18 | exit 1 19 | fi 20 | 21 | echo "" 22 | echo "✨ All checks passed!" 23 | echo "✅ Ready to commit" 24 | -------------------------------------------------------------------------------- /.readthedocs.yaml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | 7 | # Set the OS, Python version and other tools you might need 8 | build: 9 | os: ubuntu-24.04 10 | tools: 11 | python: "3.11" 12 | 13 | # Build documentation in the docs/ directory with Sphinx 14 | sphinx: 15 | configuration: docs/source/conf.py 16 | 17 | # Install Python dependencies required to build your docs 18 | python: 19 | install: 20 | - requirements: docs/requirements.txt 21 | - method: pip 22 | path: . 23 | -------------------------------------------------------------------------------- /src/core/http2_client.c: -------------------------------------------------------------------------------- 1 | /** 2 | * http2_client.c - HTTP/2 client implementation 3 | * Stub implementation 4 | */ 5 | 6 | #include "http2_client.h" 7 | #include 8 | #include 9 | 10 | /* Stub implementations for HTTP/2 client */ 11 | 12 | http2_response_t *http2_get(const char *url) { 13 | (void)url; /* Unused parameter */ 14 | /* Return NULL to indicate not implemented */ 15 | return NULL; 16 | } 17 | 18 | void http2_response_free(http2_response_t *response) { 19 | if (!response) { 20 | return; 21 | } 22 | if (response->body) { 23 | free(response->body); 24 | } 25 | free(response); 26 | } 27 | -------------------------------------------------------------------------------- /src/core/boringssl_wrapper.cc: -------------------------------------------------------------------------------- 1 | /** 2 | * boringssl_wrapper.cc - C++ wrapper for BoringSSL C++ functions 3 | */ 4 | 5 | /* Forward declare BoringSSL types */ 6 | struct ssl_ctx_st; 7 | typedef struct ssl_ctx_st SSL_CTX; 8 | 9 | /* Forward declare BoringSSL C++ function */ 10 | namespace bssl { 11 | extern void SSL_CTX_set_aes_hw_override_for_testing(SSL_CTX *ctx, bool override_value); 12 | } 13 | 14 | extern "C" { 15 | 16 | /* C wrapper for bssl::SSL_CTX_set_aes_hw_override_for_testing */ 17 | void httpmorph_set_aes_hw_override(SSL_CTX *ctx, int override_value) { 18 | bssl::SSL_CTX_set_aes_hw_override_for_testing(ctx, override_value != 0); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/core/internal/url.h: -------------------------------------------------------------------------------- 1 | /** 2 | * url.h - URL parsing and manipulation 3 | */ 4 | 5 | #ifndef URL_H 6 | #define URL_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Parse a URL into its components 12 | * 13 | * @param url Full URL string 14 | * @param scheme Output: scheme (http, https) - caller must free 15 | * @param host Output: hostname - caller must free 16 | * @param port Output: port number 17 | * @param path Output: path including query string - caller must free 18 | * @return 0 on success, -1 on error 19 | */ 20 | int httpmorph_parse_url(const char *url, char **scheme, char **host, 21 | uint16_t *port, char **path); 22 | 23 | #endif /* URL_H */ 24 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /examples/.env.example: -------------------------------------------------------------------------------- 1 | # Proxy Configuration Example 2 | # Copy this file to .env and fill in your proxy details 3 | 4 | # Proxy URL (required if using proxy) 5 | # Format: http://hostname:port or https://hostname:port 6 | PROXY_URL=http://proxy.example.com:8080 7 | 8 | # Proxy Authentication (optional) 9 | # Leave blank if your proxy doesn't require authentication 10 | PROXY_USERNAME=your_username 11 | PROXY_PASSWORD=your_password 12 | 13 | # Alternative: Embedded credentials in URL 14 | # PROXY_URL=http://username:password@proxy.example.com:8080 15 | 16 | # Examples: 17 | # - Corporate proxy: http://proxy.company.com:3128 18 | # - SOCKS proxy: socks5://localhost:1080 19 | # - Authenticated: http://user:pass@proxy.com:8080 20 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | .Python 7 | build/ 8 | develop-eggs/ 9 | dist/ 10 | downloads/ 11 | eggs/ 12 | .eggs/ 13 | lib/ 14 | lib64/ 15 | parts/ 16 | sdist/ 17 | var/ 18 | wheels/ 19 | *.egg-info/ 20 | .installed.cfg 21 | *.egg 22 | 23 | # Virtual environments 24 | venv/ 25 | ENV/ 26 | env/ 27 | .venv 28 | 29 | # IDE 30 | .vscode/ 31 | .idea/ 32 | *.swp 33 | *.swo 34 | *~ 35 | 36 | # Testing 37 | .pytest_cache/ 38 | .coverage 39 | htmlcov/ 40 | .tox/ 41 | 42 | # OS 43 | .DS_Store 44 | Thumbs.db 45 | 46 | # Git 47 | .git/ 48 | .gitignore 49 | .gitattributes 50 | 51 | # CI 52 | .github/ 53 | 54 | # Documentation 55 | docs/_build/ 56 | *.md 57 | 58 | # Local development 59 | .uv/ 60 | uv.lock 61 | -------------------------------------------------------------------------------- /.env.example: -------------------------------------------------------------------------------- 1 | # Example environment variables for httpmorph testing 2 | # Copy this file to .env and fill in your actual values 3 | 4 | # Proxy configuration for testing 5 | # Format: http://username:password@proxy-host:port 6 | # Or for country-specific proxy: http://username:password_country-CountryName@proxy-host:port 7 | TEST_PROXY_URL=http://your-username:your-password@proxy.example.com:31112 8 | 9 | # Proxy configuration for examples (separate username/password) 10 | PROXY_URL=http://proxy.example.com:31112 11 | PROXY_USERNAME=your-username 12 | PROXY_PASSWORD=your-password 13 | 14 | # HTTPBin testing host 15 | # Use httpmorph-bin.bytetunnels.com for reliable testing 16 | # (httpbin.org can be flaky and rate-limited) 17 | TEST_HTTPBIN_HOST=httpbin-of-your-own.com 18 | -------------------------------------------------------------------------------- /src/core/http2_client.h: -------------------------------------------------------------------------------- 1 | /** 2 | * http2_client.h - Minimal HTTP/2 client interface 3 | */ 4 | 5 | #ifndef HTTP2_CLIENT_H 6 | #define HTTP2_CLIENT_H 7 | 8 | #include 9 | #include 10 | 11 | #ifdef __cplusplus 12 | extern "C" { 13 | #endif 14 | 15 | typedef struct { 16 | int status_code; 17 | uint8_t *body; 18 | size_t body_len; 19 | int http_version; /* 2 for HTTP/2 */ 20 | } http2_response_t; 21 | 22 | /** 23 | * Execute a simple HTTP/2 GET request 24 | * Returns NULL on error 25 | */ 26 | http2_response_t *http2_get(const char *url); 27 | 28 | /** 29 | * Free response structure 30 | */ 31 | void http2_response_free(http2_response_t *response); 32 | 33 | #ifdef __cplusplus 34 | } 35 | #endif 36 | 37 | #endif /* HTTP2_CLIENT_H */ 38 | -------------------------------------------------------------------------------- /docker/docker-compose.benchmark.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | benchmark: 5 | build: 6 | context: .. 7 | dockerfile: docker/Dockerfile.benchmark 8 | volumes: 9 | # Mount benchmarks/results to persist results on host 10 | - ../benchmarks/results:/workspace/benchmarks/results 11 | # Mount .env for live updates (optional) 12 | - ../.env:/workspace/.env:ro 13 | environment: 14 | - PYTHONDONTWRITEBYTECODE=1 15 | - PYTHONUNBUFFERED=1 16 | # Override command to customize benchmark parameters 17 | # Example: docker-compose -f docker/docker-compose.benchmark.yml run benchmark python benchmarks/benchmark.py --libraries httpmorph --sequential 3 18 | command: python benchmarks/benchmark.py --sequential 10 --concurrent 25 --warmup 3 19 | -------------------------------------------------------------------------------- /src/core/internal/network.h: -------------------------------------------------------------------------------- 1 | /** 2 | * network.h - TCP connection and socket operations 3 | */ 4 | 5 | #ifndef NETWORK_H 6 | #define NETWORK_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Establish a TCP connection to a host 12 | * 13 | * @param host Hostname or IP address 14 | * @param port Port number 15 | * @param timeout_ms Connection timeout in milliseconds 16 | * @param connect_time Output: connection time in microseconds 17 | * @return socket file descriptor on success, -1 on error 18 | */ 19 | int httpmorph_tcp_connect(const char *host, uint16_t port, uint32_t timeout_ms, 20 | uint64_t *connect_time); 21 | 22 | /** 23 | * Cleanup expired DNS cache entries 24 | */ 25 | void dns_cache_cleanup(void); 26 | 27 | /** 28 | * Clear all DNS cache entries 29 | */ 30 | void dns_cache_clear(void); 31 | 32 | #endif /* NETWORK_H */ 33 | -------------------------------------------------------------------------------- /pytest.ini: -------------------------------------------------------------------------------- 1 | [pytest] 2 | # pytest configuration for httpmorph 3 | 4 | # Test discovery 5 | testpaths = tests 6 | python_files = test_*.py 7 | python_classes = Test* 8 | python_functions = test_* 9 | 10 | # Output options 11 | addopts = 12 | -v 13 | --strict-markers 14 | --tb=short 15 | 16 | # Test markers 17 | markers = 18 | slow: marks tests as slow (deselect with '-m "not slow"') 19 | benchmark: marks tests as benchmarks 20 | fingerprint: marks tests that check fingerprinting 21 | integration: marks tests that require network access 22 | ssl: marks tests that require SSL support 23 | proxy: marks tests that require proxy access (deselect with '-m "not proxy"') 24 | 25 | # pytest-asyncio configuration 26 | asyncio_default_fixture_loop_scope = function 27 | 28 | # Ignore specific warnings 29 | filterwarnings = 30 | ignore::DeprecationWarning:tests.test_server 31 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=source 11 | set BUILDDIR=build 12 | 13 | %SPHINXBUILD% >NUL 2>NUL 14 | if errorlevel 9009 ( 15 | echo. 16 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 17 | echo.installed, then set the SPHINXBUILD environment variable to point 18 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 19 | echo.may add the Sphinx directory to PATH. 20 | echo. 21 | echo.If you don't have Sphinx installed, grab it from 22 | echo.https://www.sphinx-doc.org/ 23 | exit /b 1 24 | ) 25 | 26 | if "%1" == "" goto help 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /src/core/string_intern.h: -------------------------------------------------------------------------------- 1 | /** 2 | * string_intern.h - String interning for common HTTP headers 3 | * 4 | * Reduces memory usage by deduplicating common header strings. 5 | * Common headers are stored once and reused across all requests/responses. 6 | */ 7 | 8 | #ifndef HTTPMORPH_STRING_INTERN_H 9 | #define HTTPMORPH_STRING_INTERN_H 10 | 11 | #include 12 | 13 | /** 14 | * Get an interned string for a common header name 15 | * Returns the interned string if found, NULL otherwise 16 | * 17 | * @param str String to intern 18 | * @param len Length of string 19 | * @return Interned string pointer or NULL if not a common header 20 | */ 21 | const char* string_intern_get(const char *str, size_t len); 22 | 23 | /** 24 | * Check if a string is interned (for optimization) 25 | * 26 | * @param str String to check 27 | * @return 1 if interned, 0 otherwise 28 | */ 29 | int string_intern_is_interned(const char *str); 30 | 31 | #endif /* HTTPMORPH_STRING_INTERN_H */ 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Python 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | *.so 6 | *.egg 7 | *.egg-info/ 8 | dist/ 9 | build/ 10 | .eggs/ 11 | *.whl 12 | .Python 13 | pip-log.txt 14 | pip-delete-this-directory.txt 15 | 16 | # Cython generated files (only in bindings directory) 17 | src/bindings/*.c 18 | src/bindings/*.cpp 19 | src/bindings/*.html 20 | 21 | # C/C++ 22 | *.o 23 | *.a 24 | *.dylib 25 | *.out 26 | *.app 27 | *.i*86 28 | *.x86_64 29 | *.hex 30 | 31 | # IDEs 32 | .vscode/ 33 | .idea/ 34 | *.swp 35 | *.swo 36 | *~ 37 | .DS_Store 38 | 39 | # Testing 40 | .pytest_cache/ 41 | .coverage 42 | htmlcov/ 43 | .tox/ 44 | .mypy_cache/ 45 | .dmypy.json 46 | dmypy.json 47 | 48 | # Documentation 49 | docs/_build/ 50 | docs/build/ 51 | 52 | # Virtual environments 53 | venv/ 54 | env/ 55 | ENV/ 56 | .venv 57 | .venv/ 58 | 59 | # uv 60 | .uv/ 61 | uv.lock 62 | 63 | # Vendor dependencies (downloaded by scripts/setup_vendors.sh) 64 | vendor/ 65 | 66 | # Environment variables 67 | .env 68 | examples/.env 69 | .plan/ 70 | -------------------------------------------------------------------------------- /.ruff.toml: -------------------------------------------------------------------------------- 1 | # Ruff configuration for httpmorph 2 | # This file provides IDE-friendly ruff settings 3 | 4 | line-length = 100 5 | target-version = "py38" 6 | fix = true 7 | 8 | # Exclude vendor directory 9 | exclude = [ 10 | "vendor/", 11 | ".venv/", 12 | "build/", 13 | "dist/", 14 | ] 15 | 16 | [lint] 17 | select = [ 18 | "E", # pycodestyle errors 19 | "W", # pycodestyle warnings 20 | "F", # pyflakes 21 | "I", # isort 22 | "B", # flake8-bugbear 23 | "C4", # flake8-comprehensions 24 | "UP", # pyupgrade 25 | ] 26 | ignore = [ 27 | "E501", # line too long (handled by formatter) 28 | "B008", # function calls in argument defaults 29 | "C901", # too complex 30 | ] 31 | 32 | [format] 33 | quote-style = "double" 34 | indent-style = "space" 35 | skip-magic-trailing-comma = false 36 | line-ending = "auto" 37 | 38 | [lint.per-file-ignores] 39 | "tests/*" = ["B", "F401", "F811"] # Allow unused imports and redefinitions in tests 40 | "benchmarks/*" = ["B"] 41 | -------------------------------------------------------------------------------- /src/core/internal/compression.h: -------------------------------------------------------------------------------- 1 | /** 2 | * compression.h - Content decompression 3 | */ 4 | 5 | #ifndef COMPRESSION_H 6 | #define COMPRESSION_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Decompress gzip-compressed response body 12 | * 13 | * @param response Response with compressed body 14 | * @return 0 on success, -1 on error 15 | */ 16 | int httpmorph_decompress_gzip(httpmorph_response_t *response); 17 | 18 | /** 19 | * Decompress deflate-compressed response body 20 | * 21 | * @param response Response with compressed body 22 | * @return 0 on success, -1 on error 23 | */ 24 | int httpmorph_decompress_deflate(httpmorph_response_t *response); 25 | 26 | /** 27 | * Automatically decompress response body based on Content-Encoding header 28 | * Supports gzip, deflate, and identity encodings 29 | * 30 | * @param response Response to decompress 31 | * @return 0 on success, -1 on error 32 | */ 33 | int httpmorph_auto_decompress(httpmorph_response_t *response); 34 | 35 | #endif /* COMPRESSION_H */ 36 | -------------------------------------------------------------------------------- /include/boringssl_compat.h: -------------------------------------------------------------------------------- 1 | /** 2 | * boringssl_compat.h - Windows compatibility layer for BoringSSL 3 | * 4 | * This header must be included before any BoringSSL headers on Windows 5 | * to avoid conflicts with Windows Cryptography API (wincrypt.h) macros. 6 | */ 7 | 8 | #ifndef BORINGSSL_COMPAT_H 9 | #define BORINGSSL_COMPAT_H 10 | 11 | #ifdef _WIN32 12 | 13 | /* Windows headers included by Python.h may define these macros 14 | * which conflict with BoringSSL type names. Undefine them before 15 | * including BoringSSL headers. */ 16 | #ifdef X509_NAME 17 | #undef X509_NAME 18 | #endif 19 | 20 | #ifdef X509_CERT_PAIR 21 | #undef X509_CERT_PAIR 22 | #endif 23 | 24 | #ifdef X509_EXTENSIONS 25 | #undef X509_EXTENSIONS 26 | #endif 27 | 28 | #ifdef PKCS7_SIGNER_INFO 29 | #undef PKCS7_SIGNER_INFO 30 | #endif 31 | 32 | #ifdef OCSP_REQUEST 33 | #undef OCSP_REQUEST 34 | #endif 35 | 36 | #ifdef OCSP_RESPONSE 37 | #undef OCSP_RESPONSE 38 | #endif 39 | 40 | #endif /* _WIN32 */ 41 | 42 | #endif /* BORINGSSL_COMPAT_H */ 43 | -------------------------------------------------------------------------------- /src/core/internal/core.h: -------------------------------------------------------------------------------- 1 | /** 2 | * core.h - Core orchestration for HTTP requests 3 | */ 4 | 5 | #ifndef CORE_H 6 | #define CORE_H 7 | 8 | #include "internal.h" 9 | #include "client.h" 10 | #include "request.h" 11 | #include "response.h" 12 | #include "../connection_pool.h" 13 | 14 | /** 15 | * Execute an HTTP request (main orchestration function) 16 | * 17 | * This function coordinates all HTTP request operations: 18 | * 1. URL parsing 19 | * 2. TCP connection (direct or via proxy) 20 | * 3. TLS handshake (if HTTPS) 21 | * 4. HTTP/2 or HTTP/1.1 request/response 22 | * 5. Gzip decompression 23 | * 6. Connection pooling 24 | * 25 | * @param client HTTP client with SSL context and configuration 26 | * @param request Request to execute 27 | * @param pool Connection pool for keep-alive (can be NULL) 28 | * @return Response object (always non-NULL, check response->error for errors) 29 | */ 30 | httpmorph_response_t* httpmorph_request_execute( 31 | httpmorph_client_t *client, 32 | const httpmorph_request_t *request, 33 | httpmorph_pool_t *pool); 34 | 35 | #endif /* CORE_H */ 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 Arman 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/core/internal/cookies.h: -------------------------------------------------------------------------------- 1 | /** 2 | * cookies.h - Cookie management 3 | */ 4 | 5 | #ifndef COOKIES_H 6 | #define COOKIES_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Free a cookie structure 12 | */ 13 | void httpmorph_cookie_free(cookie_t *cookie); 14 | 15 | /** 16 | * Parse Set-Cookie header and add to session 17 | * 18 | * @param session Session to add cookie to 19 | * @param header_value Set-Cookie header value 20 | * @param request_domain Domain of the request that received this cookie 21 | */ 22 | void httpmorph_parse_set_cookie(httpmorph_session_t *session, 23 | const char *header_value, 24 | const char *request_domain); 25 | 26 | /** 27 | * Get cookies for a request as a Cookie header value 28 | * 29 | * @param session Session containing cookies 30 | * @param domain Request domain 31 | * @param path Request path 32 | * @param is_secure Whether request uses HTTPS 33 | * @return Cookie header value (caller must free) or NULL if no cookies 34 | */ 35 | char* httpmorph_get_cookies_for_request(httpmorph_session_t *session, 36 | const char *domain, 37 | const char *path, 38 | bool is_secure); 39 | 40 | #endif /* COOKIES_H */ 41 | -------------------------------------------------------------------------------- /src/core/internal/response.h: -------------------------------------------------------------------------------- 1 | /** 2 | * response.h - HTTP response structures and operations 3 | */ 4 | 5 | #ifndef RESPONSE_H 6 | #define RESPONSE_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Create a new response structure 12 | * 13 | * @param buffer_pool Buffer pool for body allocation (can be NULL for no pooling) 14 | * @return Newly allocated response or NULL on error 15 | */ 16 | httpmorph_response_t* httpmorph_response_create(httpmorph_buffer_pool_t *buffer_pool); 17 | 18 | /** 19 | * Parse HTTP response status line 20 | * 21 | * @param line Status line (e.g., "HTTP/1.1 200 OK") 22 | * @param response Response to populate 23 | * @return 0 on success, -1 on error 24 | */ 25 | int httpmorph_parse_response_line(const char *line, httpmorph_response_t *response); 26 | 27 | /** 28 | * Add a header to response (with length for HTTP/2) 29 | * 30 | * @param response Response to add header to 31 | * @param name Header name 32 | * @param namelen Length of header name 33 | * @param value Header value 34 | * @param valuelen Length of header value 35 | * @return 0 on success, -1 on error 36 | */ 37 | int httpmorph_response_add_header_internal(httpmorph_response_t *response, 38 | const char *name, size_t namelen, 39 | const char *value, size_t valuelen); 40 | 41 | #endif /* RESPONSE_H */ 42 | -------------------------------------------------------------------------------- /scripts/setup_vendors.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # setup_vendors.sh - Download and build vendor dependencies 4 | # 5 | # This script detects the OS and delegates to OS-specific setup scripts: 6 | # - scripts/windows/setup_vendors.sh 7 | # - scripts/linux/setup_vendors.sh 8 | # - scripts/darwin/setup_vendors.sh 9 | # 10 | 11 | set -e 12 | 13 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 14 | 15 | echo "================================" 16 | echo "httpmorph - Vendor Setup" 17 | echo "================================" 18 | echo "" 19 | 20 | # Detect OS 21 | OS="$(uname -s)" 22 | case "$OS" in 23 | MINGW*|MSYS*|CYGWIN*) 24 | OS_DIR="windows" 25 | OS_NAME="Windows" 26 | ;; 27 | Darwin) 28 | OS_DIR="darwin" 29 | OS_NAME="macOS" 30 | ;; 31 | Linux) 32 | OS_DIR="linux" 33 | OS_NAME="Linux" 34 | ;; 35 | *) 36 | echo "ERROR: Unsupported OS: $OS" 37 | exit 1 38 | ;; 39 | esac 40 | 41 | echo "Detected OS: $OS_NAME" 42 | echo "Delegating to: scripts/$OS_DIR/setup_vendors.sh" 43 | echo "" 44 | 45 | # Execute OS-specific script 46 | OS_SCRIPT="$SCRIPT_DIR/$OS_DIR/setup_vendors.sh" 47 | 48 | if [ ! -f "$OS_SCRIPT" ]; then 49 | echo "ERROR: OS-specific script not found: $OS_SCRIPT" 50 | exit 1 51 | fi 52 | 53 | # Make sure the script is executable 54 | chmod +x "$OS_SCRIPT" 55 | 56 | # Execute the OS-specific script 57 | exec "$OS_SCRIPT" 58 | -------------------------------------------------------------------------------- /scripts/dev-setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # dev-setup.sh - Complete development environment setup using uv 4 | # 5 | 6 | set -e 7 | 8 | echo "================================" 9 | echo "httpmorph - Development Setup" 10 | echo "================================" 11 | echo "" 12 | 13 | # Check if uv is installed 14 | if ! command -v uv &> /dev/null; then 15 | echo "❌ uv is not installed" 16 | echo "" 17 | echo "Install uv with:" 18 | echo " curl -LsSf https://astral.sh/uv/install.sh | sh" 19 | echo "" 20 | echo "Or on macOS:" 21 | echo " brew install uv" 22 | echo "" 23 | exit 1 24 | fi 25 | 26 | echo "✓ uv found: $(uv --version)" 27 | echo "" 28 | 29 | # Sync Python dependencies 30 | echo "==> Syncing Python dependencies with uv..." 31 | uv sync --extra dev --extra build 32 | 33 | echo "" 34 | echo "==> Setting up vendor dependencies..." 35 | ./scripts/setup_vendors.sh 36 | 37 | echo "" 38 | echo "==> Building C extensions..." 39 | uv run python setup.py build_ext --inplace 40 | 41 | echo "" 42 | echo "================================" 43 | echo "Development Environment Ready!" 44 | echo "================================" 45 | echo "" 46 | echo "Quick commands:" 47 | echo " make test - Run tests" 48 | echo " make lint - Check code quality" 49 | echo " make format - Format code" 50 | echo " make build - Rebuild C extensions" 51 | echo "" 52 | echo "Or use uv directly:" 53 | echo " uv run pytest tests/ -v" 54 | echo " uv run ruff check src/" 55 | echo " uv run ruff format src/" 56 | echo "" 57 | -------------------------------------------------------------------------------- /src/bindings/_http2.pyx: -------------------------------------------------------------------------------- 1 | # cython: language_level=3 2 | """ 3 | _http2.pyx - Simple Cython wrapper for HTTP/2 client 4 | """ 5 | 6 | from libc.stdint cimport uint8_t 7 | from libc.stdlib cimport free 8 | 9 | cdef extern from "http2_client.h": 10 | ctypedef struct http2_response_t: 11 | int status_code 12 | uint8_t *body 13 | size_t body_len 14 | int http_version 15 | 16 | http2_response_t *http2_get(const char *url) 17 | void http2_response_free(http2_response_t *response) 18 | 19 | 20 | class HTTP2Response: 21 | """HTTP/2 response object""" 22 | def __init__(self, status_code, body, http_version): 23 | self.status_code = status_code 24 | self.body = body 25 | self.http_version = http_version 26 | self.text = body.decode('utf-8', errors='replace') if body else '' 27 | 28 | 29 | def get(url): 30 | """ 31 | Execute an HTTP/2 GET request 32 | 33 | Args: 34 | url: URL to request (must be https://) 35 | 36 | Returns: 37 | HTTP2Response object or None on error 38 | """ 39 | if isinstance(url, str): 40 | url = url.encode('utf-8') 41 | 42 | cdef http2_response_t *response = http2_get(url) 43 | if response == NULL: 44 | return None 45 | 46 | # Extract data before freeing 47 | status_code = response.status_code 48 | http_version = response.http_version 49 | body = response.body[:response.body_len] if response.body else b'' 50 | 51 | # Free C structure 52 | http2_response_free(response) 53 | 54 | return HTTP2Response(status_code, body, http_version) 55 | -------------------------------------------------------------------------------- /src/core/internal/proxy.h: -------------------------------------------------------------------------------- 1 | /** 2 | * proxy.h - HTTP proxy handling 3 | */ 4 | 5 | #ifndef PROXY_H 6 | #define PROXY_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Parse a proxy URL 12 | * 13 | * @param proxy_url Proxy URL (e.g., "http://user:pass@proxy.com:8080") 14 | * @param host Output: proxy hostname - caller must free 15 | * @param port Output: proxy port 16 | * @param username Output: proxy username (NULL if not present) - caller must free 17 | * @param password Output: proxy password (NULL if not present) - caller must free 18 | * @param use_tls Output: whether proxy connection should use TLS 19 | * @return 0 on success, -1 on error 20 | */ 21 | int httpmorph_parse_proxy_url(const char *proxy_url, char **host, uint16_t *port, 22 | char **username, char **password, bool *use_tls); 23 | 24 | /** 25 | * Send HTTP CONNECT request to establish tunnel through proxy 26 | * 27 | * @param sockfd Socket connected to proxy 28 | * @param proxy_ssl SSL connection to proxy (NULL if not using TLS to proxy) 29 | * @param target_host Target hostname 30 | * @param target_port Target port 31 | * @param proxy_username Proxy username (NULL if not required) 32 | * @param proxy_password Proxy password (NULL if not required) 33 | * @param timeout_ms Timeout in milliseconds 34 | * @return 0 on success, -1 on error 35 | */ 36 | int httpmorph_proxy_connect(int sockfd, SSL *proxy_ssl, const char *target_host, 37 | uint16_t target_port, const char *proxy_username, 38 | const char *proxy_password, uint32_t timeout_ms); 39 | 40 | #endif /* PROXY_H */ 41 | -------------------------------------------------------------------------------- /docker/Dockerfile.test: -------------------------------------------------------------------------------- 1 | # Test Dockerfile for httpmorph CI environment 2 | # Mimics GitHub Actions ubuntu-latest environment 3 | 4 | FROM ubuntu:22.04 5 | 6 | # Prevent interactive prompts during package installation 7 | ENV DEBIAN_FRONTEND=noninteractive 8 | 9 | # Install system dependencies (matching CI workflow) 10 | RUN apt-get update && apt-get install -y \ 11 | cmake \ 12 | ninja-build \ 13 | libssl-dev \ 14 | pkg-config \ 15 | autoconf \ 16 | automake \ 17 | libtool \ 18 | git \ 19 | curl \ 20 | build-essential \ 21 | python3.11 \ 22 | python3.11-dev \ 23 | python3-pip \ 24 | && rm -rf /var/lib/apt/lists/* 25 | 26 | # Set Python 3.11 as default 27 | RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 28 | RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 29 | 30 | # Upgrade pip 31 | RUN python3 -m pip install --upgrade pip setuptools wheel 32 | 33 | # Set working directory 34 | WORKDIR /workspace 35 | 36 | # Copy project files 37 | COPY . . 38 | 39 | # Copy .env file for test credentials 40 | COPY .env .env 41 | 42 | # Clean any macOS compiled binaries to prevent conflicts 43 | RUN find src -name "*.so" -type f -delete && \ 44 | find src -name "*.dylib" -type f -delete 45 | 46 | # Setup vendor dependencies 47 | RUN chmod +x scripts/setup_vendors.sh && ./scripts/setup_vendors.sh 48 | 49 | # Install Python package in development mode 50 | RUN pip install -e ".[dev]" 51 | 52 | # Install python-dotenv for .env file support in tests 53 | RUN pip install python-dotenv 54 | 55 | # Default command: run tests 56 | CMD ["pytest", "tests/", "-v"] 57 | -------------------------------------------------------------------------------- /examples/http2_example.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | HTTP/2 Support Example - httpx-like API 4 | 5 | httpmorph now supports HTTP/2 just like httpx! 6 | """ 7 | 8 | import httpmorph 9 | 10 | # Example 1: Using Client with HTTP/2 enabled 11 | print("Example 1: Client with HTTP/2") 12 | print("-" * 40) 13 | 14 | client = httpmorph.Client(http2=True) 15 | response = client.get("https://www.google.com") 16 | 17 | print(f"Status: {response.status_code}") 18 | print(f"HTTP Version: {response.http_version}") 19 | print() 20 | 21 | 22 | # Example 2: Using Session with HTTP/2 enabled 23 | print("Example 2: Session with HTTP/2") 24 | print("-" * 40) 25 | 26 | with httpmorph.Session(browser="chrome", http2=True) as session: 27 | response = session.get("https://www.google.com") 28 | print(f"Status: {response.status_code}") 29 | print(f"HTTP Version: {response.http_version}") 30 | print() 31 | 32 | 33 | # Example 3: Per-request HTTP/2 override 34 | print("Example 3: Per-request override") 35 | print("-" * 40) 36 | 37 | # Create client with HTTP/2 disabled by default 38 | client = httpmorph.Client(http2=False) 39 | 40 | # But enable it for a specific request 41 | response = client.get("https://www.google.com", http2=True) 42 | print(f"HTTP Version: {response.http_version}") 43 | print() 44 | 45 | 46 | # Example 4: Comparing with httpx API 47 | print("Example 4: httpx API compatibility") 48 | print("-" * 40) 49 | print(""" 50 | # httpx syntax: 51 | import httpx 52 | client = httpx.Client(http2=True) 53 | response = client.get('https://www.google.com') 54 | 55 | # httpmorph syntax (identical!): 56 | import httpmorph 57 | client = httpmorph.Client(http2=True) 58 | response = client.get('https://www.google.com') 59 | """) 60 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | # httpmorph Documentation 2 | 3 | This directory contains the documentation for httpmorph, built with [Sphinx](https://www.sphinx-doc.org/). 4 | 5 | ## Building the Documentation Locally 6 | 7 | ### Prerequisites 8 | 9 | Install the documentation dependencies: 10 | 11 | ```bash 12 | pip install -r requirements.txt 13 | ``` 14 | 15 | ### Build HTML Documentation 16 | 17 | ```bash 18 | cd docs 19 | make html 20 | ``` 21 | 22 | The built documentation will be in `build/html/`. Open `build/html/index.html` in your browser. 23 | 24 | ### Other Build Targets 25 | 26 | ```bash 27 | make clean # Remove built documentation 28 | make help # Show all available targets 29 | ``` 30 | 31 | ## Documentation Structure 32 | 33 | - `source/` - Documentation source files (reStructuredText) 34 | - `index.rst` - Main documentation page 35 | - `quickstart.rst` - Quick start guide 36 | - `api.rst` - API reference 37 | - `conf.py` - Sphinx configuration 38 | 39 | ## ReadTheDocs 40 | 41 | The documentation is automatically built and hosted on ReadTheDocs when changes are pushed to the repository. 42 | 43 | Configuration: `.readthedocs.yaml` in the project root 44 | 45 | ## Contributing to Documentation 46 | 47 | Documentation contributions are welcome! Please: 48 | 49 | 1. Edit the `.rst` files in `source/` 50 | 2. Build locally to verify your changes: `make html` 51 | 3. Submit a pull request 52 | 53 | ## Current Status 54 | 55 | This is an initial documentation release. Comprehensive documentation including: 56 | 57 | - Detailed API reference 58 | - Advanced usage guides 59 | - Examples and tutorials 60 | - Browser fingerprinting details 61 | - Performance tuning guides 62 | - HTTP/2 configuration 63 | 64 | ...will be added in future updates. 65 | -------------------------------------------------------------------------------- /src/core/url.c: -------------------------------------------------------------------------------- 1 | /** 2 | * url.c - URL parsing and manipulation 3 | */ 4 | 5 | #include "internal/url.h" 6 | 7 | /** 8 | * Parse a URL into its components 9 | */ 10 | int httpmorph_parse_url(const char *url, char **scheme, char **host, 11 | uint16_t *port, char **path) { 12 | if (!url || !scheme || !host || !port || !path) { 13 | return -1; 14 | } 15 | 16 | /* Simple URL parser */ 17 | const char *p = url; 18 | 19 | /* Parse scheme */ 20 | const char *scheme_end = strstr(p, "://"); 21 | if (!scheme_end) { 22 | return -1; 23 | } 24 | 25 | *scheme = strndup(p, scheme_end - p); 26 | p = scheme_end + 3; 27 | 28 | /* Parse host and optional port */ 29 | const char *path_start = strchr(p, '/'); 30 | const char *port_start = strchr(p, ':'); 31 | 32 | if (port_start && (!path_start || port_start < path_start)) { 33 | /* Port specified */ 34 | *host = strndup(p, port_start - p); 35 | p = port_start + 1; 36 | 37 | char *end; 38 | long port_num = strtol(p, &end, 10); 39 | if (port_num <= 0 || port_num > 65535) { 40 | free(*host); 41 | free(*scheme); 42 | return -1; 43 | } 44 | *port = (uint16_t)port_num; 45 | p = end; 46 | } else { 47 | /* Default port */ 48 | *port = (strcmp(*scheme, "https") == 0) ? 443 : 80; 49 | 50 | if (path_start) { 51 | *host = strndup(p, path_start - p); 52 | p = path_start; 53 | } else { 54 | *host = strdup(p); 55 | p = p + strlen(p); 56 | } 57 | } 58 | 59 | /* Parse path */ 60 | if (*p == '\0') { 61 | *path = strdup("/"); 62 | } else { 63 | *path = strdup(p); 64 | } 65 | 66 | return 0; 67 | } 68 | -------------------------------------------------------------------------------- /src/core/internal/http1.h: -------------------------------------------------------------------------------- 1 | /** 2 | * http1.h - HTTP/1.1 protocol implementation 3 | */ 4 | 5 | #ifndef HTTP1_H 6 | #define HTTP1_H 7 | 8 | #include "internal.h" 9 | #include "request.h" 10 | #include "response.h" 11 | #include 12 | 13 | /** 14 | * Send HTTP/1.1 request over a connection 15 | * 16 | * @param ssl SSL connection (NULL if not using TLS) 17 | * @param sockfd Socket file descriptor 18 | * @param request Request object 19 | * @param host Target host 20 | * @param path URL path 21 | * @param scheme URL scheme ("http" or "https") 22 | * @param port Target port 23 | * @param use_proxy Whether connection is via proxy 24 | * @param proxy_user Proxy username (NULL if not using auth) 25 | * @param proxy_pass Proxy password (NULL if not using auth) 26 | * @return 0 on success, -1 on error 27 | */ 28 | int httpmorph_send_http_request(SSL *ssl, int sockfd, const httpmorph_request_t *request, 29 | const char *host, const char *path, const char *scheme, 30 | uint16_t port, bool use_proxy, const char *proxy_user, 31 | const char *proxy_pass); 32 | 33 | /** 34 | * Receive HTTP/1.1 response from a connection 35 | * 36 | * @param ssl SSL connection (NULL if not using TLS) 37 | * @param sockfd Socket file descriptor 38 | * @param response Response object to populate 39 | * @param first_byte_time_us Output: time of first byte received (microseconds) 40 | * @param conn_will_close Output: whether connection should be closed 41 | * @param method HTTP method used in request (needed for HEAD handling) 42 | * @return 0 on success, error code on failure 43 | */ 44 | int httpmorph_recv_http_response(SSL *ssl, int sockfd, httpmorph_response_t *response, 45 | uint64_t *first_byte_time_us, bool *conn_will_close, 46 | httpmorph_method_t method); 47 | 48 | #endif /* HTTP1_H */ 49 | -------------------------------------------------------------------------------- /src/core/util.c: -------------------------------------------------------------------------------- 1 | /** 2 | * util.c - Utility functions 3 | */ 4 | 5 | #include "internal/util.h" 6 | 7 | /** 8 | * Get current time in microseconds 9 | */ 10 | uint64_t httpmorph_get_time_us(void) { 11 | #ifdef _WIN32 12 | LARGE_INTEGER frequency, counter; 13 | QueryPerformanceFrequency(&frequency); 14 | QueryPerformanceCounter(&counter); 15 | return (uint64_t)((counter.QuadPart * 1000000) / frequency.QuadPart); 16 | #else 17 | struct timespec ts; 18 | clock_gettime(CLOCK_MONOTONIC, &ts); 19 | return (uint64_t)ts.tv_sec * 1000000 + ts.tv_nsec / 1000; 20 | #endif 21 | } 22 | 23 | /** 24 | * Base64 encode a binary buffer 25 | * Returns newly allocated string that must be freed by caller 26 | */ 27 | char* httpmorph_base64_encode(const char *input, size_t length) { 28 | static const char base64_chars[] = 29 | "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; 30 | 31 | size_t output_length = 4 * ((length + 2) / 3); 32 | char *output = malloc(output_length + 1); 33 | if (!output) return NULL; 34 | 35 | size_t i, j; 36 | for (i = 0, j = 0; i < length;) { 37 | uint32_t octet_a = i < length ? (unsigned char)input[i++] : 0; 38 | uint32_t octet_b = i < length ? (unsigned char)input[i++] : 0; 39 | uint32_t octet_c = i < length ? (unsigned char)input[i++] : 0; 40 | 41 | uint32_t triple = (octet_a << 0x10) + (octet_b << 0x08) + octet_c; 42 | 43 | output[j++] = base64_chars[(triple >> 3 * 6) & 0x3F]; 44 | output[j++] = base64_chars[(triple >> 2 * 6) & 0x3F]; 45 | output[j++] = base64_chars[(triple >> 1 * 6) & 0x3F]; 46 | output[j++] = base64_chars[(triple >> 0 * 6) & 0x3F]; 47 | } 48 | 49 | /* Handle padding */ 50 | int padding = (3 - (length % 3)) % 3; 51 | for (int k = 0; k < padding; k++) { 52 | output[output_length - 1 - k] = '='; 53 | } 54 | 55 | output[output_length] = '\0'; 56 | return output; 57 | } 58 | -------------------------------------------------------------------------------- /docker/Dockerfile.benchmark: -------------------------------------------------------------------------------- 1 | # Benchmark Dockerfile for httpmorph 2 | # Includes all benchmark dependencies and libraries for comparison 3 | 4 | FROM ubuntu:22.04 5 | 6 | # Prevent interactive prompts during package installation 7 | ENV DEBIAN_FRONTEND=noninteractive 8 | 9 | # Install system dependencies 10 | RUN apt-get update && apt-get install -y \ 11 | cmake \ 12 | ninja-build \ 13 | libssl-dev \ 14 | pkg-config \ 15 | autoconf \ 16 | automake \ 17 | libtool \ 18 | git \ 19 | curl \ 20 | build-essential \ 21 | python3.11 \ 22 | python3.11-dev \ 23 | python3-pip \ 24 | libcurl4-openssl-dev \ 25 | && rm -rf /var/lib/apt/lists/* 26 | 27 | # Set Python 3.11 as default 28 | RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 29 | RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 1 30 | 31 | # Upgrade pip 32 | RUN python3 -m pip install --upgrade pip setuptools wheel 33 | 34 | # Set working directory 35 | WORKDIR /workspace 36 | 37 | # Copy project files 38 | COPY . . 39 | 40 | # Copy .env file for benchmark credentials 41 | COPY .env .env 42 | 43 | # Clean any macOS compiled binaries to prevent conflicts 44 | RUN find src -name "*.so" -type f -delete && \ 45 | find src -name "*.dylib" -type f -delete 46 | 47 | # Setup vendor dependencies 48 | RUN chmod +x scripts/setup_vendors.sh && ./scripts/setup_vendors.sh 49 | 50 | # Install httpmorph 51 | RUN pip install -e . 52 | 53 | # Install benchmark dependencies 54 | RUN pip install \ 55 | matplotlib \ 56 | numpy \ 57 | python-dotenv \ 58 | requests \ 59 | httpx \ 60 | h2 \ 61 | aiohttp \ 62 | urllib3 \ 63 | pycurl \ 64 | curl-cffi 65 | 66 | # Create results directory for Linux benchmarks 67 | RUN mkdir -p /workspace/benchmarks/results/linux 68 | 69 | # Default command: run benchmarks 70 | CMD ["python", "benchmarks/benchmark.py", "--sequential", "25", "--concurrent", "25", "--warmup", "5"] 71 | -------------------------------------------------------------------------------- /docker/run-benchmark.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # run-benchmark.sh - Run benchmarks in Docker container 4 | # 5 | # Usage: 6 | # ./docker/run-benchmark.sh # Run all libraries with default settings 7 | # ./docker/run-benchmark.sh httpmorph # Run only httpmorph 8 | # ./docker/run-benchmark.sh httpmorph 3 1 # Run httpmorph with 3 requests, 1 warmup 9 | # 10 | 11 | set -e 12 | 13 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 14 | ROOT_DIR="$(dirname "$SCRIPT_DIR")" 15 | 16 | cd "$ROOT_DIR" 17 | 18 | # Default values 19 | LIBRARIES="${1:-all}" 20 | SEQUENTIAL="${2:-10}" 21 | WARMUP="${3:-3}" 22 | 23 | echo "================================" 24 | echo "httpmorph Benchmark (Docker)" 25 | echo "================================" 26 | echo "" 27 | echo "Libraries: $LIBRARIES" 28 | echo "Sequential: $SEQUENTIAL requests" 29 | echo "Warmup: $WARMUP requests" 30 | echo "" 31 | 32 | # Build the Docker image 33 | echo "Building Docker image..." 34 | docker build -f docker/Dockerfile.benchmark -t httpmorph-benchmark . 35 | 36 | # Run the benchmark 37 | echo "" 38 | echo "Running benchmark..." 39 | echo "" 40 | 41 | if [ "$LIBRARIES" = "all" ]; then 42 | docker run --rm \ 43 | -v "$ROOT_DIR/benchmarks/results:/workspace/benchmarks/results" \ 44 | -v "$ROOT_DIR/.env:/workspace/.env:ro" \ 45 | httpmorph-benchmark \ 46 | python benchmarks/benchmark.py --sequential "$SEQUENTIAL" --warmup "$WARMUP" 47 | else 48 | docker run --rm \ 49 | -v "$ROOT_DIR/benchmarks/results:/workspace/benchmarks/results" \ 50 | -v "$ROOT_DIR/.env:/workspace/.env:ro" \ 51 | httpmorph-benchmark \ 52 | python benchmarks/benchmark.py --libraries "$LIBRARIES" --sequential "$SEQUENTIAL" --warmup "$WARMUP" 53 | fi 54 | 55 | echo "" 56 | echo "================================" 57 | echo "Benchmark Complete!" 58 | echo "================================" 59 | echo "" 60 | echo "Results saved to: benchmarks/results/linux/" 61 | echo "" 62 | -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # For the full list of built-in configuration values, see the documentation: 4 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 5 | 6 | import os 7 | import sys 8 | 9 | sys.path.insert(0, os.path.abspath("../..")) 10 | 11 | # -- Project information ----------------------------------------------------- 12 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information 13 | 14 | project = "httpmorph" 15 | copyright = "2025, Arman Hossain" 16 | author = "Arman Hossain" 17 | 18 | # Read version from package metadata (single source of truth: pyproject.toml) 19 | try: 20 | from importlib.metadata import version as _get_version 21 | except ImportError: 22 | # Python < 3.8 23 | from importlib_metadata import version as _get_version 24 | 25 | release = _get_version("httpmorph") 26 | 27 | # -- General configuration --------------------------------------------------- 28 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration 29 | 30 | extensions = [ 31 | "sphinx.ext.autodoc", 32 | "sphinx.ext.viewcode", 33 | "sphinx.ext.napoleon", 34 | "myst_parser", 35 | ] 36 | 37 | templates_path = ["_templates"] 38 | exclude_patterns = [] 39 | 40 | # -- Options for HTML output ------------------------------------------------- 41 | # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output 42 | 43 | html_theme = "sphinx_rtd_theme" 44 | html_static_path = ["_static"] 45 | 46 | # Theme options for Read the Docs theme 47 | html_theme_options = { 48 | "display_version": True, 49 | "prev_next_buttons_location": "bottom", 50 | "style_external_links": False, 51 | "collapse_navigation": False, 52 | "sticky_navigation": True, 53 | "navigation_depth": 4, 54 | } 55 | 56 | # Add author contact information 57 | html_context = { 58 | "display_github": True, 59 | "github_user": "arman-bd", 60 | "github_repo": "httpmorph", 61 | "github_version": "main", 62 | "conf_py_path": "/docs/source/", 63 | } 64 | 65 | # Additional metadata 66 | html_show_sourcelink = True 67 | html_show_sphinx = True 68 | html_show_copyright = True 69 | 70 | # -- Options for autodoc ----------------------------------------------------- 71 | autodoc_member_order = "bysource" 72 | autodoc_typehints = "description" 73 | -------------------------------------------------------------------------------- /src/core/string_intern.c: -------------------------------------------------------------------------------- 1 | /** 2 | * string_intern.c - String interning implementation 3 | */ 4 | 5 | #include "string_intern.h" 6 | #include 7 | #ifndef _WIN32 8 | #include 9 | #endif 10 | 11 | /* Common HTTP header names (case-insensitive) */ 12 | static const char* COMMON_HEADERS[] = { 13 | /* Request headers */ 14 | "Accept", 15 | "Accept-Encoding", 16 | "Accept-Language", 17 | "Authorization", 18 | "Cache-Control", 19 | "Connection", 20 | "Content-Length", 21 | "Content-Type", 22 | "Cookie", 23 | "Host", 24 | "If-Modified-Since", 25 | "If-None-Match", 26 | "Origin", 27 | "Referer", 28 | "User-Agent", 29 | 30 | /* Response headers */ 31 | "Age", 32 | "Content-Encoding", 33 | "Date", 34 | "ETag", 35 | "Expires", 36 | "Last-Modified", 37 | "Location", 38 | "Server", 39 | "Set-Cookie", 40 | "Transfer-Encoding", 41 | "Vary", 42 | 43 | /* Common custom headers */ 44 | "X-Forwarded-For", 45 | "X-Forwarded-Proto", 46 | "X-Real-IP", 47 | 48 | NULL /* Sentinel */ 49 | }; 50 | 51 | /* Total number of interned strings */ 52 | #define NUM_INTERNED_STRINGS (sizeof(COMMON_HEADERS) / sizeof(char*) - 1) 53 | 54 | /** 55 | * Get an interned string for a common header name 56 | */ 57 | const char* string_intern_get(const char *str, size_t len) { 58 | if (!str || len == 0) { 59 | return NULL; 60 | } 61 | 62 | /* Linear search is fine for ~30 strings (very cache-friendly) */ 63 | for (size_t i = 0; COMMON_HEADERS[i] != NULL; i++) { 64 | size_t intern_len = strlen(COMMON_HEADERS[i]); 65 | 66 | /* Fast length check first */ 67 | if (len != intern_len) { 68 | continue; 69 | } 70 | 71 | /* Case-insensitive comparison */ 72 | if (strncasecmp(str, COMMON_HEADERS[i], len) == 0) { 73 | return COMMON_HEADERS[i]; 74 | } 75 | } 76 | 77 | return NULL; /* Not a common header */ 78 | } 79 | 80 | /** 81 | * Check if a string is interned 82 | */ 83 | int string_intern_is_interned(const char *str) { 84 | if (!str) { 85 | return 0; 86 | } 87 | 88 | /* Check if pointer is in our interned array */ 89 | for (size_t i = 0; COMMON_HEADERS[i] != NULL; i++) { 90 | if (str == COMMON_HEADERS[i]) { 91 | return 1; 92 | } 93 | } 94 | 95 | return 0; 96 | } 97 | -------------------------------------------------------------------------------- /src/core/internal/tls.h: -------------------------------------------------------------------------------- 1 | /** 2 | * tls.h - TLS/SSL operations and fingerprinting 3 | */ 4 | 5 | #ifndef TLS_H 6 | #define TLS_H 7 | 8 | #include "internal.h" 9 | 10 | /** 11 | * Configure SSL context with browser profile 12 | * 13 | * @param ctx SSL context to configure 14 | * @param profile Browser profile for fingerprinting 15 | * @return 0 on success, -1 on error 16 | */ 17 | int httpmorph_configure_ssl_ctx(SSL_CTX *ctx, const browser_profile_t *profile); 18 | 19 | /** 20 | * Configure SSL context TLS version range 21 | * 22 | * @param ctx SSL context to configure 23 | * @param min_version Minimum TLS version (0 for default) 24 | * @param max_version Maximum TLS version (0 for default) 25 | * @return 0 on success, -1 on error 26 | */ 27 | int httpmorph_set_tls_version_range(SSL_CTX *ctx, uint16_t min_version, uint16_t max_version); 28 | 29 | /** 30 | * Configure SSL verification mode 31 | * 32 | * @param ctx SSL context to configure 33 | * @param verify Whether to verify SSL certificates 34 | * @return 0 on success, -1 on error 35 | */ 36 | int httpmorph_set_ssl_verification(SSL_CTX *ctx, bool verify); 37 | 38 | /** 39 | * Establish TLS connection on existing socket 40 | * 41 | * @param ctx SSL context 42 | * @param sockfd Socket file descriptor 43 | * @param hostname Hostname for SNI 44 | * @param browser_profile Browser profile for fingerprinting 45 | * @param http2_enabled Whether HTTP/2 is enabled 46 | * @param verify_cert Whether to verify server certificate 47 | * @param tls_time Output: TLS handshake time in microseconds 48 | * @return SSL* on success, NULL on error 49 | */ 50 | SSL* httpmorph_tls_connect(SSL_CTX *ctx, int sockfd, const char *hostname, 51 | const browser_profile_t *browser_profile, 52 | bool http2_enabled, bool verify_cert, uint64_t *tls_time); 53 | 54 | /** 55 | * Calculate JA3 fingerprint from SSL connection 56 | * 57 | * @param ssl SSL connection 58 | * @param profile Browser profile used 59 | * @return JA3 fingerprint string (caller must free) or NULL on error 60 | */ 61 | char* httpmorph_calculate_ja3(SSL *ssl, const browser_profile_t *profile); 62 | 63 | #ifdef _WIN32 64 | /** 65 | * Load CA certificates from Windows Certificate Store into SSL_CTX 66 | * 67 | * @param ctx SSL context to load certificates into 68 | * @return 0 on success, -1 on error 69 | */ 70 | int httpmorph_load_windows_ca_certs(SSL_CTX *ctx); 71 | #endif 72 | 73 | #endif /* TLS_H */ 74 | -------------------------------------------------------------------------------- /src/core/internal/http2_logic.h: -------------------------------------------------------------------------------- 1 | /** 2 | * http2_logic.h - HTTP/2 protocol implementation 3 | */ 4 | 5 | #ifndef HTTP2_LOGIC_H 6 | #define HTTP2_LOGIC_H 7 | 8 | #include "internal.h" 9 | #include "request.h" 10 | #include "response.h" 11 | 12 | #ifdef HAVE_NGHTTP2 13 | #include 14 | #include 15 | 16 | /** 17 | * Perform HTTP/2 request over an existing TLS connection 18 | * 19 | * @param ssl SSL connection (must be established with HTTP/2 ALPN) 20 | * @param request Request object 21 | * @param host Target host (for :authority pseudo-header) 22 | * @param path URL path (for :path pseudo-header) 23 | * @param response Response object to populate 24 | * @return 0 on success, -1 on error 25 | */ 26 | int httpmorph_http2_request(SSL *ssl, const httpmorph_request_t *request, 27 | const char *host, const char *path, 28 | httpmorph_response_t *response); 29 | 30 | /** 31 | * Perform HTTP/2 request with session reuse 32 | * Reuses nghttp2_session from pooled connection if available 33 | * 34 | * @param conn Pooled connection (may have existing HTTP/2 session) 35 | * @param request Request object 36 | * @param host Target host (for :authority pseudo-header) 37 | * @param path URL path (for :path pseudo-header) 38 | * @param response Response object to populate 39 | * @return 0 on success, -1 on error 40 | */ 41 | int httpmorph_http2_request_pooled(struct pooled_connection *conn, 42 | const httpmorph_request_t *request, 43 | const char *host, const char *path, 44 | httpmorph_response_t *response); 45 | 46 | /** 47 | * Perform HTTP/2 request with concurrent multiplexing 48 | * Uses session manager to allow multiple concurrent streams on same session 49 | * This is the high-performance version for async/concurrent workloads 50 | * 51 | * @param conn Pooled connection with session manager 52 | * @param request Request object 53 | * @param host Target host (for :authority pseudo-header) 54 | * @param path URL path (for :path pseudo-header) 55 | * @param response Response object to populate 56 | * @return 0 on success, -1 on error 57 | */ 58 | int httpmorph_http2_request_concurrent(struct pooled_connection *conn, 59 | const httpmorph_request_t *request, 60 | const char *host, const char *path, 61 | httpmorph_response_t *response); 62 | 63 | #endif /* HAVE_NGHTTP2 */ 64 | 65 | #endif /* HTTP2_LOGIC_H */ 66 | -------------------------------------------------------------------------------- /src/httpmorph/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | httpmorph - Morph into any browser 3 | 4 | High-performance HTTP/HTTPS client with dynamic browser fingerprinting. 5 | 6 | Built from scratch in C with BoringSSL. No fallback implementations. 7 | """ 8 | 9 | # Read version from package metadata (single source of truth: pyproject.toml) 10 | try: 11 | from importlib.metadata import version as _get_version 12 | except ImportError: 13 | # Python < 3.8 14 | from importlib_metadata import version as _get_version 15 | 16 | __version__ = _get_version("httpmorph") 17 | __author__ = "Arman Hossain" 18 | __email__ = "arman@bytetunnels.com" 19 | __license__ = "MIT" 20 | 21 | # Import C implementation (required - no fallback!) 22 | from httpmorph._client_c import ( 23 | HAS_C_EXTENSION, 24 | Client, 25 | ConnectionError, 26 | HTTPError, 27 | PreparedRequest, 28 | Request, 29 | RequestException, 30 | Response, 31 | Session, 32 | Timeout, 33 | TooManyRedirects, 34 | cleanup, 35 | delete, 36 | get, 37 | head, 38 | init, 39 | options, 40 | patch, 41 | post, 42 | put, 43 | version, 44 | ) 45 | 46 | # Try to import HTTP/2 C extension (optional) 47 | try: 48 | from httpmorph import _http2 # noqa: F401 49 | 50 | HAS_HTTP2 = True 51 | except ImportError: 52 | HAS_HTTP2 = False 53 | 54 | # Auto-initialize 55 | init() 56 | 57 | # Confirm C extension loaded 58 | if not HAS_C_EXTENSION: 59 | raise RuntimeError( 60 | "httpmorph C extension failed to load. " 61 | "Please ensure the package was built correctly with: " 62 | "python setup.py build_ext --inplace" 63 | ) 64 | 65 | # Import async client (C-level async I/O with kqueue/epoll) 66 | try: 67 | from httpmorph._async_client import AsyncClient, AsyncResponse 68 | 69 | HAS_ASYNC = True 70 | except ImportError: 71 | AsyncClient = None 72 | AsyncResponse = None 73 | HAS_ASYNC = False 74 | 75 | __all__ = [ 76 | # Sync API 77 | "Client", 78 | "Session", 79 | "Response", 80 | "Request", 81 | "PreparedRequest", 82 | "get", 83 | "post", 84 | "put", 85 | "delete", 86 | "head", 87 | "patch", 88 | "options", 89 | # Async API 90 | "AsyncClient", 91 | "AsyncResponse", 92 | # Exceptions 93 | "HTTPError", 94 | "ConnectionError", 95 | "Timeout", 96 | "TooManyRedirects", 97 | "RequestException", 98 | # Utilities 99 | "init", 100 | "cleanup", 101 | "version", 102 | # Feature flags 103 | "HAS_HTTP2", 104 | "HAS_ASYNC", 105 | ] 106 | -------------------------------------------------------------------------------- /src/core/buffer_pool.h: -------------------------------------------------------------------------------- 1 | /** 2 | * buffer_pool.h - Buffer pooling for reduced allocation overhead 3 | * 4 | * Implements a slab allocator for common response buffer sizes. 5 | * Reuses buffers across requests to minimize malloc/free calls. 6 | */ 7 | 8 | #ifndef HTTPMORPH_BUFFER_POOL_H 9 | #define HTTPMORPH_BUFFER_POOL_H 10 | 11 | #include 12 | #include 13 | 14 | /* Buffer size tiers (powers of 2 for efficient allocation) */ 15 | #define BUFFER_SIZE_4KB 4096 16 | #define BUFFER_SIZE_16KB 16384 17 | #define BUFFER_SIZE_64KB 65536 18 | #define BUFFER_SIZE_256KB 262144 19 | 20 | /* Number of buffers to keep per size tier */ 21 | #define BUFFERS_PER_TIER 8 22 | 23 | /* Total number of size tiers */ 24 | #define NUM_TIERS 4 25 | 26 | /** 27 | * Buffer pool structure 28 | * Thread-safe buffer allocator with size-based tiers 29 | */ 30 | typedef struct httpmorph_buffer_pool httpmorph_buffer_pool_t; 31 | 32 | /** 33 | * Create a new buffer pool 34 | * 35 | * @return Initialized buffer pool or NULL on error 36 | */ 37 | httpmorph_buffer_pool_t* buffer_pool_create(void); 38 | 39 | /** 40 | * Destroy a buffer pool and free all resources 41 | * 42 | * @param pool Buffer pool to destroy 43 | */ 44 | void buffer_pool_destroy(httpmorph_buffer_pool_t *pool); 45 | 46 | /** 47 | * Allocate a buffer from the pool 48 | * 49 | * Returns a buffer of at least the requested size. May return a buffer 50 | * from the pool if one is available, otherwise allocates a new one. 51 | * 52 | * @param pool Buffer pool 53 | * @param size Minimum required size in bytes 54 | * @param actual_size Output parameter for actual buffer size (may be NULL) 55 | * @return Pointer to buffer or NULL on error 56 | */ 57 | void* buffer_pool_get(httpmorph_buffer_pool_t *pool, size_t size, size_t *actual_size); 58 | 59 | /** 60 | * Return a buffer to the pool 61 | * 62 | * Returns the buffer to the pool for reuse. If the pool for this size 63 | * is full, the buffer is freed instead. 64 | * 65 | * @param pool Buffer pool 66 | * @param buffer Buffer to return 67 | * @param size Size of the buffer (must match size used in buffer_pool_get) 68 | */ 69 | void buffer_pool_put(httpmorph_buffer_pool_t *pool, void *buffer, size_t size); 70 | 71 | /** 72 | * Get statistics about buffer pool usage 73 | * 74 | * @param pool Buffer pool 75 | * @param hits Output: number of successful pool retrievals 76 | * @param misses Output: number of new allocations 77 | * @param returns Output: number of buffers returned to pool 78 | */ 79 | void buffer_pool_stats(httpmorph_buffer_pool_t *pool, 80 | size_t *hits, size_t *misses, size_t *returns); 81 | 82 | #endif /* HTTPMORPH_BUFFER_POOL_H */ 83 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | httpmorph 2 | ========= 3 | 4 | A Python HTTP client library with browser fingerprinting capabilities, written in C for performance. 5 | 6 | .. code-block:: python 7 | 8 | import httpmorph 9 | 10 | # Simple GET request 11 | response = httpmorph.get('https://example.com') 12 | print(response.status_code, response.text) 13 | 14 | # Use a session with browser profile 15 | session = httpmorph.Session(browser='chrome') 16 | response = session.get('https://example.com') 17 | 18 | Features 19 | -------- 20 | 21 | * **C implementation** - Native C code with Python bindings for maximum performance 22 | * **Chrome 142 fingerprinting** - Perfect JA3N, JA4, and JA4_R matching 23 | * **HTTP/2 support** - Full HTTP/2 with ALPN negotiation via nghttp2 24 | * **TLS 1.3 with post-quantum crypto** - X25519MLKEM768 support 25 | * **Certificate compression** - Brotli and Zlib support for Cloudflare sites 26 | * **OS-specific user agents** - macOS, Windows, and Linux variants 27 | * **Connection pooling** - Automatic connection reuse 28 | * **Async support** - AsyncClient with epoll/kqueue 29 | * **Compression** - Automatic gzip/deflate decompression 30 | 31 | Requirements 32 | ------------ 33 | 34 | * Python 3.8+ 35 | * BoringSSL (built from source during installation) 36 | * libnghttp2 (for HTTP/2 support) 37 | 38 | Installation 39 | ------------ 40 | 41 | .. code-block:: bash 42 | 43 | pip install httpmorph 44 | 45 | See :doc:`installation` for build requirements and troubleshooting. 46 | 47 | Quick Example 48 | ------------- 49 | 50 | .. code-block:: python 51 | 52 | import httpmorph 53 | 54 | # GET request 55 | response = httpmorph.get('https://httpbin.org/get') 56 | print(response.json()) 57 | 58 | # POST with JSON 59 | response = httpmorph.post( 60 | 'https://httpbin.org/post', 61 | json={'key': 'value'} 62 | ) 63 | 64 | # Session with cookies 65 | session = httpmorph.Session(browser='chrome') 66 | response = session.get('https://example.com') 67 | print(session.cookies) 68 | 69 | # HTTP/2 70 | client = httpmorph.Client(http2=True) 71 | response = client.get('https://www.google.com') 72 | print(response.http_version) # '2.0' 73 | 74 | Documentation 75 | ------------- 76 | 77 | .. toctree:: 78 | :maxdepth: 2 79 | 80 | installation 81 | quickstart 82 | api 83 | advanced 84 | 85 | Status 86 | ------ 87 | 88 | httpmorph is under active development. The API may change between minor versions. 89 | 90 | License 91 | ------- 92 | 93 | MIT License 94 | 95 | Author 96 | ------ 97 | 98 | **Arman Hossain** 99 | 100 | - GitHub: `arman-bd `_ 101 | - Email: arman@bytetunnels.com 102 | -------------------------------------------------------------------------------- /src/core/request_builder.h: -------------------------------------------------------------------------------- 1 | /** 2 | * request_builder.h - Fast request building utilities 3 | * 4 | * Optimized buffer building using direct memory operations 5 | * instead of snprintf for better performance. 6 | */ 7 | 8 | #ifndef HTTPMORPH_REQUEST_BUILDER_H 9 | #define HTTPMORPH_REQUEST_BUILDER_H 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | /** 16 | * Dynamic buffer for building HTTP requests 17 | */ 18 | typedef struct { 19 | char *data; 20 | size_t len; 21 | size_t capacity; 22 | } request_builder_t; 23 | 24 | /** 25 | * Create a new request builder with initial capacity 26 | * 27 | * @param initial_capacity Initial buffer size 28 | * @return New request builder or NULL on error 29 | */ 30 | request_builder_t* request_builder_create(size_t initial_capacity); 31 | 32 | /** 33 | * Destroy a request builder and free resources 34 | * 35 | * @param builder Builder to destroy 36 | */ 37 | void request_builder_destroy(request_builder_t *builder); 38 | 39 | /** 40 | * Append a string to the builder (fast memcpy) 41 | * 42 | * @param builder Builder to append to 43 | * @param str String to append 44 | * @param len Length of string 45 | * @return 0 on success, -1 on error 46 | */ 47 | int request_builder_append(request_builder_t *builder, const char *str, size_t len); 48 | 49 | /** 50 | * Append a null-terminated string (calculates length) 51 | * 52 | * @param builder Builder to append to 53 | * @param str Null-terminated string to append 54 | * @return 0 on success, -1 on error 55 | */ 56 | int request_builder_append_str(request_builder_t *builder, const char *str); 57 | 58 | /** 59 | * Append an unsigned integer as decimal string 60 | * 61 | * @param builder Builder to append to 62 | * @param value Value to append 63 | * @return 0 on success, -1 on error 64 | */ 65 | int request_builder_append_uint(request_builder_t *builder, uint64_t value); 66 | 67 | /** 68 | * Append a formatted header line: "Key: Value\r\n" 69 | * 70 | * @param builder Builder to append to 71 | * @param key Header key 72 | * @param key_len Length of key 73 | * @param value Header value 74 | * @param value_len Length of value 75 | * @return 0 on success, -1 on error 76 | */ 77 | int request_builder_append_header(request_builder_t *builder, 78 | const char *key, size_t key_len, 79 | const char *value, size_t value_len); 80 | 81 | /** 82 | * Get the current buffer data 83 | * 84 | * @param builder Builder to get data from 85 | * @param len Output: length of data 86 | * @return Pointer to buffer data (valid until next append or destroy) 87 | */ 88 | const char* request_builder_data(const request_builder_t *builder, size_t *len); 89 | 90 | #endif /* HTTPMORPH_REQUEST_BUILDER_H */ 91 | -------------------------------------------------------------------------------- /tests/test_basic.py: -------------------------------------------------------------------------------- 1 | """ 2 | Basic tests for httpmorph 3 | """ 4 | 5 | import pytest 6 | 7 | import httpmorph 8 | 9 | 10 | def test_import(): 11 | """Test httpmorph can be imported""" 12 | assert httpmorph is not None 13 | 14 | 15 | def test_version(): 16 | """Test library version""" 17 | try: 18 | version = httpmorph.version() 19 | assert version is not None 20 | assert isinstance(version, str) 21 | print(f"httpmorph version: {version}") 22 | except (NotImplementedError, AttributeError): 23 | pytest.skip("version() not yet implemented") 24 | 25 | 26 | def test_init_cleanup(): 27 | """Test library initialization and cleanup""" 28 | try: 29 | httpmorph.init() 30 | httpmorph.cleanup() 31 | except (NotImplementedError, AttributeError): 32 | pytest.skip("init/cleanup not yet implemented") 33 | 34 | 35 | def test_client_creation(): 36 | """Test client can be created""" 37 | try: 38 | client = httpmorph.Client() 39 | assert client is not None 40 | except (NotImplementedError, AttributeError): 41 | pytest.skip("Client not yet implemented") 42 | 43 | 44 | def test_session_creation(): 45 | """Test session can be created with different browsers""" 46 | try: 47 | for browser in ["chrome", "firefox", "safari", "edge", "random"]: 48 | session = httpmorph.Session(browser=browser) 49 | assert session is not None 50 | except (NotImplementedError, AttributeError): 51 | pytest.skip("Session not yet implemented") 52 | 53 | 54 | def test_simple_get(httpbin_server): 55 | """Test simple GET request""" 56 | response = httpmorph.get(f"{httpbin_server}/get") 57 | assert response.status_code == 200 58 | assert response.body is not None 59 | 60 | 61 | def test_post_with_json(httpbin_server): 62 | """Test POST request with JSON data""" 63 | data = {"key": "value", "number": 42} 64 | response = httpmorph.post(f"{httpbin_server}/post", json=data) 65 | 66 | assert response.status_code in [200, 402] # httpbingo returns 402 for HTTP/2 67 | 68 | # Test compatibility: should work with both .body and .json() 69 | if hasattr(response, "json"): 70 | response_data = response.json() 71 | else: 72 | import json 73 | 74 | response_data = json.loads(response.body) 75 | 76 | assert response_data["json"] == data 77 | 78 | 79 | def test_fingerprint_rotation(): 80 | """Test that fingerprints are rotated correctly""" 81 | _session1 = httpmorph.Session(browser="chrome") 82 | _session2 = httpmorph.Session(browser="firefox") 83 | 84 | # Sessions with different browsers should have different fingerprints 85 | # This will be testable once we implement fingerprint tracking 86 | 87 | 88 | if __name__ == "__main__": 89 | pytest.main([__file__, "-v"]) 90 | -------------------------------------------------------------------------------- /src/core/iocp_dispatcher.h: -------------------------------------------------------------------------------- 1 | /** 2 | * IOCP Completion Dispatcher for Windows Async I/O 3 | * 4 | * Provides centralized completion handling for Windows IOCP operations. 5 | * Runs a dedicated thread that blocks on GetQueuedCompletionStatus and 6 | * dispatches completions to the appropriate async_request. 7 | */ 8 | 9 | #ifndef IOCP_DISPATCHER_H 10 | #define IOCP_DISPATCHER_H 11 | 12 | #ifdef _WIN32 13 | 14 | #include 15 | #include 16 | 17 | /* Forward declarations */ 18 | typedef struct io_engine io_engine_t; 19 | typedef struct async_request async_request_t; 20 | 21 | /** 22 | * Completion notification callback 23 | * Called when an IOCP operation completes 24 | * 25 | * @param request The request that owns the completed operation 26 | * @param bytes_transferred Number of bytes transferred 27 | * @param error Error code (0 for success) 28 | */ 29 | typedef void (*iocp_completion_callback_t)(async_request_t *request, 30 | uint32_t bytes_transferred, 31 | uint32_t error); 32 | 33 | /** 34 | * Initialize and start the IOCP dispatcher thread 35 | * 36 | * @param engine The I/O engine with IOCP handle 37 | * @return 0 on success, -1 on failure 38 | */ 39 | int iocp_dispatcher_start(io_engine_t *engine); 40 | 41 | /** 42 | * Stop the IOCP dispatcher thread and cleanup 43 | * 44 | * @param engine The I/O engine 45 | */ 46 | void iocp_dispatcher_stop(io_engine_t *engine); 47 | 48 | /** 49 | * Register a completion callback for a request 50 | * Called when initiating an async operation 51 | * 52 | * @param request The request initiating the operation 53 | * @param callback Callback to invoke when operation completes 54 | */ 55 | void iocp_dispatcher_register_callback(async_request_t *request, 56 | iocp_completion_callback_t callback); 57 | 58 | /** 59 | * Unregister a request's completion callback 60 | * Called when request is destroyed 61 | * 62 | * @param request The request to unregister 63 | */ 64 | void iocp_dispatcher_unregister_callback(async_request_t *request); 65 | 66 | /** 67 | * Check if dispatcher is running 68 | * 69 | * @param engine The I/O engine 70 | * @return true if dispatcher thread is running 71 | */ 72 | bool iocp_dispatcher_is_running(io_engine_t *engine); 73 | 74 | /** 75 | * Post a custom completion packet to IOCP 76 | * Used to wake up the dispatcher or send control messages 77 | * 78 | * @param engine The I/O engine 79 | * @param completion_key Custom completion key 80 | * @param bytes_transferred Bytes value for completion 81 | * @return 0 on success, -1 on failure 82 | */ 83 | int iocp_dispatcher_post_completion(io_engine_t *engine, 84 | uintptr_t completion_key, 85 | uint32_t bytes_transferred); 86 | 87 | #endif /* _WIN32 */ 88 | #endif /* IOCP_DISPATCHER_H */ 89 | -------------------------------------------------------------------------------- /src/core/async_request_manager.h: -------------------------------------------------------------------------------- 1 | /** 2 | * async_request_manager.h - Manager for multiple concurrent async requests 3 | */ 4 | 5 | #ifndef ASYNC_REQUEST_MANAGER_H 6 | #define ASYNC_REQUEST_MANAGER_H 7 | 8 | #include "async_request.h" 9 | #include "io_engine.h" 10 | #include 11 | #include 12 | 13 | #ifndef _WIN32 14 | #include 15 | #endif 16 | 17 | #ifdef __cplusplus 18 | extern "C" { 19 | #endif 20 | 21 | /* Forward declaration for SSL_CTX */ 22 | typedef struct ssl_ctx_st SSL_CTX; 23 | 24 | /** 25 | * Request manager structure 26 | */ 27 | typedef struct async_request_manager { 28 | /* I/O engine for event-driven I/O */ 29 | io_engine_t *io_engine; 30 | 31 | /* SSL/TLS context */ 32 | SSL_CTX *ssl_ctx; 33 | 34 | /* Request tracking */ 35 | async_request_t **requests; 36 | size_t request_count; 37 | size_t request_capacity; 38 | 39 | /* Request ID generation */ 40 | uint64_t next_request_id; 41 | 42 | /* Thread safety */ 43 | pthread_mutex_t mutex; 44 | 45 | /* Event loop (optional - for standalone mode) */ 46 | pthread_t event_thread; 47 | bool event_thread_running; 48 | bool shutdown; 49 | 50 | } async_request_manager_t; 51 | 52 | /** 53 | * Create a new async request manager 54 | */ 55 | async_request_manager_t* async_manager_create(void); 56 | 57 | /** 58 | * Destroy an async request manager 59 | */ 60 | void async_manager_destroy(async_request_manager_t *mgr); 61 | 62 | /** 63 | * Submit a new async request 64 | * Returns request ID (>0 on success, 0 on failure) 65 | */ 66 | uint64_t async_manager_submit_request( 67 | async_request_manager_t *mgr, 68 | const httpmorph_request_t *request, 69 | uint32_t timeout_ms, 70 | async_request_callback_t callback, 71 | void *user_data 72 | ); 73 | 74 | /** 75 | * Get request by ID 76 | */ 77 | async_request_t* async_manager_get_request( 78 | async_request_manager_t *mgr, 79 | uint64_t request_id 80 | ); 81 | 82 | /** 83 | * Cancel a request 84 | */ 85 | int async_manager_cancel_request( 86 | async_request_manager_t *mgr, 87 | uint64_t request_id 88 | ); 89 | 90 | /** 91 | * Poll for events (non-blocking) 92 | * Returns number of events processed 93 | */ 94 | int async_manager_poll( 95 | async_request_manager_t *mgr, 96 | uint32_t timeout_ms 97 | ); 98 | 99 | /** 100 | * Process all pending requests 101 | * This is the main event loop function 102 | */ 103 | int async_manager_process(async_request_manager_t *mgr); 104 | 105 | /** 106 | * Get number of active requests 107 | */ 108 | size_t async_manager_get_active_count(const async_request_manager_t *mgr); 109 | 110 | /** 111 | * Start event loop thread (optional) 112 | */ 113 | int async_manager_start_event_loop(async_request_manager_t *mgr); 114 | 115 | /** 116 | * Stop event loop thread 117 | */ 118 | int async_manager_stop_event_loop(async_request_manager_t *mgr); 119 | 120 | #ifdef __cplusplus 121 | } 122 | #endif 123 | 124 | #endif /* ASYNC_REQUEST_MANAGER_H */ 125 | -------------------------------------------------------------------------------- /docs/source/installation.rst: -------------------------------------------------------------------------------- 1 | Installation 2 | ============ 3 | 4 | Basic Installation 5 | ------------------ 6 | 7 | Install from PyPI: 8 | 9 | .. code-block:: bash 10 | 11 | pip install httpmorph 12 | 13 | This will install pre-built wheels for: 14 | 15 | * Windows (x86_64) 16 | * macOS (Intel and Apple Silicon) 17 | * Linux (x86_64, ARM64) 18 | 19 | Build Requirements 20 | ------------------ 21 | 22 | If building from source, you'll need: 23 | 24 | **macOS:** 25 | 26 | .. code-block:: bash 27 | 28 | brew install cmake ninja libnghttp2 29 | 30 | **Linux (Ubuntu/Debian):** 31 | 32 | .. code-block:: bash 33 | 34 | sudo apt-get install cmake ninja-build libssl-dev pkg-config \ 35 | autoconf automake libtool libnghttp2-dev 36 | 37 | **Linux (Fedora/RHEL):** 38 | 39 | .. code-block:: bash 40 | 41 | sudo dnf install cmake ninja-build openssl-devel pkg-config \ 42 | autoconf automake libtool libnghttp2-devel 43 | 44 | **Windows:** 45 | 46 | .. code-block:: bash 47 | 48 | choco install cmake golang nasm visualstudio2022buildtools -y 49 | 50 | Building from Source 51 | -------------------- 52 | 53 | .. code-block:: bash 54 | 55 | git clone https://github.com/arman-bd/httpmorph.git 56 | cd httpmorph 57 | 58 | # Build vendor dependencies (BoringSSL, nghttp2) 59 | ./scripts/setup_vendors.sh 60 | 61 | # Build Python extensions 62 | python setup.py build_ext --inplace 63 | 64 | # Install in development mode 65 | pip install -e ".[dev]" 66 | 67 | The first build takes 5-10 minutes to compile BoringSSL. Subsequent builds are faster. 68 | 69 | Dependencies 70 | ------------ 71 | 72 | httpmorph has no Python runtime dependencies. All required libraries are built from source: 73 | 74 | * **BoringSSL** - TLS implementation (built from source) 75 | * **nghttp2** - HTTP/2 library (system or built from source) 76 | * **zlib** - Compression support (system library) 77 | 78 | Optional Dependencies 79 | --------------------- 80 | 81 | For development: 82 | 83 | .. code-block:: bash 84 | 85 | pip install httpmorph[dev] 86 | 87 | This includes: 88 | 89 | * pytest - Testing framework 90 | * pytest-asyncio - Async test support 91 | * pytest-benchmark - Performance testing 92 | * pytest-cov - Code coverage 93 | * mypy - Type checking 94 | * ruff - Linting and formatting 95 | 96 | Troubleshooting 97 | --------------- 98 | 99 | **Import Error on Linux:** 100 | 101 | If you see ``ImportError: cannot open shared object file``, install nghttp2: 102 | 103 | .. code-block:: bash 104 | 105 | # Ubuntu/Debian 106 | sudo apt-get install libnghttp2-14 107 | 108 | # Fedora/RHEL 109 | sudo dnf install libnghttp2 110 | 111 | **Build Errors on macOS:** 112 | 113 | Make sure Xcode Command Line Tools are installed: 114 | 115 | .. code-block:: bash 116 | 117 | xcode-select --install 118 | 119 | **Build Errors on Windows:** 120 | 121 | Ensure Visual Studio 2019+ with C++ build tools is installed. 122 | 123 | Verifying Installation 124 | ---------------------- 125 | 126 | .. code-block:: python 127 | 128 | import httpmorph 129 | 130 | print(httpmorph.__version__) 131 | print(httpmorph.version()) # C library version 132 | 133 | # Test basic functionality 134 | response = httpmorph.get('https://httpbin.org/get') 135 | assert response.status_code == 200 136 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: help setup build install test clean benchmark docs lint format sync docker-build docker-test docker-shell 2 | 3 | help: 4 | @echo "httpmorph - Development commands" 5 | @echo "" 6 | @echo "Setup:" 7 | @echo " make sync - Sync dependencies with uv" 8 | @echo " make setup - Setup vendor dependencies (llhttp, BoringSSL, liburing)" 9 | @echo " make build - Build C extensions" 10 | @echo " make install - Install in development mode" 11 | @echo "" 12 | @echo "Development:" 13 | @echo " make test - Run tests" 14 | @echo " make benchmark - Run benchmarks" 15 | @echo " make lint - Run linters (ruff, mypy)" 16 | @echo " make format - Format code (ruff)" 17 | @echo " make check-windows - Quick Windows compatibility check (no Docker)" 18 | @echo "" 19 | @echo "Docker (CI Testing):" 20 | @echo " make docker-build - Build Docker test container (mimics CI)" 21 | @echo " make docker-test - Run tests in Docker" 22 | @echo " make docker-shell - Open shell in Docker for debugging" 23 | @echo "" 24 | @echo "Cleanup:" 25 | @echo " make clean - Remove build artifacts" 26 | @echo " make clean-all - Remove build artifacts and vendor dependencies" 27 | 28 | sync: 29 | @echo "Syncing dependencies with uv..." 30 | uv sync --extra dev --extra build 31 | 32 | setup: 33 | @echo "Setting up vendor dependencies..." 34 | ./scripts/setup_vendors.sh 35 | 36 | build: 37 | @echo "Building C extensions..." 38 | uv run python setup.py build_ext --inplace 39 | 40 | install: sync setup 41 | @echo "Installing in development mode..." 42 | uv pip install -e ".[dev,build]" 43 | 44 | test: 45 | @echo "Running tests..." 46 | uv run pytest tests/ -v 47 | 48 | test-verbose: 49 | @echo "Running tests with verbose output..." 50 | uv run pytest tests/ -vv -s 51 | 52 | benchmark: 53 | @echo "Running benchmarks..." 54 | uv run pytest benchmarks/ -v --benchmark-only 55 | 56 | lint: 57 | @echo "Running linters..." 58 | uv run ruff check src/ tests/ 59 | uv run mypy src/ 60 | 61 | format: 62 | @echo "Formatting code with ruff..." 63 | uv run ruff format src/ tests/ benchmarks/ 64 | uv run ruff check --fix src/ tests/ 65 | 66 | clean: 67 | @echo "Cleaning build artifacts..." 68 | rm -rf build/ 69 | rm -rf dist/ 70 | rm -rf *.egg-info 71 | rm -rf src/bindings/*.c 72 | rm -rf src/**/*.so 73 | rm -rf src/**/*.html 74 | find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true 75 | find . -type f -name "*.pyc" -delete 76 | 77 | clean-all: clean 78 | @echo "Cleaning vendor dependencies..." 79 | rm -rf vendor/ 80 | 81 | # Quick development workflow 82 | dev: clean build test 83 | 84 | # Full rebuild 85 | rebuild: clean setup build 86 | 87 | # Check if everything is working 88 | check: lint test 89 | @echo "All checks passed!" 90 | 91 | # Quick Windows compatibility check (no Docker needed) 92 | check-windows: 93 | @echo "Running Windows compatibility check..." 94 | @./scripts/check_windows_compat.sh 95 | 96 | # Docker targets for CI testing 97 | docker-build: 98 | @echo "Building Docker test container (mimics GitHub Actions)..." 99 | docker build -f docker/Dockerfile.test -t httpmorph-test . 100 | 101 | docker-test: docker-build 102 | @echo "Running tests in Docker container..." 103 | docker run --rm httpmorph-test pytest tests/ -v 104 | 105 | docker-shell: 106 | @echo "Opening shell in Docker container for debugging..." 107 | docker run --rm -it -v $(PWD):/workspace httpmorph-test bash 108 | -------------------------------------------------------------------------------- /scripts/windows/test_build_local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Bash script to build and test httpmorph wheels on Windows 3 | # Mimics the GitHub Actions release pipeline (.github/workflows/release.yml) 4 | # Works with Git Bash, MSYS2, or WSL on Windows 5 | 6 | set -e 7 | 8 | SKIP_TESTS=false 9 | 10 | # Parse arguments 11 | while [[ $# -gt 0 ]]; do 12 | case $1 in 13 | --skip-tests) 14 | SKIP_TESTS=true 15 | shift 16 | ;; 17 | *) 18 | echo "Unknown option: $1" 19 | echo "Usage: $0 [--skip-tests]" 20 | exit 1 21 | ;; 22 | esac 23 | done 24 | 25 | echo "========================================" 26 | echo "httpmorph - Windows Wheel Builder" 27 | echo "========================================" 28 | echo "" 29 | 30 | SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 31 | PROJECT_ROOT="$(dirname "$(dirname "$SCRIPT_DIR")")" 32 | 33 | # Step 1: Setup vendors 34 | echo "==> Step 1: Setting up vendor dependencies..." 35 | 36 | # Check for required tools 37 | echo " Checking dependencies..." 38 | 39 | # Add common tool paths 40 | export PATH="/c/Program Files/CMake/bin:/c/Program Files/Go/bin:$PATH" 41 | 42 | if ! command -v cmake &> /dev/null; then 43 | echo "[FAIL] cmake not found. Install with: choco install cmake" 44 | exit 1 45 | fi 46 | echo " [OK] cmake found" 47 | 48 | if ! command -v go &> /dev/null; then 49 | echo "[FAIL] go not found. Install with: choco install golang" 50 | exit 1 51 | fi 52 | echo " [OK] go found" 53 | 54 | # Check vcpkg 55 | if [ ! -f "/c/vcpkg/vcpkg.exe" ]; then 56 | echo "[FAIL] vcpkg not found at C:\\vcpkg" 57 | echo " Install vcpkg and nghttp2/zlib:" 58 | echo " git clone https://github.com/Microsoft/vcpkg.git C:\\vcpkg" 59 | echo " C:\\vcpkg\\bootstrap-vcpkg.bat" 60 | echo " C:\\vcpkg\\vcpkg install nghttp2:x64-windows zlib:x64-windows" 61 | exit 1 62 | fi 63 | echo " [OK] vcpkg found" 64 | 65 | # Build vendors 66 | echo "" 67 | echo " Building vendor dependencies..." 68 | cd "$PROJECT_ROOT" 69 | bash scripts/windows/setup_vendors.sh 70 | echo " [OK] Vendors built successfully" 71 | 72 | # Step 2: Build wheel 73 | echo "" 74 | echo "==> Step 2: Building wheel..." 75 | 76 | export VCPKG_ROOT="C:/vcpkg" 77 | 78 | cd "$PROJECT_ROOT" 79 | 80 | # Find Python 81 | if [ -f "/c/Python311/python.exe" ]; then 82 | PYTHON="/c/Python311/python.exe" 83 | elif command -v python &> /dev/null; then 84 | PYTHON="python" 85 | else 86 | echo "[FAIL] Python not found" 87 | exit 1 88 | fi 89 | 90 | # Build wheel 91 | "$PYTHON" setup.py bdist_wheel 92 | 93 | # Find the built wheel 94 | WHEEL=$(ls -t "$PROJECT_ROOT/dist"/*.whl 2>/dev/null | head -1) 95 | 96 | if [ -z "$WHEEL" ]; then 97 | echo "[FAIL] No wheel found in dist/" 98 | exit 1 99 | fi 100 | 101 | echo " [OK] Wheel built: $(basename "$WHEEL")" 102 | 103 | # Step 3: Test wheel 104 | if [ "$SKIP_TESTS" = false ]; then 105 | echo "" 106 | echo "==> Step 3: Testing wheel..." 107 | 108 | # Install wheel 109 | echo " Installing wheel..." 110 | "$PYTHON" -m pip install --force-reinstall "$WHEEL" 111 | 112 | # Run tests 113 | echo "" 114 | echo " Running tests..." 115 | "$PYTHON" "$PROJECT_ROOT/scripts/test_local_build.py" 116 | 117 | echo "" 118 | echo " [OK] All tests passed!" 119 | fi 120 | 121 | # Summary 122 | echo "" 123 | echo "========================================" 124 | echo "BUILD COMPLETE" 125 | echo "========================================" 126 | echo "" 127 | echo "Wheel: $WHEEL" 128 | echo "Size: $(ls -lh "$WHEEL" | awk '{print $5}')" 129 | echo "" 130 | echo "To install:" 131 | echo " pip install $WHEEL" 132 | echo "" 133 | 134 | exit 0 135 | -------------------------------------------------------------------------------- /.github/workflows/_config.yml: -------------------------------------------------------------------------------- 1 | # 2 | # Global CI/CD Configuration 3 | # 4 | # This is a reusable workflow config that defines common settings across all workflows. 5 | # Edit this file to control which OS and Python versions to test against. 6 | # 7 | # During development, you can comment out OS/Python versions to speed up CI runs. 8 | # 9 | 10 | name: Global Config 11 | 12 | on: 13 | workflow_call: 14 | outputs: 15 | # OS matrix - comment out any you don't want to test 16 | os-matrix: 17 | description: "Operating systems to test" 18 | value: ${{ jobs.config.outputs.os-matrix }} 19 | 20 | # Python versions - comment out any you don't want to test 21 | python-matrix: 22 | description: "Python versions to test" 23 | value: ${{ jobs.config.outputs.python-matrix }} 24 | 25 | # Primary OS/Python for single-runner jobs (coverage, etc.) 26 | primary-os: 27 | description: "Primary OS for single-runner jobs" 28 | value: ${{ jobs.config.outputs.primary-os }} 29 | 30 | primary-python: 31 | description: "Primary Python version for single-runner jobs" 32 | value: ${{ jobs.config.outputs.primary-python }} 33 | 34 | # Job toggles 35 | enable-tests: 36 | description: "Enable test job" 37 | value: ${{ jobs.config.outputs.enable-tests }} 38 | 39 | jobs: 40 | config: 41 | runs-on: ubuntu-latest 42 | outputs: 43 | os-matrix: ${{ steps.set-config.outputs.os-matrix }} 44 | python-matrix: ${{ steps.set-config.outputs.python-matrix }} 45 | primary-os: ${{ steps.set-config.outputs.primary-os }} 46 | primary-python: ${{ steps.set-config.outputs.primary-python }} 47 | enable-tests: ${{ steps.set-config.outputs.enable-tests }} 48 | 49 | steps: 50 | - id: set-config 51 | run: | 52 | # ============================================================ 53 | # CONFIGURATION - Edit this section 54 | # ============================================================ 55 | 56 | # Operating Systems to test 57 | # Comment out any OS you want to skip during development 58 | OS_MATRIX='[ 59 | "ubuntu-latest", "ubuntu-24.04-arm", "macos-latest", "macos-15-intel", "windows-latest" 60 | ]' 61 | # OS options: "windows-latest", "ubuntu-latest", "ubuntu-24.04-arm", "macos-latest", "macos-15-intel" 62 | 63 | # Python versions to test 64 | PYTHON_MATRIX='[ 65 | "3.8", "3.11", "3.14" 66 | ]' 67 | # Python versions: "3.8", "3.9", "3.10", "3.11", "3.12", "3.13", "3.14" 68 | 69 | # Primary OS and Python for single-runner jobs 70 | PRIMARY_OS="ubuntu-latest" 71 | PRIMARY_PYTHON="3.11" 72 | 73 | # Enable/Disable Jobs 74 | # Set to "true" or "false" 75 | ENABLE_TESTS="true" 76 | 77 | # ============================================================ 78 | # END CONFIGURATION 79 | # ============================================================ 80 | 81 | # Compact JSON (remove newlines and extra spaces) 82 | OS_MATRIX_COMPACT=$(echo "$OS_MATRIX" | jq -c .) 83 | PYTHON_MATRIX_COMPACT=$(echo "$PYTHON_MATRIX" | jq -c .) 84 | 85 | echo "os-matrix=$OS_MATRIX_COMPACT" >> $GITHUB_OUTPUT 86 | echo "python-matrix=$PYTHON_MATRIX_COMPACT" >> $GITHUB_OUTPUT 87 | echo "primary-os=$PRIMARY_OS" >> $GITHUB_OUTPUT 88 | echo "primary-python=$PRIMARY_PYTHON" >> $GITHUB_OUTPUT 89 | echo "enable-tests=$ENABLE_TESTS" >> $GITHUB_OUTPUT 90 | 91 | echo "✓ Configuration loaded:" 92 | echo " OS: $OS_MATRIX_COMPACT" 93 | echo " Python: $PYTHON_MATRIX_COMPACT" 94 | echo " Primary: $PRIMARY_OS / $PRIMARY_PYTHON" 95 | echo " Tests: $ENABLE_TESTS" 96 | -------------------------------------------------------------------------------- /examples/async_example.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example demonstrating AsyncClient with C-level async I/O 3 | 4 | This example shows how to use httpmorph's true async I/O capabilities 5 | without thread pool overhead. 6 | """ 7 | 8 | import asyncio 9 | 10 | import httpmorph 11 | 12 | 13 | async def main(): 14 | """Main async function demonstrating AsyncClient""" 15 | 16 | print("=== httpmorph AsyncClient Demo ===\n") 17 | 18 | # Check if async bindings are available 19 | if not httpmorph.HAS_ASYNC: 20 | print("❌ Async bindings not available.") 21 | print("Please rebuild: python setup.py build_ext --inplace") 22 | return 23 | 24 | print("✅ Async bindings loaded successfully!\n") 25 | 26 | # Use AsyncClient with context manager 27 | async with httpmorph.AsyncClient() as client: 28 | print("📡 Making async GET request to https://httpbin.org/get...") 29 | 30 | try: 31 | # Make an async GET request 32 | response = await client.get("https://httpbin.org/get", timeout=10.0) 33 | 34 | print("\n✅ Response received!") 35 | print(f" Status Code: {response.status_code}") 36 | print(f" HTTP Version: {response.http_version}") 37 | print(f" Total Time: {response.elapsed.total_seconds():.3f}s") 38 | 39 | # Show timing breakdown 40 | print("\n⏱️ Timing Breakdown:") 41 | print(f" Connect: {response.connect_time_us / 1000:.2f}ms") 42 | print(f" TLS: {response.tls_time_us / 1000:.2f}ms") 43 | print(f" First Byte: {response.first_byte_time_us / 1000:.2f}ms") 44 | print(f" Total: {response.total_time_us / 1000:.2f}ms") 45 | 46 | # Show TLS info 47 | if response.tls_version: 48 | print("\n🔒 TLS Info:") 49 | print(f" Version: {response.tls_version}") 50 | print(f" Cipher: {response.tls_cipher}") 51 | 52 | # Parse JSON response 53 | data = response.json() 54 | print("\n📊 Response Data:") 55 | print(f" Origin: {data.get('origin', 'N/A')}") 56 | print(f" Headers: {len(data.get('headers', {}))}") 57 | 58 | except asyncio.TimeoutError: 59 | print("\n❌ Request timed out") 60 | except Exception as e: 61 | print(f"\n❌ Error: {e}") 62 | 63 | print("\n=== Demo Complete ===") 64 | 65 | 66 | async def concurrent_requests_demo(): 67 | """Demo showing concurrent requests with AsyncClient""" 68 | 69 | print("\n=== Concurrent Requests Demo ===\n") 70 | 71 | if not httpmorph.HAS_ASYNC: 72 | return 73 | 74 | async with httpmorph.AsyncClient() as client: 75 | # Make multiple concurrent requests 76 | urls = [ 77 | "https://httpbin.org/delay/1", 78 | "https://httpbin.org/delay/2", 79 | "https://httpbin.org/delay/1", 80 | ] 81 | 82 | print(f"🚀 Making {len(urls)} concurrent requests...") 83 | start_time = asyncio.get_event_loop().time() 84 | 85 | # Create tasks for all requests 86 | tasks = [client.get(url, timeout=10.0) for url in urls] 87 | 88 | # Wait for all to complete 89 | responses = await asyncio.gather(*tasks, return_exceptions=True) 90 | 91 | end_time = asyncio.get_event_loop().time() 92 | total_time = end_time - start_time 93 | 94 | # Show results 95 | print(f"\n✅ All requests completed in {total_time:.2f}s\n") 96 | 97 | for i, result in enumerate(responses): 98 | if isinstance(result, Exception): 99 | print(f" Request {i + 1}: ❌ {result}") 100 | else: 101 | print( 102 | f" Request {i + 1}: ✅ {result.status_code} ({result.elapsed.total_seconds():.2f}s)" 103 | ) 104 | 105 | print(f"\n💡 Note: With thread pool, this would take ~{sum([1, 2, 1])}s") 106 | print(f" With async I/O, it took {total_time:.2f}s (concurrent!)") 107 | 108 | 109 | if __name__ == "__main__": 110 | # Run basic demo 111 | asyncio.run(main()) 112 | 113 | # Run concurrent requests demo 114 | # asyncio.run(concurrent_requests_demo()) 115 | -------------------------------------------------------------------------------- /scripts/darwin/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | macOS (Darwin)-specific build configuration for httpmorph. 4 | 5 | This module handles macOS-specific: 6 | - Library path detection (BoringSSL, nghttp2) 7 | - Clang compiler flags 8 | - Universal binary support 9 | """ 10 | 11 | import os 12 | import subprocess 13 | from pathlib import Path 14 | 15 | 16 | def get_library_paths(): 17 | """Get macOS-specific library paths for BoringSSL and nghttp2.""" 18 | # Use vendor-built libraries for wheel compatibility 19 | vendor_dir = Path("vendor").resolve() 20 | 21 | # BoringSSL (always vendor-built) 22 | vendor_boringssl = vendor_dir / "boringssl" 23 | boringssl_include = str(vendor_boringssl / "include") 24 | 25 | # Check where BoringSSL actually built the libraries 26 | # It could be in build/ssl/, build/crypto/, or just build/ 27 | build_dir = vendor_boringssl / "build" 28 | 29 | # Debug: list what's actually in the build directory 30 | if build_dir.exists(): 31 | print("\n=== BoringSSL build directory contents ===") 32 | for item in os.listdir(build_dir): 33 | item_path = build_dir / item 34 | if item_path.is_dir(): 35 | print(f" DIR: {item}/") 36 | # Check for .a files in subdirectories 37 | try: 38 | for subitem in os.listdir(item_path): 39 | if subitem.endswith(".a"): 40 | print(f" LIB: {subitem}") 41 | except PermissionError: 42 | pass 43 | elif item.endswith(".a"): 44 | print(f" LIB: {item}") 45 | print("=" * 43 + "\n") 46 | 47 | # Determine library directory based on what exists 48 | if (build_dir / "ssl" / "libssl.a").exists(): 49 | boringssl_lib = str(build_dir / "ssl") 50 | print(f"Using BoringSSL from: {boringssl_lib}") 51 | elif (build_dir / "libssl.a").exists(): 52 | boringssl_lib = str(build_dir) 53 | print(f"Using BoringSSL from: {boringssl_lib}") 54 | else: 55 | # Fallback to build/ssl even if it doesn't exist yet 56 | boringssl_lib = str(build_dir / "ssl") 57 | print(f"WARNING: BoringSSL libraries not found, using default: {boringssl_lib}") 58 | 59 | # nghttp2 - prefer vendor build for wheel compatibility 60 | vendor_nghttp2 = vendor_dir / "nghttp2" / "install" 61 | if vendor_nghttp2.exists() and (vendor_nghttp2 / "include").exists(): 62 | print(f"Using vendor nghttp2 from: {vendor_nghttp2}") 63 | nghttp2_include = str(vendor_nghttp2 / "include") 64 | nghttp2_lib = str(vendor_nghttp2 / "lib") 65 | else: 66 | # Fall back to Homebrew if vendor not available 67 | try: 68 | nghttp2_prefix = ( 69 | subprocess.check_output( 70 | ["brew", "--prefix", "libnghttp2"], stderr=subprocess.DEVNULL 71 | ) 72 | .decode() 73 | .strip() 74 | ) 75 | nghttp2_include = f"{nghttp2_prefix}/include" 76 | nghttp2_lib = f"{nghttp2_prefix}/lib" 77 | except (subprocess.CalledProcessError, FileNotFoundError): 78 | nghttp2_include = "/opt/homebrew/opt/libnghttp2/include" 79 | nghttp2_lib = "/opt/homebrew/opt/libnghttp2/lib" 80 | 81 | return { 82 | "openssl_include": boringssl_include, 83 | "openssl_lib": boringssl_lib, 84 | "nghttp2_include": nghttp2_include, 85 | "nghttp2_lib": nghttp2_lib, 86 | } 87 | 88 | 89 | def get_compile_args(): 90 | """Get macOS-specific compiler arguments.""" 91 | return [ 92 | "-std=c11", 93 | "-O2", 94 | "-DHAVE_NGHTTP2", 95 | ] 96 | 97 | 98 | def get_link_args(): 99 | """Get macOS-specific linker arguments.""" 100 | return [] 101 | 102 | 103 | def get_libraries(): 104 | """Get macOS-specific libraries to link against.""" 105 | return ["ssl", "crypto", "nghttp2", "z"] 106 | 107 | 108 | def get_extra_objects(): 109 | """Get extra object files to link (macOS doesn't need this).""" 110 | return [] 111 | 112 | 113 | def get_language(): 114 | """Get language for compilation (C for macOS).""" 115 | return "c" 116 | -------------------------------------------------------------------------------- /.github/workflows/_build_linux_x86_64.yml: -------------------------------------------------------------------------------- 1 | name: Build Linux x86_64 Wheels 2 | 3 | on: 4 | workflow_call: 5 | 6 | jobs: 7 | build-linux-x86_64: 8 | name: Build Linux x86_64 Wheels 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v4 13 | with: 14 | submodules: recursive 15 | 16 | # Setup Go for BoringSSL build 17 | - name: Setup Go 18 | uses: actions/setup-go@v5 19 | with: 20 | go-version: '1.21' 21 | cache: true 22 | 23 | # Build wheels with cibuildwheel (handles manylinux containers) 24 | # Vendor dependencies are built inside the manylinux container and cached across Python versions 25 | # Note: GitHub Actions cache doesn't work here since vendor dir is inside the container 26 | - name: Build wheels 27 | uses: pypa/cibuildwheel@v3.3 28 | env: 29 | # Build for all Python versions on x86_64 30 | CIBW_BUILD: cp38-manylinux_x86_64 cp39-manylinux_x86_64 cp310-manylinux_x86_64 cp311-manylinux_x86_64 cp312-manylinux_x86_64 cp313-manylinux_x86_64 cp314-manylinux_x86_64 31 | # Vendor build happens inside manylinux container via before-build (from pyproject.toml) 32 | # The script will detect cached libraries and skip rebuilding across Python versions 33 | CIBW_ARCHS_LINUX: x86_64 34 | 35 | # Setup Python versions for testing 36 | - name: Setup Python versions 37 | uses: actions/setup-python@v5 38 | with: 39 | python-version: | 40 | 3.8 41 | 3.9 42 | 3.10 43 | 3.11 44 | 3.12 45 | 3.13 46 | 3.14 47 | allow-prereleases: true 48 | 49 | # Test the built wheels 50 | - name: Test wheels 51 | run: | 52 | # Test each wheel that was built 53 | for wheel in ./wheelhouse/*.whl; do 54 | echo "========================================" 55 | echo "Testing wheel: $(basename "$wheel")" 56 | echo "========================================" 57 | 58 | # Extract Python version from wheel filename (e.g., cp39, cp310) 59 | python_tag=$(basename "$wheel" | grep -oE 'cp[0-9]+' | head -1) 60 | python_version="${python_tag:2:1}.${python_tag:3}" 61 | 62 | echo "Setting up Python $python_version..." 63 | 64 | # Try to find the matching Python version 65 | python_cmd="" 66 | for cmd in "python${python_version}" "python3.${python_version#*.}" "python3"; do 67 | if command -v "$cmd" &> /dev/null; then 68 | # Verify this is the right version 69 | version_check=$($cmd --version 2>&1 | grep -oE '[0-9]+\.[0-9]+' | head -1) 70 | if [ "$version_check" = "$python_version" ]; then 71 | python_cmd="$cmd" 72 | break 73 | fi 74 | fi 75 | done 76 | 77 | # Fall back to python3 if we couldn't find exact match 78 | if [ -z "$python_cmd" ]; then 79 | echo "WARNING: Python $python_version not found, using python3" 80 | python_cmd="python3" 81 | fi 82 | 83 | echo "Using Python: $($python_cmd --version)" 84 | 85 | # Install the wheel in a fresh environment 86 | "$python_cmd" -m pip install --force-reinstall "$wheel" 87 | 88 | # Run the test script 89 | echo "" 90 | echo "Running tests..." 91 | "$python_cmd" scripts/test_local_build.py 92 | 93 | # Check exit code 94 | if [ $? -eq 0 ]; then 95 | echo "" 96 | echo "[OK] Wheel test PASSED: $(basename "$wheel")" 97 | else 98 | echo "" 99 | echo "[FAIL] Wheel test FAILED: $(basename "$wheel")" 100 | exit 1 101 | fi 102 | 103 | echo "" 104 | done 105 | 106 | echo "========================================" 107 | echo "All wheel tests PASSED" 108 | echo "========================================" 109 | 110 | # Upload wheels as artifacts 111 | - name: Upload wheels 112 | uses: actions/upload-artifact@v4 113 | with: 114 | name: wheels-linux-x86_64 115 | path: ./wheelhouse/*.whl 116 | if-no-files-found: error 117 | retention-days: 5 118 | # Retry on transient failures 119 | continue-on-error: false 120 | -------------------------------------------------------------------------------- /src/core/proxy.c: -------------------------------------------------------------------------------- 1 | /** 2 | * proxy.c - HTTP proxy handling 3 | */ 4 | 5 | #include "internal/proxy.h" 6 | #include "internal/util.h" 7 | 8 | /** 9 | * Parse a proxy URL 10 | */ 11 | int httpmorph_parse_proxy_url(const char *proxy_url, char **host, uint16_t *port, 12 | char **username, char **password, bool *use_tls) { 13 | if (!proxy_url || !host || !port) { 14 | return -1; 15 | } 16 | 17 | *host = NULL; 18 | *port = 0; 19 | if (username) *username = NULL; 20 | if (password) *password = NULL; 21 | if (use_tls) *use_tls = false; 22 | 23 | /* Parse proxy URL format: [http://][username:password@]host:port */ 24 | const char *start = proxy_url; 25 | 26 | /* Skip http:// or https:// and detect if proxy uses TLS */ 27 | if (strncmp(start, "http://", 7) == 0) { 28 | start += 7; 29 | if (use_tls) *use_tls = false; 30 | } else if (strncmp(start, "https://", 8) == 0) { 31 | start += 8; 32 | if (use_tls) *use_tls = true; 33 | } 34 | 35 | /* Check for username:password@ */ 36 | const char *at_sign = strchr(start, '@'); 37 | if (at_sign && username && password) { 38 | const char *colon = strchr(start, ':'); 39 | if (colon && colon < at_sign) { 40 | size_t user_len = colon - start; 41 | size_t pass_len = at_sign - colon - 1; 42 | 43 | *username = strndup(start, user_len); 44 | *password = strndup(colon + 1, pass_len); 45 | 46 | start = at_sign + 1; 47 | } 48 | } 49 | 50 | /* Parse host:port */ 51 | const char *colon = strchr(start, ':'); 52 | if (colon) { 53 | size_t host_len = colon - start; 54 | *host = strndup(start, host_len); 55 | *port = (uint16_t)atoi(colon + 1); 56 | } else { 57 | *host = strdup(start); 58 | *port = 8080; /* Default proxy port */ 59 | } 60 | 61 | return (*host != NULL) ? 0 : -1; 62 | } 63 | 64 | /** 65 | * Send HTTP CONNECT request to establish tunnel through proxy 66 | */ 67 | int httpmorph_proxy_connect(int sockfd, SSL *proxy_ssl, const char *target_host, 68 | uint16_t target_port, const char *proxy_username, 69 | const char *proxy_password, uint32_t timeout_ms) { 70 | char connect_req[2048]; 71 | int len; 72 | 73 | /* Build CONNECT request */ 74 | len = snprintf(connect_req, sizeof(connect_req), 75 | "CONNECT %s:%u HTTP/1.1\r\n" 76 | "Host: %s:%u\r\n", 77 | target_host, target_port, target_host, target_port); 78 | 79 | /* Add Proxy-Authorization if credentials provided */ 80 | if (proxy_username && proxy_password) { 81 | char credentials[512]; 82 | snprintf(credentials, sizeof(credentials), "%s:%s", proxy_username, proxy_password); 83 | 84 | char *encoded = httpmorph_base64_encode(credentials, strlen(credentials)); 85 | if (encoded) { 86 | len += snprintf(connect_req + len, sizeof(connect_req) - len, 87 | "Proxy-Authorization: Basic %s\r\n", encoded); 88 | free(encoded); 89 | } 90 | } 91 | 92 | /* End headers */ 93 | len += snprintf(connect_req + len, sizeof(connect_req) - len, "\r\n"); 94 | 95 | /* Send CONNECT request - use SSL if proxy uses TLS */ 96 | ssize_t sent; 97 | if (proxy_ssl) { 98 | sent = SSL_write(proxy_ssl, connect_req, len); 99 | } else { 100 | sent = send(sockfd, connect_req, len, 0); 101 | } 102 | if (sent != len) { 103 | return -1; 104 | } 105 | 106 | /* Read response - use SSL if proxy uses TLS */ 107 | char response[4096]; 108 | ssize_t received; 109 | if (proxy_ssl) { 110 | received = SSL_read(proxy_ssl, response, sizeof(response) - 1); 111 | } else { 112 | received = recv(sockfd, response, sizeof(response) - 1, 0); 113 | } 114 | if (received <= 0) { 115 | return -1; 116 | } 117 | response[received] = '\0'; 118 | 119 | /* Check for 200 Connection established */ 120 | if (strncmp(response, "HTTP/1", 6) == 0) { 121 | char *space = strchr(response, ' '); 122 | if (space) { 123 | int status = atoi(space + 1); 124 | if (status == 200) { 125 | return 0; /* Success */ 126 | } 127 | } 128 | } 129 | 130 | return -1; /* Proxy connection failed */ 131 | } 132 | -------------------------------------------------------------------------------- /.github/workflows/_build_linux_aarch64.yml: -------------------------------------------------------------------------------- 1 | name: Build Linux aarch64 Wheels 2 | 3 | on: 4 | workflow_call: 5 | 6 | jobs: 7 | build-linux-aarch64: 8 | name: Build Linux aarch64 Wheels 9 | # Use native ARM64 runners for much faster builds (vs QEMU emulation) 10 | runs-on: ubuntu-24.04-arm 11 | 12 | steps: 13 | - uses: actions/checkout@v4 14 | with: 15 | submodules: recursive 16 | 17 | # Setup Go for BoringSSL build 18 | - name: Setup Go 19 | uses: actions/setup-go@v5 20 | with: 21 | go-version: '1.21' 22 | cache: true 23 | 24 | # Build wheels with cibuildwheel (handles manylinux containers) 25 | # Vendor dependencies are built inside the manylinux container and cached across Python versions 26 | # Note: GitHub Actions cache doesn't work here since vendor dir is inside the container 27 | - name: Build wheels 28 | uses: pypa/cibuildwheel@v3.3 29 | env: 30 | # Build for all Python versions on aarch64 31 | CIBW_BUILD: cp38-manylinux_aarch64 cp39-manylinux_aarch64 cp310-manylinux_aarch64 cp311-manylinux_aarch64 cp312-manylinux_aarch64 cp313-manylinux_aarch64 cp314-manylinux_aarch64 32 | # Vendor build happens inside manylinux container via before-build (from pyproject.toml) 33 | # The script will detect cached libraries and skip rebuilding across Python versions 34 | CIBW_ARCHS_LINUX: aarch64 35 | 36 | # Setup Python versions for testing 37 | - name: Setup Python versions 38 | uses: actions/setup-python@v5 39 | with: 40 | python-version: | 41 | 3.8 42 | 3.9 43 | 3.10 44 | 3.11 45 | 3.12 46 | 3.13 47 | 3.14 48 | allow-prereleases: true 49 | 50 | # Test the built wheels on native ARM64 51 | - name: Test wheels 52 | run: | 53 | # Test each wheel that was built 54 | for wheel in ./wheelhouse/*.whl; do 55 | echo "========================================" 56 | echo "Testing wheel: $(basename "$wheel")" 57 | echo "========================================" 58 | 59 | # Extract Python version from wheel filename (e.g., cp39, cp310) 60 | python_tag=$(basename "$wheel" | grep -oE 'cp[0-9]+' | head -1) 61 | python_version="${python_tag:2:1}.${python_tag:3}" 62 | 63 | echo "Setting up Python $python_version..." 64 | 65 | # Try to find the matching Python version 66 | python_cmd="" 67 | for cmd in "python${python_version}" "python3.${python_version#*.}" "python3"; do 68 | if command -v "$cmd" &> /dev/null; then 69 | # Verify this is the right version 70 | version_check=$($cmd --version 2>&1 | grep -oE '[0-9]+\.[0-9]+' | head -1) 71 | if [ "$version_check" = "$python_version" ]; then 72 | python_cmd="$cmd" 73 | break 74 | fi 75 | fi 76 | done 77 | 78 | # Fall back to python3 if we couldn't find exact match 79 | if [ -z "$python_cmd" ]; then 80 | echo "WARNING: Python $python_version not found, using python3" 81 | python_cmd="python3" 82 | fi 83 | 84 | echo "Using Python: $($python_cmd --version)" 85 | 86 | # Install the wheel in a fresh environment 87 | "$python_cmd" -m pip install --force-reinstall "$wheel" 88 | 89 | # Run the test script 90 | echo "" 91 | echo "Running tests..." 92 | "$python_cmd" scripts/test_local_build.py 93 | 94 | # Check exit code 95 | if [ $? -eq 0 ]; then 96 | echo "" 97 | echo "[OK] Wheel test PASSED: $(basename "$wheel")" 98 | else 99 | echo "" 100 | echo "[FAIL] Wheel test FAILED: $(basename "$wheel")" 101 | exit 1 102 | fi 103 | 104 | echo "" 105 | done 106 | 107 | echo "========================================" 108 | echo "All wheel tests PASSED" 109 | echo "========================================" 110 | 111 | # Upload wheels as artifacts 112 | - name: Upload wheels 113 | uses: actions/upload-artifact@v4 114 | with: 115 | name: wheels-linux-aarch64 116 | path: ./wheelhouse/*.whl 117 | if-no-files-found: error 118 | retention-days: 5 119 | # Retry on transient failures 120 | continue-on-error: false 121 | -------------------------------------------------------------------------------- /scripts/windows/test_windows_build_local.ps1: -------------------------------------------------------------------------------- 1 | # PowerShell script to build and test httpmorph wheels on Windows 2 | # Mimics the GitHub Actions release pipeline (.github/workflows/release.yml) 3 | 4 | param( 5 | [switch]$SkipTests = $false 6 | ) 7 | 8 | $ErrorActionPreference = "Stop" 9 | 10 | Write-Host "========================================" -ForegroundColor Cyan 11 | Write-Host "httpmorph - Windows Wheel Builder" -ForegroundColor Cyan 12 | Write-Host "========================================" -ForegroundColor Cyan 13 | Write-Host "" 14 | 15 | $ProjectRoot = Split-Path (Split-Path $PSScriptRoot -Parent) -Parent 16 | 17 | # Step 1: Setup vendors (BoringSSL, nghttp2, zlib) 18 | Write-Host "==> Step 1: Setting up vendor dependencies..." -ForegroundColor Yellow 19 | 20 | # Check for required tools 21 | Write-Host " Checking dependencies..." -ForegroundColor Gray 22 | 23 | $cmake = Get-Command cmake -ErrorAction SilentlyContinue 24 | if (-not $cmake) { 25 | Write-Host "[FAIL] cmake not found. Install with: choco install cmake" -ForegroundColor Red 26 | exit 1 27 | } 28 | Write-Host " [OK] cmake found" -ForegroundColor Green 29 | 30 | $go = Get-Command go -ErrorAction SilentlyContinue 31 | if (-not $go) { 32 | Write-Host "[FAIL] go not found. Install with: choco install golang" -ForegroundColor Red 33 | exit 1 34 | } 35 | Write-Host " [OK] go found" -ForegroundColor Green 36 | 37 | # Check vcpkg 38 | if (-not (Test-Path "C:\vcpkg\vcpkg.exe")) { 39 | Write-Host "[FAIL] vcpkg not found at C:\vcpkg" -ForegroundColor Red 40 | Write-Host " Install vcpkg and nghttp2/zlib:" -ForegroundColor Red 41 | Write-Host " git clone https://github.com/Microsoft/vcpkg.git C:\vcpkg" -ForegroundColor Yellow 42 | Write-Host " C:\vcpkg\bootstrap-vcpkg.bat" -ForegroundColor Yellow 43 | Write-Host " C:\vcpkg\vcpkg install nghttp2:x64-windows zlib:x64-windows" -ForegroundColor Yellow 44 | exit 1 45 | } 46 | Write-Host " [OK] vcpkg found" -ForegroundColor Green 47 | 48 | # Build BoringSSL 49 | Write-Host "" 50 | Write-Host " Building BoringSSL..." -ForegroundColor Gray 51 | Push-Location $ProjectRoot 52 | bash scripts/setup_vendors.sh 53 | if ($LASTEXITCODE -ne 0) { 54 | Write-Host "[FAIL] Vendor setup failed" -ForegroundColor Red 55 | Pop-Location 56 | exit 1 57 | } 58 | Pop-Location 59 | Write-Host " [OK] Vendors built successfully" -ForegroundColor Green 60 | 61 | # Step 2: Build wheel 62 | Write-Host "" 63 | Write-Host "==> Step 2: Building wheel..." -ForegroundColor Yellow 64 | 65 | $env:VCPKG_ROOT = "C:\vcpkg" 66 | 67 | Push-Location $ProjectRoot 68 | 69 | # Build wheel 70 | python setup.py bdist_wheel 71 | 72 | if ($LASTEXITCODE -ne 0) { 73 | Write-Host "[FAIL] Wheel build failed" -ForegroundColor Red 74 | Pop-Location 75 | exit 1 76 | } 77 | 78 | Pop-Location 79 | 80 | # Find the built wheel 81 | $wheel = Get-ChildItem -Path "$ProjectRoot\dist" -Filter "*.whl" | Sort-Object LastWriteTime -Descending | Select-Object -First 1 82 | 83 | if (-not $wheel) { 84 | Write-Host "[FAIL] No wheel found in dist/" -ForegroundColor Red 85 | exit 1 86 | } 87 | 88 | Write-Host " [OK] Wheel built: $($wheel.Name)" -ForegroundColor Green 89 | 90 | # Step 3: Test wheel 91 | if (-not $SkipTests) { 92 | Write-Host "" 93 | Write-Host "==> Step 3: Testing wheel..." -ForegroundColor Yellow 94 | 95 | # Install wheel 96 | Write-Host " Installing wheel..." -ForegroundColor Gray 97 | python -m pip install --force-reinstall "$($wheel.FullName)" 98 | 99 | if ($LASTEXITCODE -ne 0) { 100 | Write-Host "[FAIL] Wheel installation failed" -ForegroundColor Red 101 | exit 1 102 | } 103 | 104 | # Run tests 105 | Write-Host "" 106 | Write-Host " Running tests..." -ForegroundColor Gray 107 | python "$ProjectRoot\scripts\test_local_build.py" 108 | 109 | if ($LASTEXITCODE -ne 0) { 110 | Write-Host "[FAIL] Tests failed" -ForegroundColor Red 111 | exit 1 112 | } 113 | 114 | Write-Host "" 115 | Write-Host " [OK] All tests passed!" -ForegroundColor Green 116 | } 117 | 118 | # Summary 119 | Write-Host "" 120 | Write-Host "========================================" -ForegroundColor Cyan 121 | Write-Host "BUILD COMPLETE" -ForegroundColor Cyan 122 | Write-Host "========================================" -ForegroundColor Cyan 123 | Write-Host "" 124 | Write-Host "Wheel: $($wheel.FullName)" -ForegroundColor Green 125 | Write-Host "Size: $([math]::Round($wheel.Length / 1KB, 2)) KB" -ForegroundColor Green 126 | Write-Host "" 127 | Write-Host "To install:" -ForegroundColor Yellow 128 | Write-Host " pip install $($wheel.FullName)" -ForegroundColor Gray 129 | Write-Host "" 130 | 131 | exit 0 132 | -------------------------------------------------------------------------------- /benchmarks/libs/aiohttp_bench.py: -------------------------------------------------------------------------------- 1 | """aiohttp benchmark implementations""" 2 | 3 | try: 4 | import aiohttp 5 | 6 | AVAILABLE = True 7 | except ImportError: 8 | AVAILABLE = False 9 | 10 | from .base import LibraryBenchmark 11 | 12 | 13 | class AiohttpBenchmark(LibraryBenchmark): 14 | """Benchmark tests for aiohttp library""" 15 | 16 | def is_available(self): 17 | return AVAILABLE 18 | 19 | def get_test_matrix(self): 20 | return [ 21 | ("Async Local HTTP", "async_local_http"), 22 | ("Async Remote HTTP", "async_remote_http"), 23 | ("Async Remote HTTPS", "async_remote_https"), 24 | ("Async Proxy HTTP", "async_proxy_http"), 25 | ("Async Proxy HTTPS", "async_proxy_https"), 26 | ] 27 | 28 | def async_local_http(self): 29 | async def request(): 30 | async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: 31 | async with session.get( 32 | self.local_url, timeout=aiohttp.ClientTimeout(total=10) 33 | ) as resp: 34 | assert 200 <= resp.status < 600, f"Got status {resp.status}" 35 | body = await resp.text() 36 | self.validate_response_body(self.local_url, body) 37 | 38 | return self.run_async_benchmark(request) 39 | 40 | def async_remote_http(self): 41 | async def request(): 42 | async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: 43 | async with session.get( 44 | self.remote_http_url, timeout=aiohttp.ClientTimeout(total=10) 45 | ) as resp: 46 | assert 200 <= resp.status < 600, f"Got status {resp.status}" 47 | body = await resp.text() 48 | self.validate_response_body(self.remote_http_url, body) 49 | 50 | return self.run_async_benchmark(request) 51 | 52 | def async_remote_https(self): 53 | async def request(): 54 | async with aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=False)) as session: 55 | async with session.get( 56 | self.remote_https_url, timeout=aiohttp.ClientTimeout(total=10) 57 | ) as resp: 58 | assert 200 <= resp.status < 600, f"Got status {resp.status}" 59 | body = await resp.text() 60 | self.validate_response_body(self.remote_https_url, body) 61 | 62 | return self.run_async_benchmark(request) 63 | 64 | def async_proxy_http(self): 65 | if not self.proxy_url_http: 66 | return {"error": "No proxy configured"} 67 | 68 | async def request(): 69 | try: 70 | async with aiohttp.ClientSession( 71 | connector=aiohttp.TCPConnector(ssl=False) 72 | ) as session: 73 | async with session.get( 74 | self.proxy_target_http, 75 | proxy=self.proxy_url_http, 76 | timeout=aiohttp.ClientTimeout(total=10), 77 | ) as resp: 78 | assert 200 <= resp.status < 600, f"Got status {resp.status}" 79 | body = await resp.text() 80 | self.validate_response_body(self.proxy_target_http, body) 81 | except Exception: 82 | pass 83 | 84 | try: 85 | return self.run_async_benchmark(request) 86 | except Exception: 87 | return {"total_time_s": 10.0, "req_per_sec": 0.1, "avg_ms": 10000.0} 88 | 89 | def async_proxy_https(self): 90 | if not self.proxy_url_https: 91 | return {"error": "No proxy configured"} 92 | 93 | async def request(): 94 | try: 95 | async with aiohttp.ClientSession( 96 | connector=aiohttp.TCPConnector(ssl=False) 97 | ) as session: 98 | async with session.get( 99 | self.proxy_target_https, 100 | proxy=self.proxy_url_https, 101 | timeout=aiohttp.ClientTimeout(total=10), 102 | ) as resp: 103 | assert 200 <= resp.status < 600, f"Got status {resp.status}" 104 | body = await resp.text() 105 | self.validate_response_body(self.proxy_target_https, body) 106 | except Exception: 107 | pass 108 | 109 | try: 110 | return self.run_async_benchmark(request) 111 | except Exception: 112 | return {"total_time_s": 10.0, "req_per_sec": 0.1, "avg_ms": 10000.0} 113 | -------------------------------------------------------------------------------- /src/core/request_builder.c: -------------------------------------------------------------------------------- 1 | /** 2 | * request_builder.c - Fast request building implementation 3 | */ 4 | 5 | #include "request_builder.h" 6 | #include 7 | #include 8 | #include 9 | 10 | /* Initial capacity if not specified */ 11 | #define DEFAULT_CAPACITY 512 12 | 13 | /* Growth factor for reallocation */ 14 | #define GROWTH_FACTOR 2 15 | 16 | /** 17 | * Ensure builder has at least `needed` bytes of free space 18 | */ 19 | static int ensure_capacity(request_builder_t *builder, size_t needed) { 20 | if (builder->len + needed <= builder->capacity) { 21 | return 0; /* Enough space */ 22 | } 23 | 24 | /* Calculate new capacity with overflow protection */ 25 | size_t new_capacity = builder->capacity; 26 | size_t target = builder->len + needed; 27 | 28 | /* Check for overflow in target calculation */ 29 | if (target < builder->len) { 30 | return -1; /* Overflow detected */ 31 | } 32 | 33 | while (new_capacity < target) { 34 | /* Check for overflow before multiplication */ 35 | if (new_capacity > SIZE_MAX / GROWTH_FACTOR) { 36 | return -1; /* Would overflow */ 37 | } 38 | new_capacity *= GROWTH_FACTOR; 39 | } 40 | 41 | /* Reallocate */ 42 | char *new_data = (char*)realloc(builder->data, new_capacity); 43 | if (!new_data) { 44 | return -1; 45 | } 46 | 47 | builder->data = new_data; 48 | builder->capacity = new_capacity; 49 | return 0; 50 | } 51 | 52 | /** 53 | * Create a new request builder 54 | */ 55 | request_builder_t* request_builder_create(size_t initial_capacity) { 56 | request_builder_t *builder = (request_builder_t*)malloc(sizeof(request_builder_t)); 57 | if (!builder) { 58 | return NULL; 59 | } 60 | 61 | if (initial_capacity == 0) { 62 | initial_capacity = DEFAULT_CAPACITY; 63 | } 64 | 65 | builder->data = (char*)malloc(initial_capacity); 66 | if (!builder->data) { 67 | free(builder); 68 | return NULL; 69 | } 70 | 71 | builder->len = 0; 72 | builder->capacity = initial_capacity; 73 | return builder; 74 | } 75 | 76 | /** 77 | * Destroy a request builder 78 | */ 79 | void request_builder_destroy(request_builder_t *builder) { 80 | if (!builder) { 81 | return; 82 | } 83 | free(builder->data); 84 | free(builder); 85 | } 86 | 87 | /** 88 | * Append a string with known length (fast memcpy) 89 | */ 90 | int request_builder_append(request_builder_t *builder, const char *str, size_t len) { 91 | if (!builder || !str || len == 0) { 92 | return 0; 93 | } 94 | 95 | if (ensure_capacity(builder, len) != 0) { 96 | return -1; 97 | } 98 | 99 | memcpy(builder->data + builder->len, str, len); 100 | builder->len += len; 101 | return 0; 102 | } 103 | 104 | /** 105 | * Append a null-terminated string 106 | */ 107 | int request_builder_append_str(request_builder_t *builder, const char *str) { 108 | if (!str) { 109 | return 0; 110 | } 111 | return request_builder_append(builder, str, strlen(str)); 112 | } 113 | 114 | /** 115 | * Append an unsigned integer as decimal string 116 | */ 117 | int request_builder_append_uint(request_builder_t *builder, uint64_t value) { 118 | char buf[32]; /* Enough for uint64_t */ 119 | int len = snprintf(buf, sizeof(buf), "%llu", (unsigned long long)value); 120 | if (len < 0) { 121 | return -1; 122 | } 123 | return request_builder_append(builder, buf, len); 124 | } 125 | 126 | /** 127 | * Append a formatted header line 128 | */ 129 | int request_builder_append_header(request_builder_t *builder, 130 | const char *key, size_t key_len, 131 | const char *value, size_t value_len) { 132 | /* Total: key_len + 2 (": ") + value_len + 2 ("\r\n") */ 133 | size_t total = key_len + value_len + 4; 134 | 135 | if (ensure_capacity(builder, total) != 0) { 136 | return -1; 137 | } 138 | 139 | char *p = builder->data + builder->len; 140 | 141 | /* Copy key */ 142 | memcpy(p, key, key_len); 143 | p += key_len; 144 | 145 | /* Add ": " */ 146 | *p++ = ':'; 147 | *p++ = ' '; 148 | 149 | /* Copy value */ 150 | memcpy(p, value, value_len); 151 | p += value_len; 152 | 153 | /* Add "\r\n" */ 154 | *p++ = '\r'; 155 | *p++ = '\n'; 156 | 157 | builder->len += total; 158 | return 0; 159 | } 160 | 161 | /** 162 | * Get the current buffer data 163 | */ 164 | const char* request_builder_data(const request_builder_t *builder, size_t *len) { 165 | if (!builder) { 166 | if (len) *len = 0; 167 | return NULL; 168 | } 169 | 170 | if (len) { 171 | *len = builder->len; 172 | } 173 | return builder->data; 174 | } 175 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [main, develop] 6 | pull_request: 7 | branches: [main, develop] 8 | workflow_dispatch: 9 | 10 | jobs: 11 | # ============================================================ 12 | # Load Configuration 13 | # ============================================================ 14 | load-config: 15 | name: Load Configuration 16 | uses: ./.github/workflows/_config.yml 17 | 18 | # ============================================================ 19 | # Test Job 20 | # ============================================================ 21 | test: 22 | name: Test 23 | needs: load-config 24 | if: needs.load-config.outputs.enable-tests == 'true' 25 | uses: ./.github/workflows/_test.yml 26 | with: 27 | os-matrix: ${{ needs.load-config.outputs.os-matrix }} 28 | python-matrix: ${{ needs.load-config.outputs.python-matrix }} 29 | primary-os: ${{ needs.load-config.outputs.primary-os }} 30 | primary-python: ${{ needs.load-config.outputs.primary-python }} 31 | secrets: 32 | TEST_PROXY_URL: ${{ secrets.TEST_PROXY_URL }} 33 | TEST_HTTPBIN_HOST: ${{ secrets.TEST_HTTPBIN_HOST }} 34 | CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} 35 | 36 | # ============================================================ 37 | # Build Test Jobs 38 | # ============================================================ 39 | build-test-linux-x86_64: 40 | name: Build Test (Linux x86_64) 41 | needs: load-config 42 | uses: ./.github/workflows/_build_linux_x86_64.yml 43 | 44 | build-test-linux-aarch64: 45 | name: Build Test (Linux aarch64) 46 | needs: load-config 47 | uses: ./.github/workflows/_build_linux_aarch64.yml 48 | 49 | build-test-macos: 50 | name: Build Test (macOS) 51 | needs: load-config 52 | uses: ./.github/workflows/_build_macos.yml 53 | 54 | build-test-windows: 55 | name: Build Test (Windows) 56 | needs: load-config 57 | uses: ./.github/workflows/_build_windows.yml 58 | 59 | # ============================================================ 60 | # Summary 61 | # ============================================================ 62 | summary: 63 | name: CI Summary 64 | needs: [load-config, test, build-test-linux-x86_64, build-test-linux-aarch64, build-test-macos, build-test-windows] 65 | if: always() 66 | runs-on: ubuntu-latest 67 | 68 | steps: 69 | - name: Check results 70 | run: | 71 | echo "===================================" 72 | echo "CI Pipeline Summary" 73 | echo "===================================" 74 | echo "" 75 | 76 | # Tests 77 | if [ "${{ needs.load-config.outputs.enable-tests }}" == "true" ]; then 78 | if [ "${{ needs.test.result }}" == "success" ]; then 79 | echo "✅ Tests: PASSED" 80 | else 81 | echo "❌ Tests: FAILED" 82 | EXIT_CODE=1 83 | fi 84 | else 85 | echo "⊘ Tests: DISABLED" 86 | fi 87 | 88 | # Build Tests 89 | echo "" 90 | echo "Build Tests:" 91 | 92 | # Linux x86_64 93 | if [ "${{ needs.build-test-linux-x86_64.result }}" == "success" ]; then 94 | echo " ✅ Linux x86_64: PASSED" 95 | elif [ "${{ needs.build-test-linux-x86_64.result }}" == "skipped" ]; then 96 | echo " ⊘ Linux x86_64: SKIPPED" 97 | else 98 | echo " ❌ Linux x86_64: FAILED" 99 | EXIT_CODE=1 100 | fi 101 | 102 | # Linux aarch64 103 | if [ "${{ needs.build-test-linux-aarch64.result }}" == "success" ]; then 104 | echo " ✅ Linux aarch64: PASSED" 105 | elif [ "${{ needs.build-test-linux-aarch64.result }}" == "skipped" ]; then 106 | echo " ⊘ Linux aarch64: SKIPPED" 107 | else 108 | echo " ❌ Linux aarch64: FAILED" 109 | EXIT_CODE=1 110 | fi 111 | 112 | # macOS 113 | if [ "${{ needs.build-test-macos.result }}" == "success" ]; then 114 | echo " ✅ macOS: PASSED" 115 | elif [ "${{ needs.build-test-macos.result }}" == "skipped" ]; then 116 | echo " ⊘ macOS: SKIPPED" 117 | else 118 | echo " ❌ macOS: FAILED" 119 | EXIT_CODE=1 120 | fi 121 | 122 | # Windows 123 | if [ "${{ needs.build-test-windows.result }}" == "success" ]; then 124 | echo " ✅ Windows: PASSED" 125 | elif [ "${{ needs.build-test-windows.result }}" == "skipped" ]; then 126 | echo " ⊘ Windows: SKIPPED" 127 | else 128 | echo " ❌ Windows: FAILED" 129 | EXIT_CODE=1 130 | fi 131 | 132 | echo "" 133 | if [ "${EXIT_CODE:-0}" == "1" ]; then 134 | echo "❌ CI Pipeline FAILED" 135 | exit 1 136 | else 137 | echo "✅ CI Pipeline PASSED" 138 | fi 139 | -------------------------------------------------------------------------------- /examples/advanced_features.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | """ 3 | httpmorph - Advanced Features Demo 4 | 5 | Demonstrates all implemented features: 6 | - Browser profiles with different JA3 fingerprints 7 | - TLS information extraction 8 | - Session management with cookies 9 | - Custom headers 10 | - Response timing 11 | - Gzip decompression 12 | """ 13 | 14 | import httpmorph 15 | 16 | 17 | def demo_browser_profiles(): 18 | """Demo different browser profiles""" 19 | print("\n" + "=" * 60) 20 | print("1. Browser Profiles") 21 | print("=" * 60) 22 | 23 | browsers = ["chrome", "firefox", "safari", "edge"] 24 | 25 | for browser in browsers: 26 | session = httpmorph.Session(browser=browser) 27 | response = session.get("https://example.com") 28 | 29 | print(f"\n{browser.upper()}:") 30 | print(f" Status: {response.status_code}") 31 | print(f" User-Agent: {session.user_agent[:50]}...") 32 | print(f" JA3 Hash: {response.ja3_fingerprint}") 33 | print(f" TLS Version: {response.tls_version}") 34 | print(f" TLS Cipher: {response.tls_cipher}") 35 | 36 | 37 | def demo_tls_information(): 38 | """Demo TLS information extraction""" 39 | print("\n" + "=" * 60) 40 | print("2. TLS Information Extraction") 41 | print("=" * 60) 42 | 43 | response = httpmorph.get("https://example.com") 44 | 45 | print("\nTLS Details:") 46 | print(f" Version: {response.tls_version}") 47 | print(f" Cipher: {response.tls_cipher}") 48 | print(f" JA3: {response.ja3_fingerprint}") 49 | print(f" HTTP Ver: {response.http_version}") 50 | 51 | 52 | def demo_session_management(): 53 | """Demo session with cookies and custom headers""" 54 | print("\n" + "=" * 60) 55 | print("3. Session Management") 56 | print("=" * 60) 57 | 58 | with httpmorph.Session(browser="chrome") as session: 59 | # Custom headers 60 | headers = {"X-Custom-Header": "MyValue", "Accept": "application/json"} 61 | 62 | response = session.get("https://api.github.com", headers=headers) 63 | 64 | print("\nGitHub API Request:") 65 | print(f" Status: {response.status_code}") 66 | print(f" Body length: {len(response.body)} bytes") 67 | print(f" Time taken: {response.total_time_us / 1000:.2f}ms") 68 | 69 | 70 | def demo_http_methods(): 71 | """Demo different HTTP methods""" 72 | print("\n" + "=" * 60) 73 | print("4. HTTP Methods") 74 | print("=" * 60) 75 | 76 | # GET request 77 | response = httpmorph.get("https://example.com") 78 | print("\nGET Request:") 79 | print(f" Status: {response.status_code}") 80 | print(f" Time: {response.total_time_us / 1000:.2f}ms") 81 | 82 | # POST with JSON (would work with httpbin if not rate limited) 83 | # data = {"key": "value", "number": 42} 84 | # response = httpmorph.post("https://httpbin.org/post", json=data) 85 | 86 | 87 | def demo_performance_metrics(): 88 | """Demo performance metrics""" 89 | print("\n" + "=" * 60) 90 | print("5. Performance Metrics") 91 | print("=" * 60) 92 | 93 | import time 94 | 95 | session = httpmorph.Session(browser="chrome") 96 | 97 | # Multiple requests to same domain 98 | urls = ["https://example.com", "https://www.google.com", "https://api.github.com"] 99 | 100 | print("\nSequential Requests:") 101 | for url in urls: 102 | start = time.time() 103 | response = session.get(url) 104 | elapsed = (time.time() - start) * 1000 105 | 106 | print(f" {url:30} - {response.status_code} ({elapsed:.2f}ms)") 107 | 108 | 109 | def demo_compression(): 110 | """Demo automatic gzip decompression""" 111 | print("\n" + "=" * 60) 112 | print("6. Automatic Gzip Decompression") 113 | print("=" * 60) 114 | 115 | response = httpmorph.get("https://example.com") 116 | 117 | print("\nResponse:") 118 | print(f" Encoding: {response.headers.get('Content-Encoding', 'none')}") 119 | print(f" Body length: {len(response.body)} bytes") 120 | print(f" Text length: {len(response.text)} chars") 121 | print(f" Decoded: {'Yes' if response.text else 'No'}") 122 | 123 | 124 | def main(): 125 | """Run all demos""" 126 | print("\n" + "=" * 60) 127 | print("httpmorph - Advanced Features Demo") 128 | print("=" * 60) 129 | 130 | try: 131 | demo_browser_profiles() 132 | demo_tls_information() 133 | demo_session_management() 134 | demo_http_methods() 135 | demo_performance_metrics() 136 | demo_compression() 137 | 138 | print("\n" + "=" * 60) 139 | print("✅ All features working successfully!") 140 | print("=" * 60) 141 | print() 142 | 143 | except Exception as e: 144 | print(f"\n❌ Error: {e}") 145 | import traceback 146 | 147 | traceback.print_exc() 148 | 149 | 150 | if __name__ == "__main__": 151 | main() 152 | -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | """ 2 | Pytest configuration and fixtures for httpmorph tests 3 | """ 4 | 5 | import os 6 | import subprocess 7 | import time 8 | from pathlib import Path 9 | 10 | import filelock 11 | import pytest 12 | 13 | import httpmorph 14 | 15 | # Load .env file for local testing 16 | # This allows TEST_PROXY_URL and other env vars to be loaded from .env 17 | try: 18 | from dotenv import load_dotenv 19 | 20 | env_path = Path(__file__).parent.parent / ".env" 21 | if env_path.exists(): 22 | load_dotenv(env_path) 23 | except ImportError: 24 | # python-dotenv not installed, skip 25 | pass 26 | 27 | 28 | @pytest.fixture(scope="session", autouse=True) 29 | def initialize_httpmorph(): 30 | """Initialize httpmorph library before running tests""" 31 | try: 32 | httpmorph.init() 33 | yield 34 | httpmorph.cleanup() 35 | except (NotImplementedError, AttributeError): 36 | # If init/cleanup not implemented, just skip 37 | yield 38 | 39 | 40 | @pytest.fixture 41 | def http_server(): 42 | """Create a test HTTP server""" 43 | from tests.test_server import MockHTTPServer 44 | 45 | server = MockHTTPServer() 46 | server.start() 47 | yield server 48 | server.stop() 49 | 50 | 51 | @pytest.fixture 52 | def https_server(): 53 | """Create a test HTTPS server""" 54 | from tests.test_server import MockHTTPServer 55 | 56 | try: 57 | server = MockHTTPServer(ssl_enabled=True) 58 | server.start() 59 | yield server 60 | server.stop() 61 | except RuntimeError as e: 62 | pytest.skip(f"HTTPS server not available: {e}") 63 | 64 | 65 | @pytest.fixture 66 | def chrome_session(): 67 | """Create a Chrome session""" 68 | try: 69 | session = httpmorph.Session(browser="chrome") 70 | yield session 71 | # Explicitly cleanup session resources 72 | if hasattr(session, "close"): 73 | session.close() 74 | del session 75 | except (NotImplementedError, AttributeError): 76 | pytest.skip("Session not yet implemented") 77 | 78 | 79 | @pytest.fixture 80 | def firefox_session(): 81 | """Create a Firefox session""" 82 | try: 83 | session = httpmorph.Session(browser="firefox") 84 | yield session 85 | # Explicitly cleanup session resources 86 | if hasattr(session, "close"): 87 | session.close() 88 | del session 89 | except (NotImplementedError, AttributeError): 90 | pytest.skip("Session not yet implemented") 91 | 92 | 93 | @pytest.fixture 94 | def safari_session(): 95 | """Create a Safari session""" 96 | try: 97 | session = httpmorph.Session(browser="safari") 98 | yield session 99 | # Explicitly cleanup session resources 100 | if hasattr(session, "close"): 101 | session.close() 102 | del session 103 | except (NotImplementedError, AttributeError): 104 | pytest.skip("Session not yet implemented") 105 | 106 | 107 | @pytest.fixture(scope="session") 108 | def httpbin_server(): 109 | """Use MockHTTPServer for httpbin-compatible testing""" 110 | from tests.test_server import MockHTTPServer 111 | 112 | server = MockHTTPServer() 113 | server.start() 114 | yield server.url 115 | server.stop() 116 | 117 | 118 | @pytest.fixture(scope="session") 119 | def mock_httpbin_server(): 120 | """Use MockHTTPServer for tests where Docker httpbin fails""" 121 | from tests.test_server import MockHTTPServer 122 | 123 | server = MockHTTPServer() 124 | server.start() 125 | yield server.url 126 | server.stop() 127 | 128 | 129 | @pytest.fixture(scope="session") 130 | def httpbin_host(): 131 | """Get HTTPBin host from environment, defaults to httpmorph-bin.bytetunnels.com""" 132 | return os.environ.get("TEST_HTTPBIN_HOST", "httpmorph-bin.bytetunnels.com") 133 | 134 | 135 | def pytest_configure(config): 136 | """Configure pytest""" 137 | config.addinivalue_line( 138 | "markers", "integration: mark test as integration test (requires network)" 139 | ) 140 | config.addinivalue_line("markers", "slow: mark test as slow running") 141 | config.addinivalue_line("markers", "ssl: mark test as requiring SSL support") 142 | 143 | 144 | def pytest_collection_modifyitems(config, items): 145 | """Modify test collection""" 146 | # Add markers based on test location 147 | for item in items: 148 | if "test_integration" in item.nodeid: 149 | item.add_marker(pytest.mark.integration) 150 | if "test_browser_profiles" in item.nodeid: 151 | item.add_marker(pytest.mark.slow) 152 | if "https_server" in item.fixturenames or "ssl" in item.nodeid: 153 | item.add_marker(pytest.mark.ssl) 154 | 155 | 156 | def pytest_runtest_teardown(item, nextitem): 157 | """Force garbage collection after each test to prevent resource accumulation""" 158 | import gc 159 | 160 | gc.collect() 161 | -------------------------------------------------------------------------------- /src/core/io_engine.h: -------------------------------------------------------------------------------- 1 | /** 2 | * io_engine.h - High-performance I/O engine with io_uring and epoll support 3 | * 4 | * Provides a unified interface for: 5 | * - io_uring on Linux 5.1+ (highest performance) 6 | * - epoll on Linux (fallback) 7 | * - kqueue on macOS/BSD (future) 8 | */ 9 | 10 | #ifndef IO_ENGINE_H 11 | #define IO_ENGINE_H 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | /* Platform-specific socket headers */ 18 | #ifdef _WIN32 19 | #include 20 | #include 21 | typedef int socklen_t; 22 | #else 23 | #include 24 | #endif 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | /* I/O engine types */ 31 | typedef enum { 32 | IO_ENGINE_URING, /* io_uring (Linux 5.1+) */ 33 | IO_ENGINE_EPOLL, /* epoll (Linux) */ 34 | IO_ENGINE_KQUEUE, /* kqueue (macOS/BSD) */ 35 | IO_ENGINE_IOCP, /* IOCP (Windows) */ 36 | } io_engine_type_t; 37 | 38 | /* I/O operation types */ 39 | typedef enum { 40 | IO_OP_ACCEPT, 41 | IO_OP_CONNECT, 42 | IO_OP_RECV, 43 | IO_OP_SEND, 44 | IO_OP_CLOSE, 45 | IO_OP_TIMEOUT, 46 | } io_op_type_t; 47 | 48 | /* I/O operation structure */ 49 | typedef struct io_operation { 50 | io_op_type_t type; 51 | int fd; 52 | 53 | /* Buffer for read/write */ 54 | void *buf; 55 | size_t len; 56 | 57 | /* Result */ 58 | int result; 59 | 60 | /* User data */ 61 | void *user_data; 62 | 63 | /* Callback when operation completes */ 64 | void (*callback)(struct io_operation *op); 65 | } io_operation_t; 66 | 67 | /* I/O engine structure */ 68 | typedef struct io_engine { 69 | io_engine_type_t type; 70 | int engine_fd; /* epoll fd or io_uring fd or kqueue fd */ 71 | 72 | /* io_uring specific */ 73 | #ifdef HAVE_IO_URING 74 | struct io_uring *ring; 75 | #endif 76 | 77 | /* IOCP specific (Windows) */ 78 | #ifdef _WIN32 79 | void *iocp_handle; /* HANDLE for completion port */ 80 | #endif 81 | 82 | /* Statistics */ 83 | uint64_t ops_submitted; 84 | uint64_t ops_completed; 85 | uint64_t ops_failed; 86 | 87 | /* Configuration */ 88 | uint32_t queue_depth; /* io_uring queue depth */ 89 | bool zero_copy; /* Enable zero-copy operations */ 90 | } io_engine_t; 91 | 92 | /* API */ 93 | 94 | /** 95 | * Create a new I/O engine 96 | * Automatically selects the best available engine for the platform 97 | */ 98 | io_engine_t* io_engine_create(uint32_t queue_depth); 99 | 100 | /** 101 | * Destroy an I/O engine 102 | */ 103 | void io_engine_destroy(io_engine_t *engine); 104 | 105 | /** 106 | * Submit an I/O operation 107 | * Returns 0 on success, -1 on error 108 | */ 109 | int io_engine_submit(io_engine_t *engine, io_operation_t *op); 110 | 111 | /** 112 | * Submit multiple I/O operations (batching for performance) 113 | * Returns number of operations submitted, -1 on error 114 | */ 115 | int io_engine_submit_batch( 116 | io_engine_t *engine, 117 | io_operation_t **ops, 118 | size_t count 119 | ); 120 | 121 | /** 122 | * Wait for I/O completions 123 | * Returns number of operations completed, -1 on error 124 | * 125 | * @param timeout_ms: Timeout in milliseconds, 0 for non-blocking, -1 for blocking 126 | */ 127 | int io_engine_wait( 128 | io_engine_t *engine, 129 | uint32_t timeout_ms 130 | ); 131 | 132 | /** 133 | * Process completed operations 134 | * Calls the callbacks for completed operations 135 | */ 136 | int io_engine_process_completions(io_engine_t *engine); 137 | 138 | /** 139 | * Get engine type name 140 | */ 141 | const char* io_engine_type_name(io_engine_type_t type); 142 | 143 | /* Operation helpers */ 144 | 145 | /** 146 | * Create a connect operation 147 | */ 148 | io_operation_t* io_op_connect_create( 149 | int sockfd, 150 | const struct sockaddr *addr, 151 | socklen_t addrlen, 152 | void (*callback)(io_operation_t *op), 153 | void *user_data 154 | ); 155 | 156 | /** 157 | * Create a receive operation 158 | */ 159 | io_operation_t* io_op_recv_create( 160 | int sockfd, 161 | void *buf, 162 | size_t len, 163 | void (*callback)(io_operation_t *op), 164 | void *user_data 165 | ); 166 | 167 | /** 168 | * Create a send operation 169 | */ 170 | io_operation_t* io_op_send_create( 171 | int sockfd, 172 | const void *buf, 173 | size_t len, 174 | void (*callback)(io_operation_t *op), 175 | void *user_data 176 | ); 177 | 178 | /** 179 | * Destroy an operation 180 | */ 181 | void io_op_destroy(io_operation_t *op); 182 | 183 | /** 184 | * Check if io_uring is available on this system 185 | */ 186 | bool io_engine_has_uring(void); 187 | 188 | /** 189 | * Create a non-blocking socket 190 | */ 191 | int io_socket_create_nonblocking(int domain, int type, int protocol); 192 | 193 | /** 194 | * Set socket options for performance 195 | */ 196 | int io_socket_set_performance_opts(int sockfd); 197 | 198 | #ifdef __cplusplus 199 | } 200 | #endif 201 | 202 | #endif /* IO_ENGINE_H */ 203 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ 3 | "setuptools>=68.0,<75.4.0", 4 | "wheel", 5 | "Cython>=3.0", 6 | "tomli>=2.0.0; python_version < '3.11'" 7 | ] 8 | build-backend = "setuptools.build_meta" 9 | 10 | [project] 11 | name = "httpmorph" 12 | version = "0.2.8" 13 | description = "A Python HTTP client focused on mimicking browser fingerprints." 14 | readme = "README.md" 15 | requires-python = ">=3.8" 16 | license = {text = "MIT"} 17 | authors = [ 18 | {name = "Arman Hossain", email = "arman@bytetunnels.com"} 19 | ] 20 | keywords = ["http", "https", "client", "performance", "anti-fingerprint", "tls", "ja3", "http2"] 21 | classifiers = [ 22 | "Development Status :: 3 - Alpha", 23 | "Intended Audience :: Developers", 24 | "License :: OSI Approved :: MIT License", 25 | "Programming Language :: Python :: 3", 26 | "Programming Language :: Python :: 3.8", 27 | "Programming Language :: Python :: 3.9", 28 | "Programming Language :: Python :: 3.10", 29 | "Programming Language :: Python :: 3.11", 30 | "Programming Language :: Python :: 3.12", 31 | "Programming Language :: Python :: 3.13", 32 | "Programming Language :: Python :: 3.14", 33 | "Programming Language :: C", 34 | "Operating System :: OS Independent", 35 | "Operating System :: POSIX :: Linux", 36 | "Operating System :: Microsoft :: Windows", 37 | "Operating System :: MacOS", 38 | "Topic :: Internet :: WWW/HTTP", 39 | "Topic :: Software Development :: Libraries :: Python Modules", 40 | ] 41 | 42 | dependencies = [] 43 | 44 | [project.optional-dependencies] 45 | dev = [ 46 | "pytest>=7.0", 47 | "pytest-asyncio>=0.21.0", 48 | "pytest-benchmark>=4.0", 49 | "pytest-cov>=4.0", 50 | "cryptography>=41.0", # For test HTTPS server 51 | "filelock>=3.12.0", # For test fixtures 52 | "mypy>=1.0", 53 | "ruff>=0.7.0", 54 | ] 55 | build = [ 56 | "setuptools>=68.0", 57 | "wheel", 58 | "Cython>=3.0", 59 | ] 60 | 61 | [project.urls] 62 | Homepage = "https://github.com/arman-bd/httpmorph" 63 | Documentation = "https://github.com/arman-bd/httpmorph/blob/main/README.md" 64 | Repository = "https://github.com/arman-bd/httpmorph" 65 | Issues = "https://github.com/arman-bd/httpmorph/issues" 66 | 67 | [tool.setuptools] 68 | packages = ["httpmorph"] 69 | package-dir = {"" = "src"} 70 | # Don't automatically include LICENSE file (causes PyPI upload issues) 71 | license-files = [] 72 | 73 | [tool.setuptools.package-data] 74 | httpmorph = ["*.pyx", "*.pxd", "*.c", "*.h"] 75 | 76 | [tool.ruff] 77 | line-length = 100 78 | target-version = "py38" 79 | fix = true 80 | 81 | [tool.ruff.lint] 82 | select = [ 83 | "E", # pycodestyle errors 84 | "W", # pycodestyle warnings 85 | "F", # pyflakes 86 | "I", # isort 87 | "B", # flake8-bugbear 88 | "C4", # flake8-comprehensions 89 | "UP", # pyupgrade 90 | ] 91 | ignore = [ 92 | "E501", # line too long (handled by formatter) 93 | "B008", # function calls in argument defaults 94 | "C901", # too complex 95 | ] 96 | 97 | [tool.ruff.format] 98 | quote-style = "double" 99 | indent-style = "space" 100 | skip-magic-trailing-comma = false 101 | line-ending = "auto" 102 | 103 | [tool.mypy] 104 | python_version = "3.8" 105 | warn_return_any = true 106 | warn_unused_configs = true 107 | disallow_untyped_defs = true 108 | 109 | [tool.pytest.ini_options] 110 | testpaths = ["tests"] 111 | python_files = ["test_*.py"] 112 | python_classes = ["Test*"] 113 | python_functions = ["test_*"] 114 | addopts = "-v --strict-markers" 115 | markers = [ 116 | "slow: marks tests as slow (deselect with '-m \"not slow\"')", 117 | "benchmark: marks tests as benchmarks", 118 | "fingerprint: marks tests that check fingerprinting", 119 | "integration: marks tests that require network access", 120 | "ssl: marks tests that require SSL support", 121 | ] 122 | 123 | [tool.cibuildwheel] 124 | build = "cp38-* cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" 125 | skip = "*-musllinux_* *-win32 *-win_arm64" 126 | build-verbosity = 1 127 | 128 | [tool.cibuildwheel.linux] 129 | # Enable both x86_64 and aarch64 (ARM64) builds 130 | archs = ["x86_64", "aarch64"] 131 | before-all = [ 132 | "sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* || true", 133 | "sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* || true", 134 | "yum install -y cmake openssl-devel zlib-devel brotli-devel pkgconfig autoconf automake libtool golang || yum install -y openssl-devel zlib-devel brotli-devel pkgconfig autoconf automake libtool golang", 135 | ] 136 | before-build = "bash scripts/setup_vendors.sh" 137 | 138 | [tool.cibuildwheel.macos] 139 | before-all = [ 140 | "brew install cmake ninja go", 141 | ] 142 | before-build = "bash scripts/setup_vendors.sh" 143 | repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} --ignore-missing-dependencies" 144 | 145 | [tool.cibuildwheel.windows] 146 | before-all = [ 147 | "choco install cmake ninja golang -y", 148 | "vcpkg install brotli:x64-windows --clean-after-build || echo 'vcpkg brotli install skipped'", 149 | ] 150 | before-build = [ 151 | "bash scripts/setup_vendors.sh", 152 | ] 153 | environment = { VCPKG_ROOT = "C:/vcpkg" } 154 | -------------------------------------------------------------------------------- /src/core/internal/internal.h: -------------------------------------------------------------------------------- 1 | /** 2 | * internal.h - Internal shared definitions for httpmorph modules 3 | * 4 | * This header contains common structures, types, and includes used across 5 | * all httpmorph modules. It should NOT be exposed to external users. 6 | */ 7 | 8 | #ifndef INTERNAL_H 9 | #define INTERNAL_H 10 | 11 | /* Platform-specific feature macros */ 12 | #ifndef _WIN32 13 | #define _DEFAULT_SOURCE /* For usleep() and other BSD/POSIX extensions */ 14 | #define _POSIX_C_SOURCE 200809L 15 | #define _XOPEN_SOURCE 700 16 | #endif 17 | 18 | #ifdef _WIN32 19 | #define _CRT_SECURE_NO_WARNINGS 20 | #define strcasecmp _stricmp 21 | #define strncasecmp _strnicmp 22 | #define strdup _strdup 23 | #define close closesocket 24 | #define SNPRINTF_SIZE(size) ((int)(size)) 25 | #define SELECT_NFDS(sockfd) 0 26 | #else 27 | #define SNPRINTF_SIZE(size) (size) 28 | #define SELECT_NFDS(sockfd) ((sockfd) + 1) 29 | #endif 30 | 31 | /* Standard library includes */ 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | #include 38 | 39 | /* Platform-specific headers */ 40 | #ifdef _WIN32 41 | #include 42 | #include 43 | #include 44 | typedef int socklen_t; 45 | #else 46 | #include 47 | #include 48 | #include 49 | #include 50 | #include 51 | #include 52 | #include 53 | #include 54 | #include 55 | #endif 56 | 57 | /* BoringSSL/OpenSSL includes */ 58 | #ifdef _WIN32 59 | #include "../../include/boringssl_compat.h" 60 | #endif 61 | #include 62 | #include 63 | #include 64 | #include 65 | 66 | /* HTTP/2 support */ 67 | #ifdef HAVE_NGHTTP2 68 | #include 69 | #endif 70 | 71 | /* httpmorph public API */ 72 | #include "../../include/httpmorph.h" 73 | 74 | /* Internal modules */ 75 | #include "../../tls/browser_profiles.h" 76 | #include "../io_engine.h" 77 | #include "../connection_pool.h" 78 | 79 | /* ================================================================== 80 | * INTERNAL STRUCTURES 81 | * ================================================================== */ 82 | 83 | /* Forward declare buffer pool */ 84 | typedef struct httpmorph_buffer_pool httpmorph_buffer_pool_t; 85 | 86 | /** 87 | * HTTP client structure 88 | */ 89 | struct httpmorph_client { 90 | SSL_CTX *ssl_ctx; 91 | io_engine_t *io_engine; 92 | httpmorph_pool_t *pool; 93 | httpmorph_buffer_pool_t *buffer_pool; /* Buffer pool for response bodies */ 94 | 95 | /* Configuration */ 96 | uint32_t timeout_ms; 97 | bool follow_redirects; 98 | uint32_t max_redirects; 99 | 100 | /* Browser fingerprint */ 101 | const browser_profile_t *browser_profile; 102 | bool ssl_ctx_configured; /* Whether SSL_CTX has been configured for this profile */ 103 | }; 104 | 105 | /** 106 | * Cookie structure 107 | */ 108 | typedef struct cookie { 109 | char *name; 110 | char *value; 111 | char *domain; 112 | char *path; 113 | time_t expires; /* 0 = session cookie */ 114 | bool secure; 115 | bool http_only; 116 | struct cookie *next; 117 | } cookie_t; 118 | 119 | /** 120 | * Session structure 121 | */ 122 | struct httpmorph_session { 123 | httpmorph_client_t *client; 124 | const browser_profile_t *browser_profile; 125 | 126 | /* User-Agent from browser profile */ 127 | const char *user_agent; 128 | 129 | /* Connection pool for this session */ 130 | httpmorph_pool_t *pool; 131 | 132 | /* Cookie jar */ 133 | cookie_t *cookies; 134 | size_t cookie_count; 135 | 136 | /* HTTP/2 session */ 137 | #ifdef HAVE_NGHTTP2 138 | nghttp2_session *http2_session; 139 | #endif 140 | }; 141 | 142 | /** 143 | * Connection structure (internal use only) 144 | */ 145 | typedef struct { 146 | int sockfd; 147 | SSL *ssl; 148 | bool connected; 149 | bool tls_established; 150 | 151 | char *host; 152 | uint16_t port; 153 | 154 | /* HTTP/2 state */ 155 | bool http2_enabled; 156 | #ifdef HAVE_NGHTTP2 157 | nghttp2_session *session; 158 | #endif 159 | 160 | /* Timing */ 161 | uint64_t connect_time_us; 162 | uint64_t tls_time_us; 163 | 164 | /* Last used timestamp for pool management */ 165 | time_t last_used; 166 | } connection_t; 167 | 168 | /* ================================================================== 169 | * WINDOWS COMPATIBILITY 170 | * ================================================================== */ 171 | 172 | #ifdef _WIN32 173 | /* strnlen is not available in older MSVC versions */ 174 | #if _MSC_VER < 1900 /* Before Visual Studio 2015 */ 175 | static inline size_t strnlen(const char *s, size_t n) { 176 | const char *p = (const char *)memchr(s, 0, n); 177 | return p ? (size_t)(p - s) : n; 178 | } 179 | #endif 180 | 181 | /* strndup is not available on Windows MSVC */ 182 | static inline char* strndup(const char *s, size_t n) { 183 | size_t len = strnlen(s, n); 184 | char *dup = (char*)malloc(len + 1); 185 | if (dup) { 186 | memcpy(dup, s, len); 187 | dup[len] = '\0'; 188 | } 189 | return dup; 190 | } 191 | #endif 192 | 193 | #endif /* INTERNAL_H */ 194 | -------------------------------------------------------------------------------- /src/core/cookies.c: -------------------------------------------------------------------------------- 1 | /** 2 | * cookies.c - Cookie management 3 | */ 4 | 5 | #include "internal/cookies.h" 6 | 7 | /** 8 | * Free a cookie structure 9 | */ 10 | void httpmorph_cookie_free(cookie_t *cookie) { 11 | if (!cookie) return; 12 | free(cookie->name); 13 | free(cookie->value); 14 | free(cookie->domain); 15 | free(cookie->path); 16 | free(cookie); 17 | } 18 | 19 | /** 20 | * Parse Set-Cookie header and add to session 21 | */ 22 | void httpmorph_parse_set_cookie(httpmorph_session_t *session, 23 | const char *header_value, 24 | const char *request_domain) { 25 | if (!session || !header_value || !request_domain) { 26 | return; 27 | } 28 | 29 | cookie_t *cookie = calloc(1, sizeof(cookie_t)); 30 | if (!cookie) return; 31 | 32 | /* Parse cookie: name=value; attributes */ 33 | char *header_copy = strdup(header_value); 34 | if (!header_copy) { 35 | free(cookie); 36 | return; 37 | } 38 | 39 | /* Extract name=value */ 40 | char *semicolon = strchr(header_copy, ';'); 41 | if (semicolon) *semicolon = '\0'; 42 | 43 | char *equals = strchr(header_copy, '='); 44 | if (equals) { 45 | *equals = '\0'; 46 | cookie->name = strdup(header_copy); 47 | cookie->value = strdup(equals + 1); 48 | } else { 49 | free(header_copy); 50 | free(cookie); 51 | return; 52 | } 53 | 54 | /* Set default domain and path */ 55 | cookie->domain = strdup(request_domain); 56 | cookie->path = strdup("/"); 57 | cookie->expires = 0; /* Session cookie by default */ 58 | cookie->secure = false; 59 | cookie->http_only = false; 60 | 61 | /* Parse attributes if present */ 62 | if (semicolon) { 63 | char *attr = semicolon + 1; 64 | while (attr && *attr) { 65 | /* Skip whitespace */ 66 | while (*attr == ' ') attr++; 67 | 68 | if (strncasecmp(attr, "Domain=", 7) == 0) { 69 | attr += 7; 70 | char *end = strchr(attr, ';'); 71 | size_t len = end ? (size_t)(end - attr) : strlen(attr); 72 | free(cookie->domain); 73 | cookie->domain = strndup(attr, len); 74 | attr = end; 75 | } else if (strncasecmp(attr, "Path=", 5) == 0) { 76 | attr += 5; 77 | char *end = strchr(attr, ';'); 78 | size_t len = end ? (size_t)(end - attr) : strlen(attr); 79 | free(cookie->path); 80 | cookie->path = strndup(attr, len); 81 | attr = end; 82 | } else if (strncasecmp(attr, "Secure", 6) == 0) { 83 | cookie->secure = true; 84 | attr = strchr(attr, ';'); 85 | } else if (strncasecmp(attr, "HttpOnly", 8) == 0) { 86 | cookie->http_only = true; 87 | attr = strchr(attr, ';'); 88 | } else { 89 | /* Skip unknown attribute */ 90 | attr = strchr(attr, ';'); 91 | } 92 | 93 | if (attr) attr++; 94 | } 95 | } 96 | 97 | free(header_copy); 98 | 99 | /* Add to session's cookie list */ 100 | cookie->next = session->cookies; 101 | session->cookies = cookie; 102 | session->cookie_count++; 103 | } 104 | 105 | /** 106 | * Get cookies for a request as a Cookie header value 107 | */ 108 | char* httpmorph_get_cookies_for_request(httpmorph_session_t *session, 109 | const char *domain, 110 | const char *path, 111 | bool is_secure) { 112 | if (!session || !domain || !path) return NULL; 113 | if (session->cookie_count == 0) return NULL; 114 | 115 | /* Build cookie header value with bounds checking */ 116 | const size_t buffer_size = 4096; 117 | char *cookie_header = malloc(buffer_size); 118 | if (!cookie_header) return NULL; 119 | 120 | cookie_header[0] = '\0'; 121 | size_t used = 0; 122 | bool first = true; 123 | 124 | cookie_t *cookie = session->cookies; 125 | while (cookie) { 126 | /* Check if cookie matches request */ 127 | bool domain_match = (strcasecmp(cookie->domain, domain) == 0 || 128 | (cookie->domain[0] == '.' && strstr(domain, cookie->domain + 1) != NULL)); 129 | bool path_match = (strncmp(cookie->path, path, strlen(cookie->path)) == 0); 130 | bool secure_match = (!cookie->secure || is_secure); 131 | 132 | if (domain_match && path_match && secure_match) { 133 | /* Calculate space needed: "; " + name + "=" + value */ 134 | size_t needed = strlen(cookie->name) + 1 + strlen(cookie->value); 135 | if (!first) needed += 2; /* "; " prefix */ 136 | 137 | /* Check if we have space (leave room for null terminator) */ 138 | if (used + needed >= buffer_size - 1) { 139 | /* Buffer would overflow - stop adding cookies */ 140 | break; 141 | } 142 | 143 | /* Safe concatenation with bounds checking */ 144 | if (!first) { 145 | used += snprintf(cookie_header + used, buffer_size - used, "; "); 146 | } 147 | used += snprintf(cookie_header + used, buffer_size - used, "%s=%s", 148 | cookie->name, cookie->value); 149 | first = false; 150 | } 151 | 152 | cookie = cookie->next; 153 | } 154 | 155 | if (cookie_header[0] == '\0') { 156 | free(cookie_header); 157 | return NULL; 158 | } 159 | 160 | return cookie_header; 161 | } 162 | -------------------------------------------------------------------------------- /scripts/darwin/test_build_local.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # test_macos_build_local.sh - Test macOS wheel build locally 4 | # 5 | # This script mirrors the macOS build pipeline from .github/workflows/release.yml 6 | # but runs locally for testing before pushing to CI. 7 | # 8 | 9 | set -e 10 | 11 | SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 12 | ROOT_DIR="$(dirname "$(dirname "$SCRIPT_DIR")")" 13 | 14 | echo "================================" 15 | echo "Local macOS Build Test" 16 | echo "================================" 17 | echo "" 18 | echo "This script tests the macOS wheel build pipeline locally." 19 | echo "" 20 | 21 | # Parse arguments 22 | VERBOSE=false 23 | CLEAN=false 24 | 25 | while [[ $# -gt 0 ]]; do 26 | case $1 in 27 | --verbose|-v) 28 | VERBOSE=true 29 | shift 30 | ;; 31 | --clean) 32 | CLEAN=true 33 | shift 34 | ;; 35 | --help|-h) 36 | echo "Usage: $0 [OPTIONS]" 37 | echo "" 38 | echo "Options:" 39 | echo " --clean Clean build artifacts before building" 40 | echo " --verbose, -v Show verbose output" 41 | echo " --help, -h Show this help message" 42 | echo "" 43 | echo "Examples:" 44 | echo " $0 # Build wheel for current Python" 45 | echo " $0 --clean # Clean and build" 46 | echo " $0 --verbose # Show detailed build output" 47 | echo "" 48 | echo "Note: Uses the currently active python3 version" 49 | echo "" 50 | exit 0 51 | ;; 52 | *) 53 | echo "Unknown option: $1" 54 | echo "Use --help for usage information" 55 | exit 1 56 | ;; 57 | esac 58 | done 59 | 60 | PYTHON_VERSION=$(python3 --version) 61 | 62 | echo "Configuration:" 63 | echo " Platform: macOS $(sw_vers -productVersion)" 64 | echo " Architecture: $(uname -m)" 65 | echo " Python: $PYTHON_VERSION" 66 | echo " Output Dir: $ROOT_DIR/wheelhouse" 67 | echo "" 68 | 69 | # Create wheelhouse directory 70 | mkdir -p "$ROOT_DIR/wheelhouse" 71 | 72 | # Clean if requested 73 | if $CLEAN; then 74 | echo "🧹 Cleaning build artifacts..." 75 | rm -rf "$ROOT_DIR/wheelhouse"/*.whl 76 | rm -rf "$ROOT_DIR/build" 77 | rm -rf "$ROOT_DIR/dist" 78 | rm -rf "$ROOT_DIR"/*.egg-info 79 | rm -rf "$ROOT_DIR/src"/*.egg-info 80 | echo "✅ Clean complete" 81 | echo "" 82 | fi 83 | 84 | # Check for required tools 85 | echo "🔍 Checking prerequisites..." 86 | 87 | # Check for Homebrew 88 | if ! command -v brew &> /dev/null; then 89 | echo "❌ Error: Homebrew is not installed." 90 | echo " Install it from: https://brew.sh" 91 | exit 1 92 | fi 93 | echo " ✓ Homebrew found" 94 | 95 | # Check for Python 3 96 | if ! command -v python3 &> /dev/null; then 97 | echo "❌ Error: Python 3 is not installed." 98 | exit 1 99 | fi 100 | echo " ✓ Python found: $PYTHON_VERSION" 101 | 102 | # Check for build tool 103 | if ! python3 -m pip list 2>/dev/null | grep -q "^build "; then 104 | echo "⚠️ build tool not found, installing..." 105 | python3 -m pip install build --user -q 106 | fi 107 | echo " ✓ build tool found" 108 | 109 | echo "" 110 | 111 | # Install build dependencies (mirrors the before-all step) 112 | echo "📦 Installing build dependencies..." 113 | BREW_PACKAGES="cmake ninja go" 114 | for pkg in $BREW_PACKAGES; do 115 | if brew list "$pkg" &>/dev/null; then 116 | echo " ✓ $pkg already installed" 117 | else 118 | echo " Installing $pkg..." 119 | if $VERBOSE; then 120 | brew install "$pkg" 121 | else 122 | brew install "$pkg" >/dev/null 2>&1 123 | fi 124 | fi 125 | done 126 | echo "✅ Dependencies installed" 127 | echo "" 128 | 129 | # Setup vendor dependencies (mirrors the before-build step) 130 | echo "🔨 Building vendor dependencies..." 131 | echo " This may take several minutes (BoringSSL, nghttp2)..." 132 | if $VERBOSE; then 133 | bash "$ROOT_DIR/scripts/darwin/setup_vendors.sh" 134 | else 135 | bash "$ROOT_DIR/scripts/darwin/setup_vendors.sh" 2>&1 | grep -E "(===|✓|✅|⊘|ERROR|WARNING)" || true 136 | fi 137 | echo "" 138 | 139 | # Build wheel using python -m build 140 | echo "🔨 Building wheel..." 141 | echo "" 142 | 143 | cd "$ROOT_DIR" 144 | 145 | # Clean previous builds 146 | rm -rf build/ dist/ 147 | 148 | if $VERBOSE; then 149 | python3 -m build --wheel 150 | else 151 | python3 -m build --wheel 2>&1 | grep -E "(Successfully|ERROR|error|failed|Building)" || true 152 | fi 153 | 154 | BUILD_EXIT_CODE=$? 155 | 156 | # Copy wheel to wheelhouse if build succeeded 157 | if [[ $BUILD_EXIT_CODE -eq 0 ]] && [[ -d dist ]]; then 158 | mkdir -p wheelhouse 159 | cp dist/*.whl wheelhouse/ 2>/dev/null || true 160 | fi 161 | 162 | echo "" 163 | echo "════════════════════════════════════════════════════════════════" 164 | if [[ $BUILD_EXIT_CODE -eq 0 ]]; then 165 | echo "✅ Build completed successfully!" 166 | echo "════════════════════════════════════════════════════════════════" 167 | echo "" 168 | echo "Wheels are available in: $ROOT_DIR/wheelhouse" 169 | echo "" 170 | ls -lh "$ROOT_DIR/wheelhouse"/*.whl 2>/dev/null || echo "No wheels found in wheelhouse" 171 | echo "" 172 | echo "To install locally:" 173 | echo " pip install wheelhouse/httpmorph-*.whl" 174 | echo "" 175 | echo "To test the wheel:" 176 | echo " python3 -c 'import httpmorph; print(httpmorph.version())'" 177 | else 178 | echo "❌ Build failed with exit code $BUILD_EXIT_CODE" 179 | echo "════════════════════════════════════════════════════════════════" 180 | exit $BUILD_EXIT_CODE 181 | fi 182 | -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.4/benchmark.md: -------------------------------------------------------------------------------- 1 | # httpmorph Benchmark Results 2 | 3 | **Version:** 0.2.4 | **Generated:** 2025-11-07 4 | 5 | ## System Information 6 | 7 | | Property | Value | 8 | |----------|-------| 9 | | **OS** | Darwin (macOS-15.6-arm64-arm-64bit) | 10 | | **Processor** | arm | 11 | | **CPU Cores** | 10 | 12 | | **Memory** | 16.0 GB | 13 | | **Python** | 3.11.5 (CPython) | 14 | 15 | ## Test Configuration 16 | 17 | - **Sequential Requests:** 25 (warmup: 5) 18 | - **Concurrent Requests:** 25 (workers: 5) 19 | 20 | ## Library Versions 21 | 22 | | Library | Version | Status | 23 | |---------|---------|--------| 24 | | **httpmorph** | `0.2.4` | Installed | 25 | | **requests** | `2.31.0` | Installed | 26 | | **httpx** | `0.27.2` | Installed | 27 | | **aiohttp** | `3.8.5` | Installed | 28 | | **urllib3** | `1.26.16` | Installed | 29 | | **urllib** | `built-in (Python 3.11.5)` | Installed | 30 | | **pycurl** | `PycURL/7.45.2 libcurl/8.7.1 (SecureTransport) LibreSSL/3.3.6 zlib/1.2.12 nghttp2/1.64.0` | Installed | 31 | | **curl_cffi** | `0.13.0` | Installed | 32 | 33 | ## Sequential Tests (Lower is Better) 34 | 35 | Mean response time in milliseconds 36 | 37 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 38 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 39 | | **curl_cffi** | 0.43ms | ERROR | ERROR | 1576.17ms | 65.70ms | 101.59ms | 94.96ms | 40 | | **httpmorph** | 0.31ms | 891.02ms | 1620.74ms | 1679.02ms | 62.20ms | 98.39ms | 97.47ms | 41 | | **httpx** | 1.14ms | 596.42ms | 338.85ms | 367.00ms | 66.42ms | 96.58ms | 102.12ms | 42 | | **pycurl** | 0.43ms | 1347.29ms | 1700.27ms | ERROR | 62.17ms | 103.57ms | 100.06ms | 43 | | **requests** | 0.94ms | 349.35ms | N/A | 338.11ms | 31.50ms | N/A | 26.71ms | 44 | | **urllib** | 15.83ms | 1094.31ms | N/A | 1785.94ms | 65.10ms | N/A | 120.19ms | 45 | | **urllib3** | 0.36ms | 397.95ms | N/A | 387.72ms | 30.93ms | N/A | 36.31ms | 46 | 47 | **Winners (Sequential):** 48 | - Local HTTP: **httpmorph** (0.31ms) 49 | - Proxy HTTP: **requests** (349.35ms) 50 | - Proxy HTTP2: **httpx** (338.85ms) 51 | - Proxy HTTPs: **requests** (338.11ms) 52 | - Remote HTTP: **urllib3** (30.93ms) 53 | - Remote HTTP2: **httpx** (96.58ms) 54 | - Remote HTTPs: **requests** (26.71ms) 55 | 56 | ## Concurrent Tests (Higher is Better) 57 | 58 | Throughput in requests per second 59 | 60 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 61 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 62 | | **curl_cffi** | 1645.08 | 2.31 | 2.73 | ERROR | 68.06 | 46.08 | 48.57 | 63 | | **httpmorph** | 6769.95 | 5.43 | 3.24 | 2.31 | 76.03 | 50.09 | 51.35 | 64 | | **httpx** | 803.58 | 10.68 | ERROR | 3.84 | 75.23 | 49.41 | 48.82 | 65 | | **pycurl** | 2757.38 | 2.35 | 2.58 | 2.39 | 70.46 | 46.50 | 36.88 | 66 | | **requests** | 1040.44 | 10.51 | N/A | 6.45 | 126.58 | N/A | 57.34 | 67 | | **urllib** | 25.74 | 2.73 | N/A | 2.25 | 72.05 | N/A | 28.49 | 68 | | **urllib3** | 1422.15 | 5.35 | N/A | 7.94 | 103.52 | N/A | 57.53 | 69 | 70 | **Winners (Concurrent):** 71 | - Local HTTP: **httpmorph** (6769.95 req/s) 72 | - Proxy HTTP: **httpx** (10.68 req/s) 73 | - Proxy HTTP2: **httpmorph** (3.24 req/s) 74 | - Proxy HTTPs: **urllib3** (7.94 req/s) 75 | - Remote HTTP: **requests** (126.58 req/s) 76 | - Remote HTTP2: **httpmorph** (50.09 req/s) 77 | - Remote HTTPs: **urllib3** (57.53 req/s) 78 | 79 | ## Async Tests (Higher is Better) 80 | 81 | Throughput in requests per second 82 | 83 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 84 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 85 | | **aiohttp** | 165.03 | 4.39 | N/A | 10.76 | 254.00 | N/A | 115.49 | 86 | | **httpmorph** | 193.16 | 66.86 | 3.78 | 3.79 | 241.08 | 170.36 | 165.16 | 87 | | **httpx** | 158.09 | 4.29 | 4.01 | 3.99 | 221.29 | 137.07 | 165.55 | 88 | 89 | **Winners (Async):** 90 | - Local HTTP: **httpmorph** (193.16 req/s) 91 | - Proxy HTTP: **httpmorph** (66.86 req/s) 92 | - Proxy HTTP2: **httpx** (4.01 req/s) 93 | - Proxy HTTPs: **aiohttp** (10.76 req/s) 94 | - Remote HTTP: **aiohttp** (254.00 req/s) 95 | - Remote HTTP2: **httpmorph** (170.36 req/s) 96 | - Remote HTTPs: **httpx** (165.55 req/s) 97 | 98 | ## Overall Performance Summary 99 | 100 | ### Sequential Tests: httpmorph vs requests Speedup 101 | 102 | | Test | httpmorph | requests | Speedup | 103 | |------|----------:|---------:|--------:| 104 | | Local HTTP | 0.31ms | 0.94ms | **3.01x** faster | 105 | | Proxy HTTP | 891.02ms | 349.35ms | 0.39x slower | 106 | | Proxy HTTPs | 1679.02ms | 338.11ms | 0.20x slower | 107 | | Remote HTTP | 62.20ms | 31.50ms | 0.51x slower | 108 | | Remote HTTPs | 97.47ms | 26.71ms | 0.27x slower | 109 | 110 | ### Concurrent Tests: httpmorph vs requests Speedup 111 | 112 | | Test | httpmorph | requests | Speedup | 113 | |------|----------:|---------:|--------:| 114 | | Local HTTP | 6769.95 req/s | 1040.44 req/s | **6.51x** faster | 115 | | Proxy HTTP | 5.43 req/s | 10.51 req/s | 0.52x slower | 116 | | Proxy HTTPs | 2.31 req/s | 6.45 req/s | 0.36x slower | 117 | | Remote HTTP | 76.03 req/s | 126.58 req/s | 0.60x slower | 118 | | Remote HTTPs | 51.35 req/s | 57.34 req/s | 0.90x slower | 119 | 120 | ### Async Tests: httpmorph vs httpx Speedup 121 | 122 | | Test | httpmorph | httpx | Speedup | 123 | |------|----------:|------:|--------:| 124 | | Local HTTP | 193.16 req/s | 158.09 req/s | **1.22x** faster | 125 | | Proxy HTTP | 66.86 req/s | 4.29 req/s | **15.60x** faster | 126 | | Proxy HTTP2 | 3.78 req/s | 4.01 req/s | 0.94x slower | 127 | | Proxy HTTPs | 3.79 req/s | 3.99 req/s | 0.95x slower | 128 | | Remote HTTP | 241.08 req/s | 221.29 req/s | **1.09x** faster | 129 | | Remote HTTP2 | 170.36 req/s | 137.07 req/s | **1.24x** faster | 130 | | Remote HTTPs | 165.16 req/s | 165.55 req/s | 1.00x slower | 131 | 132 | --- 133 | *Generated by httpmorph benchmark suite* 134 | -------------------------------------------------------------------------------- /src/core/http2_session_manager.h: -------------------------------------------------------------------------------- 1 | /** 2 | * http2_session_manager.h - HTTP/2 Session Manager for Concurrent Multiplexing 3 | * 4 | * Manages concurrent HTTP/2 streams on a single session. 5 | * Allows multiple threads to submit requests and share one HTTP/2 connection. 6 | */ 7 | 8 | #ifndef HTTPMORPH_HTTP2_SESSION_MANAGER_H 9 | #define HTTPMORPH_HTTP2_SESSION_MANAGER_H 10 | 11 | #ifdef HAVE_NGHTTP2 12 | 13 | #include 14 | #include 15 | #ifndef _WIN32 16 | #include 17 | #endif 18 | #include 19 | #include 20 | 21 | /* Forward declarations */ 22 | typedef struct http2_session_manager http2_session_manager_t; 23 | typedef struct http2_pending_stream http2_pending_stream_t; 24 | 25 | /** 26 | * Pending stream structure 27 | * Tracks a single stream until completion 28 | */ 29 | struct http2_pending_stream { 30 | int32_t stream_id; 31 | void *stream_data; /* http2_stream_data_t* - avoid circular dependency */ 32 | 33 | /* Synchronization for this specific stream */ 34 | pthread_mutex_t stream_mutex; 35 | pthread_cond_t stream_complete; 36 | bool completed; 37 | bool has_error; 38 | 39 | /* Linked list */ 40 | http2_pending_stream_t *next; 41 | }; 42 | 43 | /** 44 | * HTTP/2 Session Manager 45 | * Coordinates concurrent access to a single HTTP/2 session 46 | */ 47 | struct http2_session_manager { 48 | /* HTTP/2 session */ 49 | nghttp2_session *session; 50 | nghttp2_session_callbacks *callbacks; 51 | 52 | /* Connection */ 53 | SSL *ssl; 54 | int sockfd; 55 | 56 | /* I/O thread management */ 57 | pthread_t io_thread; 58 | pthread_mutex_t mutex; /* Protects session and stream list */ 59 | bool io_thread_running; 60 | bool shutdown_requested; 61 | 62 | /* Stream tracking */ 63 | http2_pending_stream_t *pending_streams; 64 | int active_stream_count; 65 | 66 | /* Statistics */ 67 | uint64_t total_streams_submitted; 68 | uint64_t total_streams_completed; 69 | }; 70 | 71 | /* === Session Manager Lifecycle === */ 72 | 73 | /** 74 | * Create a new HTTP/2 session manager 75 | * 76 | * @param session Existing nghttp2_session (takes ownership) 77 | * @param callbacks nghttp2 callbacks (takes ownership) 78 | * @param ssl SSL connection 79 | * @param sockfd Socket file descriptor 80 | * @return New session manager or NULL on error 81 | */ 82 | http2_session_manager_t* http2_session_manager_create( 83 | nghttp2_session *session, 84 | nghttp2_session_callbacks *callbacks, 85 | SSL *ssl, 86 | int sockfd 87 | ); 88 | 89 | /** 90 | * Destroy a session manager 91 | * Stops I/O thread, cleans up resources 92 | * 93 | * @param mgr Session manager to destroy 94 | */ 95 | void http2_session_manager_destroy(http2_session_manager_t *mgr); 96 | 97 | /** 98 | * Start the I/O thread 99 | * Must be called before submitting streams 100 | * 101 | * @param mgr Session manager 102 | * @return 0 on success, -1 on error 103 | */ 104 | int http2_session_manager_start(http2_session_manager_t *mgr); 105 | 106 | /** 107 | * Stop the I/O thread 108 | * Waits for thread to finish 109 | * 110 | * @param mgr Session manager 111 | */ 112 | void http2_session_manager_stop(http2_session_manager_t *mgr); 113 | 114 | /* === Stream Operations === */ 115 | 116 | /** 117 | * Submit a new HTTP/2 stream 118 | * Non-blocking operation that queues the stream 119 | * 120 | * @param mgr Session manager 121 | * @param stream_data Stream-specific data (takes ownership) - void* to http2_stream_data_t* 122 | * @param pri_spec Priority specification (can be NULL for default priority) 123 | * @param hdrs Request headers 124 | * @param hdr_count Number of headers 125 | * @param data_prd Data provider for request body (can be NULL) 126 | * @param stream_id_out Output parameter for assigned stream ID 127 | * @return 0 on success, -1 on error 128 | */ 129 | int http2_session_manager_submit_stream( 130 | http2_session_manager_t *mgr, 131 | void *stream_data, 132 | const nghttp2_priority_spec *pri_spec, 133 | const nghttp2_nv *hdrs, 134 | size_t hdr_count, 135 | nghttp2_data_provider *data_prd, 136 | int32_t *stream_id_out 137 | ); 138 | 139 | /** 140 | * Wait for a stream to complete 141 | * Blocking operation with timeout 142 | * 143 | * @param mgr Session manager 144 | * @param stream_id Stream ID to wait for 145 | * @param timeout_ms Timeout in milliseconds 146 | * @return 0 on success, -1 on timeout or error 147 | */ 148 | int http2_session_manager_wait_for_stream( 149 | http2_session_manager_t *mgr, 150 | int32_t stream_id, 151 | uint32_t timeout_ms 152 | ); 153 | 154 | /** 155 | * Remove a completed stream from tracking 156 | * Frees resources associated with the stream 157 | * 158 | * @param mgr Session manager 159 | * @param stream_id Stream ID to remove 160 | */ 161 | void http2_session_manager_remove_stream( 162 | http2_session_manager_t *mgr, 163 | int32_t stream_id 164 | ); 165 | 166 | /* === Internal Helpers (used by callbacks) === */ 167 | 168 | /** 169 | * Mark a stream as completed 170 | * Called by nghttp2 callbacks when stream finishes 171 | * 172 | * @param mgr Session manager 173 | * @param stream_id Stream ID that completed 174 | * @param has_error Whether the stream had an error 175 | */ 176 | void http2_session_manager_mark_stream_complete( 177 | http2_session_manager_t *mgr, 178 | int32_t stream_id, 179 | bool has_error 180 | ); 181 | 182 | /** 183 | * Find a pending stream by ID 184 | * Must be called with mgr->mutex held 185 | * 186 | * @param mgr Session manager 187 | * @param stream_id Stream ID to find 188 | * @return Pending stream or NULL if not found 189 | */ 190 | http2_pending_stream_t* http2_session_manager_find_stream( 191 | http2_session_manager_t *mgr, 192 | int32_t stream_id 193 | ); 194 | 195 | #endif /* HAVE_NGHTTP2 */ 196 | 197 | #endif /* HTTPMORPH_HTTP2_SESSION_MANAGER_H */ 198 | -------------------------------------------------------------------------------- /benchmarks/results/darwin/0.2.2/benchmark.md: -------------------------------------------------------------------------------- 1 | # httpmorph Benchmark Results 2 | 3 | **Version:** 0.2.2 | **Generated:** 2025-11-06 4 | 5 | ## System Information 6 | 7 | | Property | Value | 8 | |----------|-------| 9 | | **OS** | Darwin (macOS-15.6-arm64-arm-64bit-Mach-O) | 10 | | **Processor** | arm | 11 | | **CPU Cores** | 10 | 12 | | **Memory** | 16.0 GB | 13 | | **Python** | 3.14.0 (CPython) | 14 | 15 | ## Test Configuration 16 | 17 | - **Sequential Requests:** 25 (warmup: 5) 18 | - **Concurrent Requests:** 25 (workers: 5) 19 | 20 | ## Library Versions 21 | 22 | | Library | Version | Status | 23 | |---------|---------|--------| 24 | | **httpmorph** | `0.2.2` | Installed | 25 | | **requests** | `2.32.5` | Installed | 26 | | **httpx** | `0.28.1` | Installed | 27 | | **aiohttp** | `3.13.1` | Installed | 28 | | **urllib3** | `2.5.0` | Installed | 29 | | **urllib** | `built-in (Python 3.14.0)` | Installed | 30 | | **pycurl** | `PycURL/7.45.7 libcurl/8.16.0-DEV OpenSSL/3.5.2 zlib/1.3.1 brotli/1.1.0 libssh2/1.11.1_DEV nghttp2/1.67.0` | Installed | 31 | | **curl_cffi** | `0.13.0` | Installed | 32 | 33 | ## Sequential Tests (Lower is Better) 34 | 35 | Mean response time in milliseconds 36 | 37 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 38 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 39 | | **curl_cffi** | 1.25ms | 1212.26ms | 1646.60ms | 2131.77ms | 64.09ms | 100.49ms | 99.58ms | 40 | | **httpmorph** | 1.34ms | 1260.38ms | 1922.90ms | 2272.14ms | 61.63ms | 103.33ms | 99.36ms | 41 | | **httpx** | 5.54ms | 411.45ms | 433.30ms | 404.70ms | 62.93ms | 101.15ms | 97.87ms | 42 | | **pycurl** | 0.40ms | 1777.37ms | 2499.49ms | 2265.31ms | 65.92ms | 97.96ms | 103.89ms | 43 | | **requests** | 0.76ms | 351.83ms | N/A | 437.27ms | 26.97ms | N/A | 31.75ms | 44 | | **urllib** | 7.94ms | 1624.63ms | N/A | 2161.49ms | 63.97ms | N/A | 126.57ms | 45 | | **urllib3** | 0.61ms | 345.31ms | N/A | 429.76ms | 30.93ms | N/A | 27.19ms | 46 | 47 | **Winners (Sequential):** 48 | - Local HTTP: **pycurl** (0.40ms) 49 | - Proxy HTTP: **urllib3** (345.31ms) 50 | - Proxy HTTP2: **httpx** (433.30ms) 51 | - Proxy HTTPs: **httpx** (404.70ms) 52 | - Remote HTTP: **requests** (26.97ms) 53 | - Remote HTTP2: **pycurl** (97.96ms) 54 | - Remote HTTPs: **urllib3** (27.19ms) 55 | 56 | ## Concurrent Tests (Higher is Better) 57 | 58 | Throughput in requests per second 59 | 60 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 61 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 62 | | **curl_cffi** | 253.97 | 2.86 | 2.16 | 1.90 | 74.04 | 47.88 | 47.67 | 63 | | **httpmorph** | 926.81 | 2.04 | 1.62 | 1.92 | 65.03 | 51.93 | 50.31 | 64 | | **httpx** | 763.12 | 4.30 | 7.23 | 8.71 | 72.56 | 47.69 | 49.16 | 65 | | **pycurl** | 2156.50 | 4.99 | 1.75 | 1.31 | 68.81 | 50.28 | 44.22 | 66 | | **requests** | 757.97 | 4.26 | N/A | 3.51 | 119.46 | N/A | 93.74 | 67 | | **urllib** | 259.93 | 2.65 | N/A | 2.11 | 69.41 | N/A | 45.28 | 68 | | **urllib3** | 1262.19 | 4.30 | N/A | 3.79 | 111.02 | N/A | 104.16 | 69 | 70 | **Winners (Concurrent):** 71 | - Local HTTP: **pycurl** (2156.50 req/s) 72 | - Proxy HTTP: **pycurl** (4.99 req/s) 73 | - Proxy HTTP2: **httpx** (7.23 req/s) 74 | - Proxy HTTPs: **httpx** (8.71 req/s) 75 | - Remote HTTP: **requests** (119.46 req/s) 76 | - Remote HTTP2: **httpmorph** (51.93 req/s) 77 | - Remote HTTPs: **urllib3** (104.16 req/s) 78 | 79 | ## Async Tests (Higher is Better) 80 | 81 | Throughput in requests per second 82 | 83 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 84 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 85 | | **aiohttp** | 133.56 | 4.27 | N/A | 2.33 | 247.87 | N/A | 176.24 | 86 | | **httpmorph** | 154.43 | 65.93 | 3.72 | 3.72 | 246.05 | 154.19 | 165.44 | 87 | | **httpx** | 156.96 | 4.28 | 11.19 | 3.60 | 220.18 | 141.84 | 134.49 | 88 | 89 | **Winners (Async):** 90 | - Local HTTP: **httpx** (156.96 req/s) 91 | - Proxy HTTP: **httpmorph** (65.93 req/s) 92 | - Proxy HTTP2: **httpx** (11.19 req/s) 93 | - Proxy HTTPs: **httpmorph** (3.72 req/s) 94 | - Remote HTTP: **aiohttp** (247.87 req/s) 95 | - Remote HTTP2: **httpmorph** (154.19 req/s) 96 | - Remote HTTPs: **aiohttp** (176.24 req/s) 97 | 98 | ## Overall Performance Summary 99 | 100 | ### Sequential Tests: httpmorph vs requests Speedup 101 | 102 | | Test | httpmorph | requests | Speedup | 103 | |------|----------:|---------:|--------:| 104 | | Local HTTP | 1.34ms | 0.76ms | 0.57x slower | 105 | | Proxy HTTP | 1260.38ms | 351.83ms | 0.28x slower | 106 | | Proxy HTTPs | 2272.14ms | 437.27ms | 0.19x slower | 107 | | Remote HTTP | 61.63ms | 26.97ms | 0.44x slower | 108 | | Remote HTTPs | 99.36ms | 31.75ms | 0.32x slower | 109 | 110 | ### Concurrent Tests: httpmorph vs requests Speedup 111 | 112 | | Test | httpmorph | requests | Speedup | 113 | |------|----------:|---------:|--------:| 114 | | Local HTTP | 926.81 req/s | 757.97 req/s | **1.22x** faster | 115 | | Proxy HTTP | 2.04 req/s | 4.26 req/s | 0.48x slower | 116 | | Proxy HTTPs | 1.92 req/s | 3.51 req/s | 0.55x slower | 117 | | Remote HTTP | 65.03 req/s | 119.46 req/s | 0.54x slower | 118 | | Remote HTTPs | 50.31 req/s | 93.74 req/s | 0.54x slower | 119 | 120 | ### Async Tests: httpmorph vs httpx Speedup 121 | 122 | | Test | httpmorph | httpx | Speedup | 123 | |------|----------:|------:|--------:| 124 | | Local HTTP | 154.43 req/s | 156.96 req/s | 0.98x slower | 125 | | Proxy HTTP | 65.93 req/s | 4.28 req/s | **15.40x** faster | 126 | | Proxy HTTP2 | 3.72 req/s | 11.19 req/s | 0.33x slower | 127 | | Proxy HTTPs | 3.72 req/s | 3.60 req/s | **1.03x** faster | 128 | | Remote HTTP | 246.05 req/s | 220.18 req/s | **1.12x** faster | 129 | | Remote HTTP2 | 154.19 req/s | 141.84 req/s | **1.09x** faster | 130 | | Remote HTTPs | 165.44 req/s | 134.49 req/s | **1.23x** faster | 131 | 132 | --- 133 | *Generated by httpmorph benchmark suite* 134 | -------------------------------------------------------------------------------- /src/tls/browser_profiles.h: -------------------------------------------------------------------------------- 1 | /** 2 | * browser_profiles.h - Browser TLS/HTTP fingerprint profiles 3 | * 4 | * Contains detailed profiles for popular browsers to enable accurate impersonation 5 | */ 6 | 7 | #ifndef BROWSER_PROFILES_H 8 | #define BROWSER_PROFILES_H 9 | 10 | #include 11 | #include 12 | 13 | #ifdef __cplusplus 14 | extern "C" { 15 | #endif 16 | 17 | /* Maximum sizes */ 18 | #define MAX_CIPHER_SUITES 32 19 | #define MAX_EXTENSIONS 24 20 | #define MAX_CURVES 16 21 | #define MAX_SIG_ALGORITHMS 24 22 | #define MAX_ALPN_PROTOCOLS 8 23 | #define MAX_HTTP2_SETTINGS 16 24 | 25 | /* OS types for user agent generation */ 26 | typedef enum { 27 | OS_MACOS = 0, /* macOS (default) */ 28 | OS_WINDOWS = 1, /* Windows */ 29 | OS_LINUX = 2, /* Linux */ 30 | } os_type_t; 31 | 32 | /* TLS version */ 33 | typedef enum { 34 | TLS_VERSION_1_0 = 0x0301, 35 | TLS_VERSION_1_1 = 0x0302, 36 | TLS_VERSION_1_2 = 0x0303, 37 | TLS_VERSION_1_3 = 0x0304, 38 | } tls_version_t; 39 | 40 | /* TLS extension types */ 41 | typedef enum { 42 | TLS_EXT_SERVER_NAME = 0, 43 | TLS_EXT_STATUS_REQUEST = 5, 44 | TLS_EXT_SUPPORTED_GROUPS = 10, 45 | TLS_EXT_EC_POINT_FORMATS = 11, 46 | TLS_EXT_SIGNATURE_ALGORITHMS = 13, 47 | TLS_EXT_ALPN = 16, 48 | TLS_EXT_SIGNED_CERTIFICATE_TIMESTAMP = 18, 49 | TLS_EXT_PADDING = 21, 50 | TLS_EXT_EXTENDED_MASTER_SECRET = 23, 51 | TLS_EXT_COMPRESS_CERTIFICATE = 27, 52 | TLS_EXT_SESSION_TICKET = 35, 53 | TLS_EXT_PRE_SHARED_KEY = 41, 54 | TLS_EXT_SUPPORTED_VERSIONS = 43, 55 | TLS_EXT_PSK_KEY_EXCHANGE_MODES = 45, 56 | TLS_EXT_KEY_SHARE = 51, 57 | TLS_EXT_APPLICATION_SETTINGS = 17513, 58 | TLS_EXT_ENCRYPTED_CLIENT_HELLO = 65037, 59 | TLS_EXT_RENEGOTIATION_INFO = 65281, 60 | TLS_EXT_GREASE = 0x0a0a, /* GREASE values vary */ 61 | } tls_extension_t; 62 | 63 | /* Browser TLS profile */ 64 | typedef struct { 65 | const char *name; /* e.g., "Chrome 131" */ 66 | const char *version; /* e.g., "131.0.6778.109" */ 67 | const char *user_agent; /* Full user agent string (macOS - default) */ 68 | 69 | /* OS-specific user agents */ 70 | const char *user_agent_windows; /* Windows user agent */ 71 | const char *user_agent_linux; /* Linux user agent */ 72 | 73 | /* TLS configuration */ 74 | tls_version_t min_tls_version; 75 | tls_version_t max_tls_version; 76 | 77 | /* Cipher suites in exact order */ 78 | uint16_t cipher_suites[MAX_CIPHER_SUITES]; 79 | int cipher_suite_count; 80 | 81 | /* Extensions in exact order */ 82 | uint16_t extensions[MAX_EXTENSIONS]; 83 | int extension_count; 84 | 85 | /* Supported curves/groups */ 86 | uint16_t curves[MAX_CURVES]; 87 | int curve_count; 88 | 89 | /* Signature algorithms */ 90 | uint16_t signature_algorithms[MAX_SIG_ALGORITHMS]; 91 | int signature_algorithm_count; 92 | 93 | /* ALPN protocols in order */ 94 | const char *alpn_protocols[MAX_ALPN_PROTOCOLS]; 95 | int alpn_protocol_count; 96 | 97 | /* GREASE configuration */ 98 | bool use_grease; 99 | uint16_t grease_cipher; 100 | uint16_t grease_extension; 101 | uint16_t grease_group; 102 | 103 | /* HTTP/2 fingerprint */ 104 | struct { 105 | uint32_t settings[MAX_HTTP2_SETTINGS][2]; /* [id, value] pairs */ 106 | int setting_count; 107 | uint32_t window_update; 108 | uint8_t priority_frames[16]; /* Stream priority configuration */ 109 | int priority_frame_count; 110 | } http2; 111 | 112 | /* JA3 fingerprint (precomputed) */ 113 | char ja3_hash[33]; /* MD5 hash as hex string */ 114 | 115 | } browser_profile_t; 116 | 117 | /* Browser profile database */ 118 | 119 | /** 120 | * Get profile by name 121 | */ 122 | const browser_profile_t* browser_profile_get(const char *name); 123 | 124 | /** 125 | * Get random profile 126 | */ 127 | const browser_profile_t* browser_profile_random(void); 128 | 129 | /** 130 | * Get profile by browser type 131 | */ 132 | const browser_profile_t* browser_profile_by_type(const char *browser_type); 133 | 134 | /** 135 | * Get user agent for specific OS from profile 136 | * Returns appropriate user agent string based on os_type 137 | * Falls back to macOS user agent if OS-specific version not available 138 | */ 139 | const char* browser_profile_get_user_agent(const browser_profile_t *profile, os_type_t os); 140 | 141 | /** 142 | * List all available profiles 143 | */ 144 | const char** browser_profile_list(int *count); 145 | 146 | /** 147 | * Generate dynamic profile based on real browser with variations 148 | */ 149 | browser_profile_t* browser_profile_generate_variant(const browser_profile_t *base); 150 | 151 | /** 152 | * Destroy a generated profile 153 | */ 154 | void browser_profile_destroy(browser_profile_t *profile); 155 | 156 | /* Predefined profiles - Chrome 127-143 (all with exact JA4 fingerprint matches) */ 157 | extern const browser_profile_t PROFILE_CHROME_127; 158 | extern const browser_profile_t PROFILE_CHROME_128; 159 | extern const browser_profile_t PROFILE_CHROME_129; 160 | extern const browser_profile_t PROFILE_CHROME_130; 161 | extern const browser_profile_t PROFILE_CHROME_131; 162 | extern const browser_profile_t PROFILE_CHROME_132; 163 | extern const browser_profile_t PROFILE_CHROME_133; 164 | extern const browser_profile_t PROFILE_CHROME_134; 165 | extern const browser_profile_t PROFILE_CHROME_135; 166 | extern const browser_profile_t PROFILE_CHROME_136; 167 | extern const browser_profile_t PROFILE_CHROME_137; 168 | extern const browser_profile_t PROFILE_CHROME_138; 169 | extern const browser_profile_t PROFILE_CHROME_139; 170 | extern const browser_profile_t PROFILE_CHROME_140; 171 | extern const browser_profile_t PROFILE_CHROME_141; 172 | extern const browser_profile_t PROFILE_CHROME_142; 173 | extern const browser_profile_t PROFILE_CHROME_143; /* Latest Chrome */ 174 | 175 | #ifdef __cplusplus 176 | } 177 | #endif 178 | 179 | #endif /* BROWSER_PROFILES_H */ 180 | -------------------------------------------------------------------------------- /benchmarks/results/windows/0.2.4/benchmark.md: -------------------------------------------------------------------------------- 1 | # httpmorph Benchmark Results 2 | 3 | **Version:** 0.2.4 | **Generated:** 2025-11-07 4 | 5 | ## System Information 6 | 7 | | Property | Value | 8 | |----------|-------| 9 | | **OS** | Windows (Windows-10-10.0.20348-SP0) | 10 | | **Processor** | AMD64 Family 23 Model 1 Stepping 2, AuthenticAMD | 11 | | **CPU Cores** | 3 | 12 | | **Python** | 3.11.9 (CPython) | 13 | 14 | ## Test Configuration 15 | 16 | - **Sequential Requests:** 25 (warmup: 5) 17 | - **Concurrent Requests:** 25 (workers: 5) 18 | 19 | ## Library Versions 20 | 21 | | Library | Version | Status | 22 | |---------|---------|--------| 23 | | **httpmorph** | `0.1.3` | Installed | 24 | | **requests** | `2.32.5` | Installed | 25 | | **httpx** | `0.28.1` | Installed | 26 | | **aiohttp** | `3.13.2` | Installed | 27 | | **urllib3** | `2.5.0` | Installed | 28 | | **urllib** | `built-in (Python 3.11.9)` | Installed | 29 | | **pycurl** | `PycURL/"7.45.7" libcurl/8.15.0-DEV (OpenSSL/3.5.2) Schannel zlib/1.3.1 brotli/1.1.0 libssh2/1.11.1_DEV nghttp2/1.67.0` | Installed | 30 | | **curl_cffi** | `0.13.0` | Installed | 31 | 32 | ## Sequential Tests (Lower is Better) 33 | 34 | Mean response time in milliseconds 35 | 36 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 37 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 38 | | **curl_cffi** | 0.89ms | 1382.07ms | 1637.57ms | 1252.71ms | 1.66ms | 3.40ms | 3.43ms | 39 | | **httpmorph** | 6.88ms | 1334.71ms | 1676.44ms | 1571.33ms | 1.69ms | 13.41ms | 5.78ms | 40 | | **httpx** | 3.35ms | 342.21ms | 441.69ms | 402.97ms | 8.17ms | 17.26ms | 8.97ms | 41 | | **pycurl** | 0.59ms | 821.61ms | 1255.96ms | 1529.15ms | 1.19ms | 8.58ms | 6.62ms | 42 | | **requests** | 6.30ms | 362.45ms | N/A | 333.30ms | 2.12ms | N/A | 1.64ms | 43 | | **urllib** | 16.74ms | 1316.49ms | N/A | 1316.87ms | 3.82ms | N/A | 23.50ms | 44 | | **urllib3** | 1.41ms | 321.69ms | N/A | 370.93ms | 0.85ms | N/A | 0.71ms | 45 | 46 | **Winners (Sequential):** 47 | - Local HTTP: **pycurl** (0.59ms) 48 | - Proxy HTTP: **urllib3** (321.69ms) 49 | - Proxy HTTP2: **httpx** (441.69ms) 50 | - Proxy HTTPs: **requests** (333.30ms) 51 | - Remote HTTP: **urllib3** (0.85ms) 52 | - Remote HTTP2: **curl_cffi** (3.40ms) 53 | - Remote HTTPs: **urllib3** (0.71ms) 54 | 55 | ## Concurrent Tests (Higher is Better) 56 | 57 | Throughput in requests per second 58 | 59 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 60 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 61 | | **curl_cffi** | 654.76 | 2.91 | 2.06 | 2.98 | 410.53 | 571.77 | 551.06 | 62 | | **httpmorph** | 936.27 | 6.04 | 3.28 | 2.15 | 823.39 | 478.66 | 573.54 | 63 | | **httpx** | 436.32 | 4.42 | 2.73 | 9.00 | 441.09 | 272.52 | 374.32 | 64 | | **pycurl** | 1698.89 | 3.51 | 2.26 | 1.80 | 1084.42 | 384.91 | 424.51 | 65 | | **requests** | 460.28 | 4.30 | N/A | 7.55 | 615.74 | N/A | 201.24 | 66 | | **urllib** | 56.43 | 3.94 | N/A | 1.88 | 790.93 | N/A | 60.85 | 67 | | **urllib3** | 1011.78 | 11.59 | N/A | 7.23 | 674.37 | N/A | 217.71 | 68 | 69 | **Winners (Concurrent):** 70 | - Local HTTP: **pycurl** (1698.89 req/s) 71 | - Proxy HTTP: **urllib3** (11.59 req/s) 72 | - Proxy HTTP2: **httpmorph** (3.28 req/s) 73 | - Proxy HTTPs: **httpx** (9.00 req/s) 74 | - Remote HTTP: **pycurl** (1084.42 req/s) 75 | - Remote HTTP2: **curl_cffi** (571.77 req/s) 76 | - Remote HTTPs: **httpmorph** (573.54 req/s) 77 | 78 | ## Async Tests (Higher is Better) 79 | 80 | Throughput in requests per second 81 | 82 | | Library | Local HTTP | Proxy HTTP | Proxy HTTP2 | Proxy HTTPs | Remote HTTP | Remote HTTP2 | Remote HTTPs | 83 | |---------|--------:|--------:|--------:|--------:|--------:|--------:|--------:| 84 | | **aiohttp** | 24.26 | 4.31 | N/A | 12.24 | 788.43 | N/A | 282.15 | 85 | | **httpmorph** | 22.74 | 66.99 | 3.66 | 3.99 | 282.35 | 205.69 | 182.08 | 86 | | **httpx** | 23.53 | 4.27 | 3.87 | 3.82 | 354.75 | 142.11 | 228.86 | 87 | 88 | **Winners (Async):** 89 | - Local HTTP: **aiohttp** (24.26 req/s) 90 | - Proxy HTTP: **httpmorph** (66.99 req/s) 91 | - Proxy HTTP2: **httpx** (3.87 req/s) 92 | - Proxy HTTPs: **aiohttp** (12.24 req/s) 93 | - Remote HTTP: **aiohttp** (788.43 req/s) 94 | - Remote HTTP2: **httpmorph** (205.69 req/s) 95 | - Remote HTTPs: **aiohttp** (282.15 req/s) 96 | 97 | ## Overall Performance Summary 98 | 99 | ### Sequential Tests: httpmorph vs requests Speedup 100 | 101 | | Test | httpmorph | requests | Speedup | 102 | |------|----------:|---------:|--------:| 103 | | Local HTTP | 6.88ms | 6.30ms | 0.92x slower | 104 | | Proxy HTTP | 1334.71ms | 362.45ms | 0.27x slower | 105 | | Proxy HTTPs | 1571.33ms | 333.30ms | 0.21x slower | 106 | | Remote HTTP | 1.69ms | 2.12ms | **1.26x** faster | 107 | | Remote HTTPs | 5.78ms | 1.64ms | 0.28x slower | 108 | 109 | ### Concurrent Tests: httpmorph vs requests Speedup 110 | 111 | | Test | httpmorph | requests | Speedup | 112 | |------|----------:|---------:|--------:| 113 | | Local HTTP | 936.27 req/s | 460.28 req/s | **2.03x** faster | 114 | | Proxy HTTP | 6.04 req/s | 4.30 req/s | **1.41x** faster | 115 | | Proxy HTTPs | 2.15 req/s | 7.55 req/s | 0.29x slower | 116 | | Remote HTTP | 823.39 req/s | 615.74 req/s | **1.34x** faster | 117 | | Remote HTTPs | 573.54 req/s | 201.24 req/s | **2.85x** faster | 118 | 119 | ### Async Tests: httpmorph vs httpx Speedup 120 | 121 | | Test | httpmorph | httpx | Speedup | 122 | |------|----------:|------:|--------:| 123 | | Local HTTP | 22.74 req/s | 23.53 req/s | 0.97x slower | 124 | | Proxy HTTP | 66.99 req/s | 4.27 req/s | **15.68x** faster | 125 | | Proxy HTTP2 | 3.66 req/s | 3.87 req/s | 0.94x slower | 126 | | Proxy HTTPs | 3.99 req/s | 3.82 req/s | **1.05x** faster | 127 | | Remote HTTP | 282.35 req/s | 354.75 req/s | 0.80x slower | 128 | | Remote HTTP2 | 205.69 req/s | 142.11 req/s | **1.45x** faster | 129 | | Remote HTTPs | 182.08 req/s | 228.86 req/s | 0.80x slower | 130 | 131 | --- 132 | *Generated by httpmorph benchmark suite* 133 | -------------------------------------------------------------------------------- /.github/workflows/_build_macos.yml: -------------------------------------------------------------------------------- 1 | name: Build macOS Wheels 2 | 3 | on: 4 | workflow_call: 5 | 6 | jobs: 7 | build-macos: 8 | name: Build macOS Wheels 9 | runs-on: macos-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v4 13 | with: 14 | submodules: recursive 15 | 16 | # Install build tools 17 | - name: Install build dependencies 18 | run: | 19 | brew install cmake ninja go 20 | 21 | # Setup Go for BoringSSL build 22 | - name: Setup Go 23 | uses: actions/setup-go@v5 24 | with: 25 | go-version: '1.21' 26 | cache: true 27 | 28 | # Cache vendor dependencies to speed up builds 29 | # Note: Cache is architecture-specific (ARM64 vs x86_64) 30 | - name: Restore vendor cache 31 | id: cache-vendor 32 | uses: actions/cache/restore@v4 33 | with: 34 | path: vendor 35 | key: vendor-macos-${{ runner.arch }}-${{ hashFiles('scripts/setup_vendors.sh', 'scripts/darwin/**/*.sh') }}-v10 36 | restore-keys: | 37 | vendor-macos-${{ runner.arch }}-${{ hashFiles('scripts/setup_vendors.sh', 'scripts/darwin/**/*.sh') }}-v10 38 | 39 | # Build vendor dependencies (always build to ensure clean state) 40 | - name: Build vendor dependencies 41 | run: | 42 | bash scripts/darwin/setup_vendors.sh 43 | 44 | # Save vendor cache for next time 45 | - name: Save vendor cache 46 | if: steps.cache-vendor.outputs.cache-hit != 'true' 47 | uses: actions/cache/save@v4 48 | with: 49 | path: vendor 50 | key: vendor-macos-${{ runner.arch }}-${{ hashFiles('scripts/setup_vendors.sh', 'scripts/darwin/**/*.sh') }}-v10 51 | 52 | # Verify vendor build 53 | - name: Verify vendor build 54 | run: | 55 | echo "=== Vendor directory contents ===" 56 | ls -la vendor/ || true 57 | echo "" 58 | echo "=== BoringSSL build ===" 59 | ls -la vendor/boringssl/build/ || true 60 | if [ -d "vendor/boringssl/build/ssl" ]; then 61 | echo " ssl:" 62 | ls -la vendor/boringssl/build/ssl/ || true 63 | fi 64 | if [ -d "vendor/boringssl/build/crypto" ]; then 65 | echo " crypto:" 66 | ls -la vendor/boringssl/build/crypto/ || true 67 | fi 68 | echo "" 69 | echo "=== nghttp2 install ===" 70 | ls -la vendor/nghttp2/install/ || true 71 | if [ -d "vendor/nghttp2/install/lib" ]; then 72 | echo " lib:" 73 | ls -la vendor/nghttp2/install/lib/ || true 74 | fi 75 | 76 | # Build wheels for all Python versions 77 | - name: Build wheels 78 | uses: pypa/cibuildwheel@v3.3 79 | env: 80 | # Skip before-build since we already built vendors 81 | CIBW_BEFORE_BUILD: "" 82 | # Build for all Python versions (universal2 for both Intel and Apple Silicon) 83 | CIBW_BUILD: cp38-macosx_* cp39-macosx_* cp310-macosx_* cp311-macosx_* cp312-macosx_* cp313-macosx_* cp314-macosx_* 84 | # Use delocate to bundle dependencies 85 | CIBW_REPAIR_WHEEL_COMMAND_MACOS: "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel} --ignore-missing-dependencies" 86 | 87 | # Setup Python versions for testing 88 | - name: Setup Python versions 89 | uses: actions/setup-python@v5 90 | with: 91 | python-version: | 92 | 3.8 93 | 3.9 94 | 3.10 95 | 3.11 96 | 3.12 97 | 3.13 98 | 3.14 99 | allow-prereleases: true 100 | 101 | # Test the built wheels 102 | - name: Test wheels 103 | run: | 104 | # Test each wheel that was built 105 | for wheel in ./wheelhouse/*.whl; do 106 | echo "========================================" 107 | echo "Testing wheel: $(basename "$wheel")" 108 | echo "========================================" 109 | 110 | # Extract Python version from wheel filename (e.g., cp39, cp310) 111 | python_tag=$(basename "$wheel" | grep -oE 'cp[0-9]+' | head -1) 112 | python_version="${python_tag:2:1}.${python_tag:3}" 113 | 114 | echo "Setting up Python $python_version..." 115 | 116 | # Use the appropriate Python version 117 | python_cmd="python${python_version}" 118 | if ! command -v "$python_cmd" &> /dev/null; then 119 | python_cmd="python3" 120 | fi 121 | 122 | # Upgrade pip to ensure we have the latest features 123 | "$python_cmd" -m pip install --upgrade pip || true 124 | 125 | # Install the wheel in a fresh environment 126 | # Use --break-system-packages if supported (pip >= 22.1) 127 | if "$python_cmd" -m pip install --help | grep -q "break-system-packages"; then 128 | "$python_cmd" -m pip install --force-reinstall --break-system-packages "$wheel" 129 | else 130 | "$python_cmd" -m pip install --force-reinstall "$wheel" 131 | fi 132 | 133 | # Run the test script 134 | echo "" 135 | echo "Running tests..." 136 | "$python_cmd" scripts/test_local_build.py 137 | 138 | # Check exit code 139 | if [ $? -eq 0 ]; then 140 | echo "" 141 | echo "[OK] Wheel test PASSED: $(basename "$wheel")" 142 | else 143 | echo "" 144 | echo "[FAIL] Wheel test FAILED: $(basename "$wheel")" 145 | exit 1 146 | fi 147 | 148 | echo "" 149 | done 150 | 151 | echo "========================================" 152 | echo "All wheel tests PASSED" 153 | echo "========================================" 154 | 155 | # Upload wheels as artifacts 156 | - name: Upload wheels 157 | uses: actions/upload-artifact@v4 158 | with: 159 | name: wheels-macos 160 | path: ./wheelhouse/*.whl 161 | if-no-files-found: error 162 | retention-days: 5 163 | # Retry on transient failures 164 | continue-on-error: false 165 | -------------------------------------------------------------------------------- /.github/workflows/_build_windows.yml: -------------------------------------------------------------------------------- 1 | name: Build Windows Wheels 2 | 3 | on: 4 | workflow_call: 5 | 6 | jobs: 7 | build-windows: 8 | name: Build Windows Wheels 9 | runs-on: windows-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v4 13 | with: 14 | submodules: recursive 15 | 16 | # Install build tools 17 | - name: Install build dependencies 18 | run: | 19 | choco install cmake ninja golang -y 20 | shell: bash 21 | 22 | # Setup Go for BoringSSL build 23 | - name: Setup Go 24 | uses: actions/setup-go@v5 25 | with: 26 | go-version: '1.21' 27 | cache: true 28 | 29 | # Cache vendor dependencies to speed up builds 30 | - name: Restore vendor cache 31 | id: cache-vendor 32 | uses: actions/cache/restore@v4 33 | with: 34 | path: vendor 35 | key: vendor-windows-${{ hashFiles('scripts/setup_vendors.sh', 'scripts/windows/**/*.sh') }}-v10 36 | restore-keys: | 37 | vendor-windows-${{ hashFiles('scripts/setup_vendors.sh', 'scripts/windows/**/*.sh') }}-v10 38 | 39 | # Build vendor dependencies (always build to ensure clean state) 40 | - name: Build vendor dependencies 41 | run: | 42 | bash scripts/windows/setup_vendors.sh 43 | shell: bash 44 | 45 | # Save vendor cache for next time 46 | - name: Save vendor cache 47 | if: steps.cache-vendor.outputs.cache-hit != 'true' 48 | uses: actions/cache/save@v4 49 | with: 50 | path: vendor 51 | key: vendor-windows-${{ hashFiles('scripts/setup_vendors.sh', 'scripts/windows/**/*.sh') }}-v10 52 | 53 | # Verify vendor build 54 | - name: Verify vendor build 55 | run: | 56 | echo "=== Vendor directory contents ===" 57 | ls -la vendor/ || true 58 | echo "" 59 | echo "=== BoringSSL build ===" 60 | ls -la vendor/boringssl/build/ || true 61 | if [ -d "vendor/boringssl/build/ssl/Release" ]; then 62 | echo " ssl/Release:" 63 | ls -la vendor/boringssl/build/ssl/Release/ || true 64 | fi 65 | if [ -d "vendor/boringssl/build/crypto/Release" ]; then 66 | echo " crypto/Release:" 67 | ls -la vendor/boringssl/build/crypto/Release/ || true 68 | fi 69 | echo "" 70 | echo "=== nghttp2 build ===" 71 | ls -la vendor/nghttp2/build/ || true 72 | if [ -d "vendor/nghttp2/build/lib/Release" ]; then 73 | echo " lib/Release:" 74 | ls -la vendor/nghttp2/build/lib/Release/ || true 75 | fi 76 | echo "" 77 | echo "=== zlib build ===" 78 | ls -la vendor/zlib/build/ || true 79 | if [ -d "vendor/zlib/build/Release" ]; then 80 | echo " Release:" 81 | ls -la vendor/zlib/build/Release/ || true 82 | fi 83 | shell: bash 84 | 85 | # Build wheels for all Python versions 86 | - name: Build wheels 87 | uses: pypa/cibuildwheel@v3.3 88 | env: 89 | # Skip before-build since we already built vendors 90 | CIBW_BEFORE_BUILD: "" 91 | # Build for all Python versions on AMD64 only 92 | # Note: ARM64 support requires vendor libraries to be built for ARM64, which needs separate build infrastructure 93 | CIBW_BUILD: cp38-win_amd64 cp39-win_amd64 cp310-win_amd64 cp311-win_amd64 cp312-win_amd64 cp313-win_amd64 cp314-win_amd64 94 | 95 | # Setup Python versions for testing 96 | - name: Setup Python versions 97 | uses: actions/setup-python@v5 98 | with: 99 | python-version: | 100 | 3.8 101 | 3.9 102 | 3.10 103 | 3.11 104 | 3.12 105 | 3.13 106 | 3.14 107 | allow-prereleases: true 108 | 109 | # Test the built wheels 110 | - name: Test wheels 111 | run: | 112 | # Test each wheel that was built 113 | for wheel in ./wheelhouse/*.whl; do 114 | echo "========================================" 115 | echo "Testing wheel: $(basename "$wheel")" 116 | echo "========================================" 117 | 118 | # Extract Python version from wheel filename (e.g., cp39, cp310) 119 | python_tag=$(basename "$wheel" | grep -oE 'cp[0-9]+' | head -1) 120 | python_version="${python_tag:2:1}.${python_tag:3}" 121 | 122 | echo "Setting up Python $python_version..." 123 | 124 | # On Windows, use py launcher with minor version only 125 | # Extract minor version (e.g., "3.10" -> "10") 126 | minor_version="${python_version#*.}" 127 | 128 | # Try py launcher first, then fall back to python 129 | if py -3.$minor_version --version &> /dev/null; then 130 | python_cmd="py -3.$minor_version" 131 | elif python --version 2>&1 | grep -q "Python $python_version"; then 132 | python_cmd="python" 133 | else 134 | echo "WARNING: Python $python_version not found, using default python" 135 | python_cmd="python" 136 | fi 137 | 138 | echo "Using Python: $($python_cmd --version)" 139 | 140 | # Install the wheel in a fresh environment 141 | $python_cmd -m pip install --force-reinstall "$wheel" 142 | 143 | # Run the test script 144 | echo "" 145 | echo "Running tests..." 146 | $python_cmd scripts/test_local_build.py 147 | 148 | # Check exit code 149 | if [ $? -eq 0 ]; then 150 | echo "" 151 | echo "[OK] Wheel test PASSED: $(basename "$wheel")" 152 | else 153 | echo "" 154 | echo "[FAIL] Wheel test FAILED: $(basename "$wheel")" 155 | exit 1 156 | fi 157 | 158 | echo "" 159 | done 160 | 161 | echo "========================================" 162 | echo "All wheel tests PASSED" 163 | echo "========================================" 164 | shell: bash 165 | 166 | # Upload wheels as artifacts 167 | - name: Upload wheels 168 | uses: actions/upload-artifact@v4 169 | with: 170 | name: wheels-windows 171 | path: ./wheelhouse/*.whl 172 | if-no-files-found: error 173 | retention-days: 5 174 | # Retry on transient failures 175 | continue-on-error: false 176 | --------------------------------------------------------------------------------