From c2fcd65eabf5e87586ad86eb6101ac220c86cdc0 Mon Sep 17 00:00:00 2001 From: icanhasmath Date: Thu, 29 Oct 2026 10:19:21 -0500 Subject: [PATCH] pip 24.0.post4 (Py3.7): vendored urllib3 1.26.20.post4; requests CVE-2024-47081, CVE-2026-25645 - Vendored urllib3 1.26.17 -> ActiveState urllib3 1.26.20.post4, vendored by replaying pip 24.0's own vendoring diff against upstream 1.26.17 onto the fork's src/urllib3 (pip._vendor imports, Brotli disabled, LICENSE.txt). The result is byte-identical to the urllib3 vendored in pip 26.0.1.post3. urllib3.__version__ stays "1.26.20" so requests' version check does not warn. - Vendored requests 2.31.0: backport CVE-2024-47081 (get_netrc_auth uses the parsed hostname, psf/requests 96ba401c, 2.32.4) and CVE-2026-25645 (extract_zipped_paths uses mkstemp, psf/requests 66d21cb07b, plus the companion e331a288f3 cert_verify change). - CVE-2024-35195 not ported: pip never mixes verify=False and verify=True on one adapter. --- NEWS.rst | 51 +++ src/pip/__init__.py | 2 +- src/pip/_vendor/requests/adapters.py | 3 +- src/pip/_vendor/requests/utils.py | 21 +- src/pip/_vendor/urllib3/_collections.py | 18 + src/pip/_vendor/urllib3/_version.py | 7 +- src/pip/_vendor/urllib3/connection.py | 70 +++- src/pip/_vendor/urllib3/connectionpool.py | 26 +- .../urllib3/contrib/securetransport.py | 3 +- src/pip/_vendor/urllib3/poolmanager.py | 25 +- src/pip/_vendor/urllib3/response.py | 330 ++++++++++++++++-- src/pip/_vendor/urllib3/util/retry.py | 4 +- src/pip/_vendor/urllib3/util/ssl_.py | 17 +- src/pip/_vendor/vendor.txt | 2 +- 14 files changed, 500 insertions(+), 79 deletions(-) diff --git a/NEWS.rst b/NEWS.rst index ab2982210d..6c0b8cdac8 100644 --- a/NEWS.rst +++ b/NEWS.rst @@ -9,6 +9,57 @@ .. towncrier release notes start +24.0.post4 (2026-10-29) +======================= + +ActiveState security release for the Python 3.7-compatible pip 24.0 line. + +Security Fixes +-------------- + +- Upgrade vendored urllib3 from 1.26.17 to 1.26.20.post4, ActiveState's + security release of urllib3 1.26.20, with pip's usual vendoring changes + (Brotli support disabled). This is the same vendored urllib3 as pip + 26.0.1.post3. It fixes: + + - **CVE-2023-45803**: request bodies were not removed when a redirect + turned the request into a GET (fixed upstream in 1.26.18). (`GHSA-g4mx-q9vg-27p4 `_) + - **CVE-2024-37891**: the ``Proxy-Authorization`` header was kept on + cross-origin redirects (fixed upstream in 1.26.19). (`GHSA-34jh-p97f-mpxf `_) + - **CVE-2025-66418**: unbounded ``Content-Encoding`` decoder chain. (`GHSA-gm62-xv2j-4w53 `_) + - **CVE-2025-66471**: the streaming API fully decompressed highly + compressed data in one operation. (`GHSA-2xpw-w6gg-jr37 `_) + - **CVE-2026-21441**: redirect bodies were decompressed while draining. (`GHSA-38jv-5279-wg99 `_) + - **CVE-2026-44431**: sensitive headers were forwarded on cross-origin + redirects. (`GHSA-qccp-gfcp-xxvc `_) + - **CVE-2026-97687**: the TLS connection to an HTTPS proxy used the + target's TLS settings. (`GHSA-8988-9cw3-xx77 `_) + - **CVE-2026-97689**: chunk-size and trailer lines were read without a + length limit. (`GHSA-vxq7-64xx-v4gw `_) + - **CVE-2025-50181**: ``retries`` at the ``PoolManager`` level did not + limit redirects. (`GHSA-pq67-6m6q-mj2v `_) + +- **CVE-2024-47081**: Backport the requests 2.32.4 fix to the vendored + requests 2.31.0. ``requests.utils.get_netrc_auth()`` took the host name by + splitting the URL's netloc at the first ``:``, so a URL with credentials + in it could make it look up (and send) ``.netrc`` credentials for a + different host. It now uses the parsed host name. pip calls it for index + and find-links URLs. (`GHSA-9hjg-9r4m-mvj7 `_) +- **CVE-2026-25645**: Backport the requests 2.33.0 fixes to the vendored + requests 2.31.0. ``requests.utils.extract_zipped_paths()`` extracted a + zip member to a predictable file name in the shared temporary directory + and reused an existing file there; it now extracts to a new ``mkstemp`` + file. ``HTTPAdapter.cert_verify()`` no longer calls it for the default CA + bundle (pip's vendored certifi already extracts its bundle to a real + file), as in upstream requests. pip did not reach the vulnerable path; + the fix is defense in depth. (`GHSA-gc5v-m9x4-r6x2 `_) +- **CVE-2024-35195** is not ported. In requests before 2.32.0, a connection + pool first used with ``verify=False`` was reused by later requests to the + same host with verification on. pip never mixes the two on one adapter: + ``--trusted-host`` hosts are mounted on their own ``InsecureHTTPAdapter`` + and every other request goes through the verifying adapter, so pip does + not reach this case. (`GHSA-9wx4-h78v-vm56 `_) + 24.0.post3 (2026-10-28) ======================= diff --git a/src/pip/__init__.py b/src/pip/__init__.py index c617699f5d..d3291659c9 100644 --- a/src/pip/__init__.py +++ b/src/pip/__init__.py @@ -1,6 +1,6 @@ from typing import List, Optional -__version__ = "24.0.post3" +__version__ = "24.0.post4" def main(args: Optional[List[str]] = None) -> int: diff --git a/src/pip/_vendor/requests/adapters.py b/src/pip/_vendor/requests/adapters.py index 10c176790b..fd86fdcfb3 100644 --- a/src/pip/_vendor/requests/adapters.py +++ b/src/pip/_vendor/requests/adapters.py @@ -45,7 +45,6 @@ from .structures import CaseInsensitiveDict from .utils import ( DEFAULT_CA_BUNDLE_PATH, - extract_zipped_paths, get_auth_from_url, get_encoding_from_headers, prepend_scheme_if_needed, @@ -255,7 +254,7 @@ def cert_verify(self, conn, url, verify, cert): cert_loc = verify if not cert_loc: - cert_loc = extract_zipped_paths(DEFAULT_CA_BUNDLE_PATH) + cert_loc = DEFAULT_CA_BUNDLE_PATH if not cert_loc or not os.path.exists(cert_loc): raise OSError( diff --git a/src/pip/_vendor/requests/utils.py b/src/pip/_vendor/requests/utils.py index 36607eda2e..c325b77e06 100644 --- a/src/pip/_vendor/requests/utils.py +++ b/src/pip/_vendor/requests/utils.py @@ -60,6 +60,7 @@ NETRC_FILES = (".netrc", "_netrc") +# Certificate is extracted by certifi when needed. DEFAULT_CA_BUNDLE_PATH = certs.where() DEFAULT_PORTS = {"http": 80, "https": 443} @@ -229,12 +230,7 @@ def get_netrc_auth(url, raise_errors=False): ri = urlparse(url) - # Strip port numbers from netloc. This weird `if...encode`` dance is - # used for Python 3.2, which doesn't support unicode literals. - splitstr = b":" - if isinstance(url, str): - splitstr = splitstr.decode("ascii") - host = ri.netloc.split(splitstr)[0] + host = ri.hostname try: _netrc = netrc(netrc_path).authenticators(host) @@ -288,12 +284,13 @@ def extract_zipped_paths(path): return path # we have a valid zip archive and a valid member of that archive - tmp = tempfile.gettempdir() - extracted_path = os.path.join(tmp, member.split("/")[-1]) - if not os.path.exists(extracted_path): - # use read + write to avoid the creating nested folders, we only want the file, avoids mkdir racing condition - with atomic_open(extracted_path) as file_handler: - file_handler.write(zip_file.read(member)) + suffix = os.path.splitext(member.split("/")[-1])[-1] + fd, extracted_path = tempfile.mkstemp(suffix=suffix) + try: + os.write(fd, zip_file.read(member)) + finally: + os.close(fd) + return extracted_path diff --git a/src/pip/_vendor/urllib3/_collections.py b/src/pip/_vendor/urllib3/_collections.py index da9857e986..bceb8451f0 100644 --- a/src/pip/_vendor/urllib3/_collections.py +++ b/src/pip/_vendor/urllib3/_collections.py @@ -268,6 +268,24 @@ def getlist(self, key, default=__marker): else: return vals[1:] + def _prepare_for_method_change(self): + """ + Remove content-specific header fields before changing the request + method to GET or HEAD according to RFC 9110, Section 15.4. + """ + content_specific_headers = [ + "Content-Encoding", + "Content-Language", + "Content-Location", + "Content-Type", + "Content-Length", + "Digest", + "Last-Modified", + ] + for header in content_specific_headers: + self.discard(header) + return self + # Backwards compatibility for httplib getheaders = getlist getallmatchingheaders = getlist diff --git a/src/pip/_vendor/urllib3/_version.py b/src/pip/_vendor/urllib3/_version.py index cad75fb5df..2dbf93a6c8 100644 --- a/src/pip/_vendor/urllib3/_version.py +++ b/src/pip/_vendor/urllib3/_version.py @@ -1,2 +1,7 @@ # This file is protected via CODEOWNERS -__version__ = "1.26.17" +# ActiveState security release. The package metadata carries the post-release +# (__package_version__, which setup.py reads), while __version__ stays +# "1.26.20": requests unpacks urllib3.__version__ as major.minor.patch and +# emits RequestsDependencyWarning for anything longer. +__version__ = "1.26.20" +__package_version__ = "1.26.20.post4" diff --git a/src/pip/_vendor/urllib3/connection.py b/src/pip/_vendor/urllib3/connection.py index 54b96b1915..080ce0cdfb 100644 --- a/src/pip/_vendor/urllib3/connection.py +++ b/src/pip/_vendor/urllib3/connection.py @@ -68,7 +68,7 @@ class BrokenPipeError(Exception): # When it comes time to update this value as a part of regular maintenance # (ie test_recent_date is failing) update it to ~6 months before the current date. -RECENT_DATE = datetime.date(2022, 1, 1) +RECENT_DATE = datetime.date(2024, 1, 1) _CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") @@ -364,6 +364,15 @@ def connect(self): hostname = self.host tls_in_tls = False + if self._is_forwarding_through_https_proxy(): + # The TLS connection being made is the one to the HTTPS proxy, + # which forwards requests without a CONNECT tunnel. It must use + # the proxy's TLS policy: never the target's SNI, hostname or + # fingerprint assertions, client certificate, or a cert_reqs + # that would override proxy_ssl_context's verify_mode. + self.sock, self.is_verified = self._wrap_proxy_socket(self.host, conn) + return + if self._is_using_tunnel(): if self.tls_in_tls_required: self.sock = conn = self._connect_tls_proxy(hostname, conn) @@ -437,7 +446,7 @@ def connect(self): and self.ssl_version is None and hasattr(self.sock, "version") and self.sock.version() in {"TLSv1", "TLSv1.1"} - ): + ): # Defensive: warnings.warn( "Negotiating TLSv1/TLSv1.1 by default is deprecated " "and will be disabled in urllib3 v2.0.0. Connecting to " @@ -476,28 +485,60 @@ def connect(self): or self.assert_fingerprint is not None ) + def _is_forwarding_through_https_proxy(self): + return ( + self.proxy is not None + and self.proxy.scheme == "https" + and not self._is_using_tunnel() + ) + def _connect_tls_proxy(self, hostname, conn): """ - Establish a TLS connection to the proxy using the provided SSL context. + Establish a TLS connection to the proxy using proxy-specific policy. + """ + socket, verified = self._wrap_proxy_socket(hostname, conn) + if verified is not None: + self.proxy_is_verified = verified + return socket + + def _wrap_proxy_socket(self, hostname, conn): + """ + Wrap ``conn`` in TLS to the proxy at ``hostname``. + + Returns the socket and whether the proxy's certificate was verified, + or None for a user-provided proxy context (left for the context to + report, as before). """ proxy_config = self.proxy_config ssl_context = proxy_config.ssl_context if ssl_context: # If the user provided a proxy context, we assume CA and client - # certificates have already been set - return ssl_wrap_socket( + # certificates have already been set. Its verification policy is + # used as-is. + socket = ssl_wrap_socket( sock=conn, server_hostname=hostname, ssl_context=ssl_context, ) - - ssl_context = create_proxy_ssl_context( - self.ssl_version, - self.cert_reqs, - self.ca_certs, - self.ca_cert_dir, - self.ca_cert_data, - ) + if self._is_forwarding_through_https_proxy(): + return socket, ssl_context.verify_mode == ssl.CERT_REQUIRED + return socket, None + + if self._is_forwarding_through_https_proxy() and self.ssl_context is not None: + # Forwarding with no proxy_ssl_context: for backwards + # compatibility the connection's ssl_context, with the pool's + # certificate policy, is used for the proxy handshake, as it was + # before proxy and target TLS settings were separated. + ssl_context = self.ssl_context + ssl_context.verify_mode = resolve_cert_reqs(self.cert_reqs) + else: + ssl_context = create_proxy_ssl_context( + self.ssl_version, + self.cert_reqs, + self.ca_certs, + self.ca_cert_dir, + self.ca_cert_data, + ) # If no cert was provided, use only the default options for server # certificate validation @@ -529,8 +570,7 @@ def _connect_tls_proxy(self, hostname, conn): ) _match_hostname(cert, hostname) - self.proxy_is_verified = ssl_context.verify_mode == ssl.CERT_REQUIRED - return socket + return socket, ssl_context.verify_mode == ssl.CERT_REQUIRED def _match_hostname(cert, asserted_hostname): diff --git a/src/pip/_vendor/urllib3/connectionpool.py b/src/pip/_vendor/urllib3/connectionpool.py index 96844d9337..c8ad09de5e 100644 --- a/src/pip/_vendor/urllib3/connectionpool.py +++ b/src/pip/_vendor/urllib3/connectionpool.py @@ -9,6 +9,7 @@ from socket import error as SocketError from socket import timeout as SocketTimeout +from ._collections import HTTPHeaderDict from .connection import ( BaseSSLError, BrokenPipeError, @@ -422,12 +423,13 @@ def _make_request( pass except IOError as e: # Python 2 and macOS/Linux - # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE is needed on macOS + # EPIPE and ESHUTDOWN are BrokenPipeError on Python 2, and EPROTOTYPE/ECONNRESET are needed on macOS # https://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ if e.errno not in { errno.EPIPE, errno.ESHUTDOWN, errno.EPROTOTYPE, + errno.ECONNRESET, }: raise @@ -767,7 +769,9 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): # so we try to cover our bases here! message = " ".join(re.split("[^a-z]", str(ssl_error).lower())) return ( - "wrong version number" in message or "unknown protocol" in message + "wrong version number" in message + or "unknown protocol" in message + or "record layer failure" in message ) # Try to detect a common user error with proxies which is to @@ -843,7 +847,25 @@ def _is_ssl_error_message_from_http_proxy(ssl_error): redirect_location = redirect and response.get_redirect_location() if redirect_location: if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + body = None + headers = HTTPHeaderDict(headers)._prepare_for_method_change() + + # Strip headers marked as unsafe to forward to the redirected location. + # Check remove_headers_on_redirect to avoid a potential network call within + # self.is_same_host() which may use socket.gethostbyname() in the future. + if ( + headers + and retries.remove_headers_on_redirect + and not self.is_same_host(redirect_location) + ): + new_headers = headers.copy() + for header in headers: + if header.lower() in retries.remove_headers_on_redirect: + new_headers.pop(header, None) + headers = new_headers try: retries = retries.increment(method, url, response=response, _pool=self) diff --git a/src/pip/_vendor/urllib3/contrib/securetransport.py b/src/pip/_vendor/urllib3/contrib/securetransport.py index 4a06bc69d5..722ee4e124 100644 --- a/src/pip/_vendor/urllib3/contrib/securetransport.py +++ b/src/pip/_vendor/urllib3/contrib/securetransport.py @@ -64,9 +64,8 @@ import threading import weakref -from pip._vendor import six - from .. import util +from ..packages import six from ..util.ssl_ import PROTOCOL_TLS_CLIENT from ._securetransport.bindings import CoreFoundation, Security, SecurityConst from ._securetransport.low_level import ( diff --git a/src/pip/_vendor/urllib3/poolmanager.py b/src/pip/_vendor/urllib3/poolmanager.py index 14b10daf3a..a8de7c6dfc 100644 --- a/src/pip/_vendor/urllib3/poolmanager.py +++ b/src/pip/_vendor/urllib3/poolmanager.py @@ -4,7 +4,7 @@ import functools import logging -from ._collections import RecentlyUsedContainer +from ._collections import HTTPHeaderDict, RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, port_by_scheme from .exceptions import ( LocationValueError, @@ -170,6 +170,22 @@ class PoolManager(RequestMethods): def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) + if "retries" in connection_pool_kw: + retries = connection_pool_kw["retries"] + if not isinstance(retries, Retry): + # When Retry is initialized, raise_on_redirect is based + # on a redirect boolean value. + # But requests made via a pool manager always set + # redirect to False, and raise_on_redirect always ends + # up being False consequently. + # Here we fix the issue by setting raise_on_redirect to + # a value needed by the pool manager without considering + # the redirect boolean. + raise_on_redirect = retries is not False + retries = Retry.from_int(retries, redirect=False) + retries.raise_on_redirect = raise_on_redirect + connection_pool_kw = connection_pool_kw.copy() + connection_pool_kw["retries"] = retries self.connection_pool_kw = connection_pool_kw self.pools = RecentlyUsedContainer(num_pools) @@ -382,11 +398,14 @@ def urlopen(self, method, url, redirect=True, **kw): # Support relative URLs for redirecting. redirect_location = urljoin(url, redirect_location) - # RFC 7231, Section 6.4.4 if response.status == 303: + # Change the method according to RFC 9110, Section 15.4.4. method = "GET" + # And lose the body not to transfer anything sensitive. + kw["body"] = None + kw["headers"] = HTTPHeaderDict(kw["headers"])._prepare_for_method_change() - retries = kw.get("retries") + retries = kw.get("retries", response.retries) if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) diff --git a/src/pip/_vendor/urllib3/response.py b/src/pip/_vendor/urllib3/response.py index 8909f8454e..6a9271c0a7 100644 --- a/src/pip/_vendor/urllib3/response.py +++ b/src/pip/_vendor/urllib3/response.py @@ -17,6 +17,7 @@ from .exceptions import ( BodyNotHttplibCompatible, DecodeError, + DependencyWarning, HTTPError, IncompleteRead, InvalidChunkLength, @@ -31,37 +32,87 @@ log = logging.getLogger(__name__) +# Maximum length of a chunk-size line (and the trailing CRLF line) when reading a +# chunked-transfer-encoded response. Matches ``http.client._MAXLINE`` so urllib3's +# streaming path bounds these reads exactly like the stdlib ``.read()`` path does, +# preventing a malicious server from forcing unbounded buffering (memory exhaustion) +# via an unterminated chunk-size line. +_MAX_CHUNK_LINE_LENGTH = 2 ** 16 + +# Read in 64 KiB chunks +_READ_CHUNK_SIZE = 2 ** 16 + class DeflateDecoder(object): def __init__(self): self._first_try = True - self._data = b"" + self._first_try_data = b"" + self._unfed_data = b"" self._obj = zlib.decompressobj() def __getattr__(self, name): return getattr(self._obj, name) - def decompress(self, data): - if not data: + def decompress(self, data, max_length=-1): + data = self._unfed_data + data + self._unfed_data = b"" + # Data received after EOF cannot produce more output from this + # Deflate stream. + if self._stream_finished(): + return b"" + if not data and not self._obj.unconsumed_tail: return data + original_max_length = max_length + if original_max_length < 0: + max_length = 0 + elif original_max_length == 0: + # We should not pass 0 to the zlib decompressor because 0 is + # the default value that will make zlib decompress without a + # length limit. + # Data should be stored for subsequent calls. + self._unfed_data = data + return b"" + # Subsequent calls always reuse `self._obj`. zlib requires + # passing the unconsumed tail if decompression is to continue. + # max_length is positional: Python 2 does not accept it as a keyword. if not self._first_try: - return self._obj.decompress(data) + return self._obj.decompress(self._obj.unconsumed_tail + data, max_length) - self._data += data + # First call tries with RFC 1950 ZLIB format. + self._first_try_data += data try: - decompressed = self._obj.decompress(data) + decompressed = self._obj.decompress(data, max_length) if decompressed: self._first_try = False - self._data = None + self._first_try_data = b"" return decompressed + # On failure, it falls back to RFC 1951 DEFLATE format. except zlib.error: self._first_try = False self._obj = zlib.decompressobj(-zlib.MAX_WBITS) try: - return self.decompress(self._data) + return self.decompress( + self._first_try_data, max_length=original_max_length + ) finally: - self._data = None + self._first_try_data = b"" + + def _stream_finished(self): + eof = getattr(self._obj, "eof", None) + if eof is not None: + return eof + # Python 2's decompressobj has no ``eof``. Input left over after + # the end of the stream is kept in ``unused_data``. + return bool(self._obj.unused_data) + + @property + def has_unconsumed_tail(self): + return bool(self._unfed_data) or ( + bool(self._obj.unconsumed_tail) + and not self._first_try + and not self._stream_finished() + ) class GzipDecoderState(object): @@ -75,30 +126,80 @@ class GzipDecoder(object): def __init__(self): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) self._state = GzipDecoderState.FIRST_MEMBER + self._unconsumed_tail = b"" def __getattr__(self, name): return getattr(self._obj, name) - def decompress(self, data): + def decompress(self, data, max_length=-1): ret = bytearray() - if self._state == GzipDecoderState.SWALLOW_DATA or not data: + if self._state == GzipDecoderState.SWALLOW_DATA: + return bytes(ret) + + if max_length == 0: + # We should not pass 0 to the zlib decompressor because 0 is + # the default value that will make zlib decompress without a + # length limit. + # Data should be stored for subsequent calls. + self._unconsumed_tail += data + return b"" + + # zlib requires passing the unconsumed tail to the subsequent + # call if decompression is to continue. + data = self._unconsumed_tail + data + if not data: return bytes(ret) + while True: + # When the end of a gzip member is reached, a new decompressor + # must be created for unused (possibly future) data. This is + # checked before decompressing because a bounded read can stop + # right at the end of a member, leaving the next member as the + # tail for the following call. + if self._member_finished(): + self._state = GzipDecoderState.OTHER_MEMBERS + self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) try: - ret += self._obj.decompress(data) + # max_length is positional: Python 2 does not accept it + # as a keyword. + ret += self._obj.decompress(data, max(max_length - len(ret), 0)) except zlib.error: previous_state = self._state # Ignore data after the first error self._state = GzipDecoderState.SWALLOW_DATA + self._unconsumed_tail = b"" if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients return bytes(ret) raise - data = self._obj.unused_data + + self._unconsumed_tail = data = ( + self._obj.unconsumed_tail or self._obj.unused_data + ) + if max_length > 0 and len(ret) >= max_length: + break + if not data: return bytes(ret) - self._state = GzipDecoderState.OTHER_MEMBERS - self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) + + return bytes(ret) + + def _member_finished(self): + eof = getattr(self._obj, "eof", None) + if eof is not None: + return eof + # Python 2's decompressobj has no ``eof``. Input left over after + # the end of a member is kept in ``unused_data``. + return bool(self._obj.unused_data) + + @property + def has_unconsumed_tail(self): + return bool(self._unconsumed_tail) + + def flush(self): + if self._state == GzipDecoderState.SWALLOW_DATA: + return b"" + return self._obj.flush() if brotli is not None: @@ -110,9 +211,45 @@ class BrotliDecoder(object): def __init__(self): self._obj = brotli.Decompressor() if hasattr(self._obj, "decompress"): - self.decompress = self._obj.decompress + self._decompress = self._obj.decompress else: - self.decompress = self._obj.process + self._decompress = self._obj.process + self._output_limited = False + + # Requires Brotli >= 1.2.0 for `output_buffer_limit`. + def decompress(self, data, max_length=-1): + try: + if max_length > 0: + out = self._decompress(data, output_buffer_limit=max_length) + # Brotli can hold back output while still reporting that + # it accepts more input + # (https://github.com/google/brotli/issues/1396), so a + # call that filled the limit may have more to give. + self._output_limited = len(out) >= max_length + return out + else: + self._output_limited = False + return self._decompress(data) + except TypeError: + # Fallback for Brotli/brotlicffi/brotlipy versions without + # the `output_buffer_limit` parameter. + warnings.warn( + "Brotli >= 1.2.0 is required to prevent decompression bombs.", + DependencyWarning, + ) + return self._decompress(data) + + @property + def has_unconsumed_tail(self): + try: + if not self._obj.can_accept_more_data(): + return True + except AttributeError: + return False + except brotli.error: + # The decoder failed earlier; it holds nothing decodable. + return False + return self._output_limited def flush(self): if hasattr(self._obj, "flush"): @@ -129,16 +266,51 @@ class MultiDecoder(object): they were applied. """ + # Maximum allowed number of chained HTTP encodings in the + # Content-Encoding header. + max_decode_links = 5 + def __init__(self, modes): - self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")] + encodings = [m.strip() for m in modes.split(",")] + if len(encodings) > self.max_decode_links: + raise DecodeError( + "Too many content encodings in the chain: %d > %d" + % (len(encodings), self.max_decode_links) + ) + self._decoders = [_get_decoder(e) for e in encodings] def flush(self): return self._decoders[0].flush() - def decompress(self, data): - for d in reversed(self._decoders): - data = d.decompress(data) - return data + def decompress(self, data, max_length=-1): + if max_length <= 0: + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + ret = bytearray() + # Every while loop iteration goes through all decoders once. + # It exits when enough data is read or no more data can be read. + # It is possible that the while loop iteration does not produce + # any data because we retrieve up to `max_length` from every + # decoder, and the amount of bytes may be insufficient for the + # next decoder to produce enough/any output. + while True: + any_data = False + for d in reversed(self._decoders): + data = d.decompress(data, max_length=max_length - len(ret)) + if data: + any_data = True + # We should not break when no data is returned because + # next decoders may produce data even with empty input. + ret += data + if not any_data or len(ret) >= max_length: + return bytes(ret) + data = b"" + + @property + def has_unconsumed_tail(self): + return any(d.has_unconsumed_tail for d in self._decoders) def _get_decoder(mode): @@ -222,6 +394,8 @@ def __init__( self.reason = reason self.strict = strict self.decode_content = decode_content + self._has_decoded_content = False + self._decode_failed = False self.retries = retries self.enforce_content_length = enforce_content_length self.auto_close = auto_close @@ -285,10 +459,18 @@ def drain_conn(self): Unread data in the HTTPResponse connection blocks the connection from being released back to the pool. """ + # The body is discarded without decompression, so drop the decoder + # and any data pending in it first. + self._decoder = None try: - self.read() + # Discard the rest of the body without decoding it, even if + # decoding has already started, 64 KiB at a time. + while self.read(_READ_CHUNK_SIZE, decode_content=False): + pass except (HTTPError, SocketError, BaseSSLError, HTTPException): pass + # read() set up a fresh decoder; it is not needed either. + self._decoder = None @property def data(self): @@ -389,17 +571,24 @@ def _init_decoder(self): if brotli is not None: DECODER_ERROR_CLASSES += (brotli.error,) - def _decode(self, data, decode_content, flush_decoder): + def _decode(self, data, decode_content, flush_decoder, max_length=None): """ Decode the data passed in and potentially flush the decoder. """ if not decode_content: return data + if max_length is None or flush_decoder: + max_length = -1 + try: if self._decoder: - data = self._decoder.decompress(data) + data = self._decoder.decompress(data, max_length=max_length) + self._has_decoded_content = True except self.DECODER_ERROR_CLASSES as e: + # Whatever the decoder still holds can no longer be decoded, so a + # raw read afterwards loses nothing that could have been returned. + self._decode_failed = True content_encoding = self.headers.get("content-encoding", "").lower() raise DecodeError( "Received response with content-encoding: %s, but " @@ -554,6 +743,33 @@ def read(self, amt=None, decode_content=None, cache_content=False): if self._fp is None: return + if ( + not decode_content + and self._has_decoded_content + and not self._decode_failed + and self._decoder + and self._decoder.has_unconsumed_tail + ): + # Input that a decoding read already handed to the decoder can no + # longer be returned raw, so switching modes here would silently + # lose it. + raise RuntimeError( + "Calling read(decode_content=False) is not supported while " + "data from read(decode_content=True) is still being decoded." + ) + + if ( + amt is not None + and decode_content + and self._decoder + and self._decoder.has_unconsumed_tail + ): + # Earlier input is still waiting in the decoder. Hand out up to + # ``amt`` bytes of it before reading more from the connection. + data = self._decode(b"", decode_content, False, max_length=amt) + if data: + return data + flush_decoder = False fp_closed = getattr(self._fp, "closed", False) @@ -591,14 +807,20 @@ def read(self, amt=None, decode_content=None, cache_content=False): if self.length_remaining is not None: self.length_remaining -= len(data) - data = self._decode(data, decode_content, flush_decoder) + data = self._decode(data, decode_content, flush_decoder, max_length=amt) if cache_content: self._body = data + elif flush_decoder and decode_content and self._decoder: + # The connection is exhausted, but the decoder may still hold + # output from input it has already consumed (a bounded read can + # stop with all input fed in and output still pending). Flush it + # instead of dropping it. + data = self._decode(b"", decode_content, flush_decoder) return data - def stream(self, amt=2 ** 16, decode_content=None): + def stream(self, amt=_READ_CHUNK_SIZE, decode_content=None): """ A generator wrapper for the read() method. A call will block until ``amt`` bytes have been read from the connection or until the @@ -614,16 +836,31 @@ def stream(self, amt=2 ** 16, decode_content=None): If True, will attempt to decode the body based on the 'content-encoding' header. """ + if amt == 0: + return + if self.chunked and self.supports_chunked_reads(): for line in self.read_chunked(amt, decode_content=decode_content): yield line else: - while not is_fp_closed(self._fp): + while not is_fp_closed(self._fp) or ( + self._decoder and self._decoder.has_unconsumed_tail + ): data = self.read(amt=amt, decode_content=decode_content) if data: yield data + # httplib closes the file as soon as Content-Length is reached, + # which can end the loop above while the decoder still holds output + # from input it has already consumed. Flush it; the file is done. + if decode_content is None: + decode_content = self.decode_content + if decode_content and self._decoder: + data = self._decode(b"", decode_content, flush_decoder=True) + if data: + yield data + @classmethod def from_httplib(ResponseCls, r, **response_kw): """ @@ -749,7 +986,12 @@ def _update_chunk_length(self): # we'll try to read it from socket. if self.chunk_left is not None: return - line = self._fp.fp.readline() + line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1) + if len(line) > _MAX_CHUNK_LINE_LENGTH: + self.close() + raise ProtocolError( + "Response chunk size line exceeded maximum allowed length" + ) line = line.split(b";", 1)[0] try: self.chunk_left = int(line, 16) @@ -818,13 +1060,27 @@ def read_chunked(self, amt=None, decode_content=None): if self._fp.fp is None: return + if amt == 0: + return + elif amt and amt < 0: + # Negative numbers and `None` should be treated the same, + # but httplib handles only `None` correctly. + amt = None + while True: - self._update_chunk_length() - if self.chunk_left == 0: - break - chunk = self._handle_chunk(amt) + # First, check if any data is left in the decoder's buffer. + if self._decoder and self._decoder.has_unconsumed_tail: + chunk = b"" + else: + self._update_chunk_length() + if self.chunk_left == 0: + break + chunk = self._handle_chunk(amt) decoded = self._decode( - chunk, decode_content=decode_content, flush_decoder=False + chunk, + decode_content=decode_content, + flush_decoder=False, + max_length=amt, ) if decoded: yield decoded @@ -839,7 +1095,11 @@ def read_chunked(self, amt=None, decode_content=None): # Chunk content ends with \r\n: discard it. while True: - line = self._fp.fp.readline() + line = self._fp.fp.readline(_MAX_CHUNK_LINE_LENGTH + 1) + if len(line) > _MAX_CHUNK_LINE_LENGTH: + raise ProtocolError( + "Response chunk trailer line exceeded maximum allowed length" + ) if not line: # Some sites may not end with '\r\n'. break diff --git a/src/pip/_vendor/urllib3/util/retry.py b/src/pip/_vendor/urllib3/util/retry.py index 60ef6c4f3f..9a1e90d0b2 100644 --- a/src/pip/_vendor/urllib3/util/retry.py +++ b/src/pip/_vendor/urllib3/util/retry.py @@ -235,7 +235,9 @@ class Retry(object): RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) #: Default headers to be used for ``remove_headers_on_redirect`` - DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset(["Cookie", "Authorization"]) + DEFAULT_REMOVE_HEADERS_ON_REDIRECT = frozenset( + ["Cookie", "Authorization", "Proxy-Authorization"] + ) #: Maximum backoff time. DEFAULT_BACKOFF_MAX = 120 diff --git a/src/pip/_vendor/urllib3/util/ssl_.py b/src/pip/_vendor/urllib3/util/ssl_.py index 2b45d391d4..0a6a0e06a0 100644 --- a/src/pip/_vendor/urllib3/util/ssl_.py +++ b/src/pip/_vendor/urllib3/util/ssl_.py @@ -1,11 +1,11 @@ from __future__ import absolute_import +import hashlib import hmac import os import sys import warnings from binascii import hexlify, unhexlify -from hashlib import md5, sha1, sha256 from ..exceptions import ( InsecurePlatformWarning, @@ -24,7 +24,10 @@ ALPN_PROTOCOLS = ["http/1.1"] # Maps the length of a digest to a possible hash function producing this digest -HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} +HASHFUNC_MAP = { + length: getattr(hashlib, algorithm, None) + for length, algorithm in ((32, "md5"), (40, "sha1"), (64, "sha256")) +} def _const_compare_digest_backport(a, b): @@ -191,9 +194,15 @@ def assert_fingerprint(cert, fingerprint): fingerprint = fingerprint.replace(":", "").lower() digest_length = len(fingerprint) - hashfunc = HASHFUNC_MAP.get(digest_length) - if not hashfunc: + if digest_length not in HASHFUNC_MAP: raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) + hashfunc = HASHFUNC_MAP.get(digest_length) + if hashfunc is None: + raise SSLError( + "Hash function implementation unavailable for fingerprint length: {0}".format( + digest_length + ) + ) # We need encode() here for py32; works on py2 and p33. fingerprint_bytes = unhexlify(fingerprint.encode()) diff --git a/src/pip/_vendor/vendor.txt b/src/pip/_vendor/vendor.txt index 348d7bc217..6aa48a3ad4 100644 --- a/src/pip/_vendor/vendor.txt +++ b/src/pip/_vendor/vendor.txt @@ -11,7 +11,7 @@ requests==2.31.0 certifi==2026.10.23 chardet==5.1.0 idna==3.4 - urllib3==1.26.17 + urllib3==1.26.20.post4 rich==13.4.2 pygments==2.15.1 typing_extensions==4.7.1