[pull] master from aio-libs:master

CHANGES/13681.bugfix.rst

@@ -0,0 +1 @@
+Fixed idle connections not being closed if no request was received -- by :user:`Dreamsorcerer`.

CHANGES/13829.deprecation.rst

@@ -0,0 +1,5 @@
+Deprecated the ``ssl`` parameter of :class:`~aiohttp.TCPConnector`, scheduled for
+removal in 5.0 -- by :user:`Dreamsorcerer`.
+
+Pass ``ssl`` to :class:`~aiohttp.ClientSession` for a session-wide default, or to
+:meth:`~aiohttp.ClientSession.get` and the other request methods per request.

CHANGES/13830.bugfix.rst

@@ -0,0 +1 @@
+Fixed ``CookieJar.filter_cookies()`` sending shared cookies (cookies without a ``Domain`` attribute) marked ``Secure`` over unencrypted connections -- by :user:`Dreamsorcerer`.

CHANGES/13833.bugfix.rst

@@ -0,0 +1 @@
+Fixed per-request cookies (the ``cookies`` argument of a request method) marked ``Secure`` not being sent to origins listed in ``CookieJar``'s ``treat_as_secure_origin`` -- by :user:`Dreamsorcerer`.

aiohttp/abc.py

@@ -164,6 +164,11 @@ def unsafe(self) -> bool:
     def quote_cookie(self) -> bool:
         """Return True if cookies should be quoted."""
 
+    @property
+    def treat_as_secure_origin(self) -> frozenset[URL]:
+        """Return origins considered secure even over cleartext connections."""
+        return frozenset()
+
     @property
     @abstractmethod
     def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]:

aiohttp/client.py

@@ -650,6 +650,7 @@ async def _request(
                         tmp_cookie_jar = CookieJar(
                             unsafe=self._cookie_jar.unsafe,
                             quote_cookie=self._cookie_jar.quote_cookie,
+                            treat_as_secure_origin=self._cookie_jar.treat_as_secure_origin,
                         )
                         tmp_cookie_jar.update_cookies(cookies)
                         req_cookies = tmp_cookie_jar.filter_cookies(url)

aiohttp/connector.py

@@ -954,11 +954,11 @@ def _make_ssl_context(verified: bool) -> SSLContext:
 class TCPConnector(BaseConnector):
     """TCP connector.
 
-    verify_ssl - Set to True to check ssl certifications.
-    fingerprint - Pass the binary sha256
-        digest of the expected certificate in DER format to verify
-        that the certificate the server presents matches. See also
-        https://en.wikipedia.org/wiki/HTTP_Public_Key_Pinning
+    ssl - DEPRECATED. Will be removed in aiohttp 5.0.
+        SSL validation mode: ``True`` for the default checks, ``False`` to
+        skip certificate validation, a Fingerprint for certificate pinning
+        or an ssl.SSLContext for custom validation. Pass ``ssl`` to
+        ClientSession or to the individual request instead.
     resolver - Enable DNS lookups and use this
         resolver
     use_dns_cache - Use memory cache for DNS lookups.
@@ -999,7 +999,7 @@ def __init__(
         ttl_dns_cache: int | None = 10,
         dns_cache_max_size: int = 1000,
         family: socket.AddressFamily = socket.AddressFamily.AF_UNSPEC,
-        ssl: bool | Fingerprint | SSLContext = True,
+        ssl: bool | Fingerprint | SSLContext | _SENTINEL = sentinel,
         local_addr: tuple[str, int] | None = None,
         resolver: AbstractResolver | None = None,
         keepalive_timeout: None | float | _SENTINEL = sentinel,
@@ -1022,12 +1022,23 @@ def __init__(
             timeout_ceil_threshold=timeout_ceil_threshold,
         )
 
-        if not isinstance(ssl, SSL_ALLOWED_TYPES):
-            raise TypeError(
-                "ssl should be SSLContext, Fingerprint, or bool, "
-                f"got {ssl!r} instead."
+        self._ssl: bool | Fingerprint | SSLContext
+        if ssl is sentinel:
+            self._ssl = True
+        else:
+            if not isinstance(ssl, SSL_ALLOWED_TYPES):
+                raise TypeError(
+                    "ssl should be SSLContext, Fingerprint, or bool, "
+                    f"got {ssl!r} instead."
+                )
+            warnings.warn(
+                "The ssl parameter is deprecated since 4.0 and scheduled for "
+                "removal in 5.0, pass ssl to ClientSession() or to the "
+                "individual request instead",
+                DeprecationWarning,
+                stacklevel=2,
             )
-        self._ssl = ssl
+            self._ssl = ssl
 
         self._resolver: AbstractResolver
         if resolver is None:

aiohttp/cookiejar.py

@@ -122,6 +122,11 @@ def unsafe(self) -> bool:
     def quote_cookie(self) -> bool:
         return self._quote_cookie
 
+    @property
+    def treat_as_secure_origin(self) -> frozenset[URL]:
+        """Return origins considered secure even over cleartext connections."""
+        return self._treat_as_secure_origin
+
     @property
     def cookies(self) -> MappingProxyType[tuple[str, str], SimpleCookie]:
         """Return the cookies stored in this jar."""
@@ -455,6 +460,9 @@ def filter_cookies(self, request_url: URL) -> "BaseCookie[str]":
         # Send shared cookie
         key = ("", "")
         for c in self._cookies[key].values():
+            if is_not_secure and c["secure"]:
+                continue
+
             # Check cache first
             if c.key in self._morsel_cache[key]:
                 filtered[c.key] = self._morsel_cache[key][c.key]

aiohttp/web_protocol.py

@@ -140,8 +140,7 @@ class RequestHandler(BaseProtocol, Generic[_Request]):
     status line, bad headers or incomplete payload. If any error occurs,
     connection gets closed.
 
-    keepalive_timeout -- number of seconds before closing
-                         keep-alive connection
+    keepalive_timeout -- number of seconds before closing an idle connection.
 
     tcp_keepalive -- TCP keep-alive is on, default is on
 
@@ -240,7 +239,9 @@ def __init__(
         super().__init__(loop, parser)
 
         self._request_count = 0
-        self._keepalive = False
+        # True from the start so the deadline armed in connection_made()
+        # closes connections that never deliver a complete first request.
+        self._keepalive = True
         self._current_request: _Request | None = None
         self._manager: Server[_Request] | None = manager
         self._request_handler: _RequestHandler[_Request] | None = (
@@ -401,6 +402,13 @@ def connection_made(self, transport: asyncio.BaseTransport) -> None:
         self._manager.connection_made(self, real_transport)
 
         loop = self._loop
+        # Need to enable keepalive timeout at start of connection, as there's no other
+        # protection against a dead connection that doesn't send a request at all.
+        if self._keepalive_timeout > 0:
+            close_time = loop.time() + self._keepalive_timeout
+            self._next_keepalive_close_time = close_time
+            self._keepalive_handle = loop.call_at(close_time, self._process_keepalive)
+
         if sys.version_info >= (3, 14):
             if isinstance(loop, BaseEventLoop):
                 task = asyncio.create_task(self.start(), eager_start=True)

docs/client_advanced.rst

@@ -618,7 +618,7 @@ checks can be relaxed by setting *ssl* to ``False``::
 If you need to setup custom ssl parameters (use own certification
 files for example) you can create a :class:`ssl.SSLContext` instance and
 pass it into the :meth:`ClientSession.request` methods or set it for the
-entire session with ``ClientSession(connector=TCPConnector(ssl=ssl_context))``.
+entire session with ``ClientSession(ssl=ssl_context)``.
 
 There are explicit errors when ssl verification fails
 
@@ -660,7 +660,7 @@ installed or Python is unable to find them, resulting in a error like
 One way to work around this problem is to use the `certifi` package::
 
   ssl_context = ssl.create_default_context(cafile=certifi.where())
-  async with ClientSession(connector=TCPConnector(ssl=ssl_context)) as sess:
+  async with ClientSession(ssl=ssl_context) as sess:
       ...
 
 Example: Use self-signed certificate
@@ -706,9 +706,9 @@ DER with e.g::
    Tip: to convert from a hexadecimal digest to a binary byte-string,
    you can use :func:`binascii.unhexlify`.
 
-   *ssl* parameter could be passed
-   to :class:`TCPConnector` as default, the value from
-   :meth:`ClientSession.get` and others override default.
+   *ssl* parameter could be passed to :class:`ClientSession` as the
+   session-wide default, the value from :meth:`ClientSession.get` and
+   others override it.
 
 .. _aiohttp-client-proxy-support:
 

docs/client_reference.rst

@@ -1231,8 +1231,8 @@ is controlled by *force_close* constructor's parameter).
    Constructor accepts all parameters suitable for
    :class:`BaseConnector` plus several TCP-specific ones:
 
-      :param ssl: SSL validation mode. ``True`` for default SSL check
-                  (:func:`ssl.create_default_context` is used),
+      :param ssl: **(DEPRECATED)** SSL validation mode. ``True`` for default
+                  SSL check (:func:`ssl.create_default_context` is used),
                   ``False`` for skip SSL certificate validation,
                   :class:`aiohttp.Fingerprint` for fingerprint
                   validation, :class:`ssl.SSLContext` for custom SSL
@@ -1243,6 +1243,12 @@ is controlled by *force_close* constructor's parameter).
 
          .. versionadded:: 3.0
 
+         .. deprecated:: 4.0
+
+            Scheduled for removal in 5.0. Pass *ssl* to
+            :class:`ClientSession` for a session-wide default, or to
+            :meth:`ClientSession.get` and others per request.
+
    :param bool verify_ssl: perform SSL certificate validation for
       *HTTPS* requests (enabled by default). May be disabled to
       skip validation for sites with invalid certificates.
@@ -2482,7 +2488,7 @@ Utilities
       .. versionadded:: 3.7
 
    :param treat_as_secure_origin: (optional) Mark origins as secure
-                                  for cookies marked as Secured. Possible types are
+                                  for cookies marked as Secured.
 
                                   Possible types are:
 
@@ -2505,7 +2511,9 @@ Utilities
       :param ~yarl.URL response_url: URL of response, ``None`` for *shared
          cookies*.  Regular cookies are coupled with server's URL and
          are sent only to this server, shared ones are sent in every
-         client request.
+         client request (except that shared cookies marked ``Secure``
+         are only sent over encrypted connections or to origins listed
+         in *treat_as_secure_origin*).
 
    .. method:: filter_cookies(request_url)
 
@@ -2569,6 +2577,14 @@ Utilities
          per ``(domain, path, name)`` cookie identity so that same-named
          cookies on other paths cannot affect it.
 
+   .. attribute:: treat_as_secure_origin
+
+      A :class:`frozenset` of :class:`~yarl.URL` origins that are
+      treated as secure even when the connection is not encrypted, as
+      configured by the *treat_as_secure_origin* parameter.
+
+      .. versionadded:: 3.14.4
+
 
 .. class:: DummyCookieJar(*, loop=None)
    :canonical: aiohttp.cookiejar.DummyCookieJar

docs/spelling_wordlist.txt

@@ -378,6 +378,7 @@ un
 unawaited
 unclosed
 undercounting
+unencrypted
 unescaped
 unhandled
 unicode

docs/web_reference.rst

@@ -3139,7 +3139,7 @@ Utilities
 
 .. function:: run_app(app, *, debug=False, host=None, port=None, \
                       path=None, sock=None, shutdown_timeout=60.0, \
-                      keepalive_timeout=3630, ssl_context=None, \
+                      keepalive_timeout=75.0, ssl_context=None, \
                       print=print, backlog=128, \
                       access_log_class=aiohttp.helpers.AccessLogger, \
                       access_log_format=aiohttp.helpers.AccessLogger.LOG_FORMAT, \
@@ -3208,6 +3208,10 @@ Utilities
    :param float keepalive_timeout: a delay before a TCP connection is
                                    closed after a HTTP request. The delay
                                    allows for reuse of a TCP connection.
+                                   The same delay bounds how long a newly
+                                   accepted connection may take to deliver
+                                   its first complete request; connections
+                                   that stay idle or incomplete are closed.
 
                                    When deployed behind a reverse proxy
                                    it's important for this value to be
@@ -3217,6 +3221,11 @@ Utilities
 
       .. versionadded:: 3.8
 
+      .. versionchanged:: 3.14.4
+
+         The timeout is now also applied while waiting for the first
+         request, closing connections that never send a complete request.
+
    :param ssl_context: :class:`ssl.SSLContext` for HTTPS server,
                        ``None`` for HTTP connection.
 

examples/fake_server.py

@@ -101,9 +101,9 @@ async def main() -> None:
     fake_facebook = FakeFacebook()
     info = await fake_facebook.start()
     resolver = FakeResolver(info)
-    connector = TCPConnector(resolver=resolver, ssl=False)
+    connector = TCPConnector(resolver=resolver)
 
-    async with ClientSession(connector=connector) as session:
+    async with ClientSession(connector=connector, ssl=False) as session:
         async with session.get(
             "https://graph.facebook.com/v2.7/me", params={"access_token": token}
         ) as resp: