fix(httpx): Scope cookies to the session and origin on every request

src/crawlee/fingerprint_suite/_header_generator.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import warnings
 from typing import TYPE_CHECKING, Literal
 
 from crawlee._types import HttpHeaders
@@ -48,9 +49,17 @@ def get_specific_headers(
     def get_common_headers(self) -> HttpHeaders:
         """Get common HTTP headers ("Accept", "Accept-Language").
 
+        Deprecated, use `get_specific_headers` instead.
+
         We do not modify the "Accept-Encoding", "Connection" and other headers. They should be included and handled
         by the HTTP client or browser.
         """
+        warnings.warn(
+            '`HeaderGenerator.get_common_headers` is deprecated and will be removed in v2.0.0. '
+            'Use `get_specific_headers` instead.',
+            DeprecationWarning,
+            stacklevel=2,
+        )
         all_headers = self._generator.generate()
         return self._select_specific_headers(all_headers, header_names={'Accept', 'Accept-Language'})
 

src/crawlee/http_clients/_httpx.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import warnings
 from contextlib import asynccontextmanager
 from logging import DEBUG, WARNING, getLogger
 from typing import TYPE_CHECKING, Any, cast
@@ -62,20 +63,42 @@ async def read_stream(self) -> AsyncIterator[bytes]:
                 yield chunk
 
 
+def _same_origin(url: httpx.URL, other: httpx.URL) -> bool:
+    """Check whether two URLs share an origin."""
+    return url.scheme == other.scheme and url.host == other.host and url.port == other.port
+
+
 class _HttpxTransport(httpx.AsyncHTTPTransport):
-    """HTTP transport adapter that stores response cookies in a `Session`.
+    """HTTP transport adapter that keeps cookies in a `Session` instead of in the `HTTPX` client.
 
-    This transport adapter modifies the handling of HTTP requests to update the session cookies
-    based on the response cookies, ensuring that the cookies are stored in the session object
-    rather than the `HTTPX` client itself.
+    Response cookies are stored in the session when `persist_cookies_per_session` is enabled, and the `Cookie`
+    header is rebuilt from it before every hop, so one client can be shared by all sessions. A `Cookie` header
+    passed by the caller wins for as long as the redirect chain stays on its origin.
     """
 
+    def __init__(self, *, persist_cookies_per_session: bool, **kwargs: Any) -> None:
+        """Initialize a new instance. Extra arguments are passed to `httpx.AsyncHTTPTransport`."""
+        self._persist_cookies_per_session = persist_cookies_per_session
+        super().__init__(**kwargs)
+
     @override
     async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        session = cast('Session | None', request.extensions.get('crawlee_session'))
+        original_url, user_cookie = request.extensions.get('crawlee_caller_cookie', (None, None))
+
+        # The transport owns the `Cookie` header. Anything already on the request came from the `httpx` jar,
+        # which is scoped to no session and no origin, so it is always replaced or dropped.
+        if original_url is not None and _same_origin(original_url, request.url):
+            request.headers['cookie'] = user_cookie
+        elif session and (cookies := session.cookies.get_cookie_string(str(request.url))):
+            request.headers['cookie'] = cookies
+        else:
+            request.headers.pop('cookie', None)
+
         response = await super().handle_async_request(request)
         response.request = request
 
-        if session := cast('Session', request.extensions.get('crawlee_session')):
+        if self._persist_cookies_per_session and session:
             session.cookies.store_cookies(list(response.cookies.jar))
 
         if 'Set-Cookie' in response.headers:
@@ -123,8 +146,11 @@ def __init__(
             http1: Whether to enable HTTP/1.1 support.
             http2: Whether to enable HTTP/2 support.
             verify: SSL certificates used to verify the identity of requested hosts.
-            header_generator: Header generator instance to use for generating common headers.
-            async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`.
+            header_generator: Header generator instance to use for generating browser-like headers.
+            async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. The `mounts` and `transport`
+                arguments are ignored, they would bypass the cookie handling. The `proxy` argument covers only the
+                requests made without a `ProxyInfo`, a `ProxyConfiguration` takes precedence over it. The `limits`
+                argument applies per proxy, because every proxy gets a connection pool of its own.
         """
         super().__init__(
             persist_cookies_per_session=persist_cookies_per_session,
@@ -138,13 +164,25 @@ def __init__(
         self._http1 = http1
         self._http2 = http2
 
+        # `httpx.AsyncClient` turns a `proxy` into a mount that bypasses the cookie handling, so it is handed to
+        # the transport instead. It covers the requests that carry no `ProxyInfo` of their own.
+        self._proxy = async_client_kwargs.pop('proxy', None)
+
+        # These two would put a transport of their own in front of the one that handles the cookies.
+        for ignored_kwarg in ('mounts', 'transport'):
+            if async_client_kwargs.pop(ignored_kwarg, None) is not None:
+                warnings.warn(
+                    f'The `{ignored_kwarg}` argument of `HttpxHttpClient` is ignored, requests are sent through '
+                    'the transport that handles the cookies.',
+                    UserWarning,
+                    stacklevel=2,
+                )
+
         self._async_client_kwargs = async_client_kwargs
         self._header_generator = header_generator
 
         self._ssl_context = httpx.create_ssl_context(verify=verify)
 
-        self._transport: _HttpxTransport | None = None
-
         self._client_by_proxy_url = dict[str | None, httpx.AsyncClient]()
 
     @override
@@ -158,16 +196,15 @@ async def crawl(
         timeout: timedelta | None = None,
     ) -> HttpCrawlingResult:
         client = self._get_client(proxy_info.url if proxy_info else None)
-        headers = self._combine_headers(request.headers)
 
-        http_request = client.build_request(
+        http_request = self._build_request(
+            client=client,
+            session=session,
             url=request.url,
             method=request.method,
-            headers=headers,
-            content=request.payload,
-            cookies=session.cookies.jar if session else None,
-            extensions={'crawlee_session': session if self._persist_cookies_per_session else None},
-            timeout=timeout.total_seconds() if timeout is not None else httpx.USE_CLIENT_DEFAULT,
+            headers=request.headers,
+            payload=request.payload,
+            timeout=httpx.Timeout(timeout.total_seconds()) if timeout is not None else None,
         )
 
         try:
@@ -279,37 +316,45 @@ def _build_request(
 
         headers = self._combine_headers(headers)
 
-        return client.build_request(
+        request = client.build_request(
             url=url,
             method=method,
             headers=dict(headers) if headers else None,
             content=payload,
-            extensions={'crawlee_session': session if self._persist_cookies_per_session else None},
+            extensions={'crawlee_session': session},
             timeout=timeout or httpx.USE_CLIENT_DEFAULT,
         )
 
+        # Extensions survive a redirect, the `Cookie` header does not, so the caller's value rides along there.
+        # An empty value is kept too: it means "no cookies" and outranks the session on the origin it came from.
+        if (caller_cookie := request.headers.get('cookie')) is not None:
+            request.extensions['crawlee_caller_cookie'] = (request.url, caller_cookie)
+
+        return request
+
     def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
         """Retrieve or create an HTTP client for the given proxy URL.
 
         If a client for the specified proxy URL does not exist, create and store a new one.
         """
-        if not self._transport:
-            # Configure connection pool limits and keep-alive connections for transport
-            limits = self._async_client_kwargs.get(
-                'limits', httpx.Limits(max_connections=1000, max_keepalive_connections=200)
-            )
-
-            self._transport = _HttpxTransport(
+        if proxy_url not in self._client_by_proxy_url:
+            # A client built with `proxy=` mounts its own transport and never calls the one given to `transport=`,
+            # so the proxy has to go on the transport for the cookie handling to run.
+            transport = _HttpxTransport(
                 http1=self._http1,
                 http2=self._http2,
                 verify=self._ssl_context,
-                limits=limits,
+                proxy=proxy_url or self._proxy,
+                persist_cookies_per_session=self._persist_cookies_per_session,
+                # Above the `httpx` default of 20 kept-alive connections every request pays a TCP and TLS handshake.
+                limits=self._async_client_kwargs.get(
+                    'limits',
+                    httpx.Limits(max_connections=1000, max_keepalive_connections=200),
+                ),
             )
 
-        if proxy_url not in self._client_by_proxy_url:
             # Prepare a default kwargs for the new client.
             kwargs: dict[str, Any] = {
-                'proxy': proxy_url,
                 'http1': self._http1,
                 'http2': self._http2,
                 'follow_redirects': True,
@@ -320,7 +365,7 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
 
             kwargs.update(
                 {
-                    'transport': self._transport,
+                    'transport': transport,
                     'verify': self._ssl_context,
                 }
             )
@@ -330,19 +375,21 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient:
 
         return self._client_by_proxy_url[proxy_url]
 
-    def _combine_headers(self, explicit_headers: HttpHeaders | None) -> HttpHeaders | None:
-        """Merge default headers with explicit headers for an HTTP request.
+    def _combine_headers(self, explicit_headers: HttpHeaders | None) -> HttpHeaders:
+        """Merge generated headers with explicit headers for an HTTP request.
 
-        Generate a final set of request headers by combining default headers, a random User-Agent header,
-        and any explicitly provided headers.
+        The generated headers come from a single browser profile, so that the set stays consistent. Explicit
+        headers win over the generated ones.
         """
-        common_headers = self._header_generator.get_common_headers() if self._header_generator else HttpHeaders()
-        user_agent_header = (
-            self._header_generator.get_random_user_agent_header() if self._header_generator else HttpHeaders()
-        )
+        if self._header_generator:
+            generated_headers = self._header_generator.get_specific_headers(
+                header_names={'Accept', 'Accept-Language', 'User-Agent'},
+            )
+        else:
+            generated_headers = HttpHeaders()
+
         explicit_headers = explicit_headers or HttpHeaders()
-        headers = common_headers | user_agent_header | explicit_headers
-        return headers or None
+        return generated_headers | explicit_headers
 
     @staticmethod
     def _is_proxy_error(error: httpx.TransportError) -> bool:
@@ -363,6 +410,3 @@ async def cleanup(self) -> None:
         for client in self._client_by_proxy_url.values():
             await client.aclose()
         self._client_by_proxy_url.clear()
-        if self._transport:
-            await self._transport.aclose()
-            self._transport = None

tests/unit/fingerprint_suite/test_header_generator.py

@@ -16,7 +16,9 @@
 
 def test_get_common_headers(header_network: dict) -> None:
     header_generator = HeaderGenerator()
-    headers = header_generator.get_common_headers()
+
+    with pytest.warns(DeprecationWarning, match='get_common_headers'):
+        headers = header_generator.get_common_headers()
 
     assert 'Accept' in headers
     assert headers['Accept'] in get_available_header_values(header_network, {'Accept', 'accept'})