From 5e82ad322998285106428d2c3dfaebb38253e0c6 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 20 Aug 2026 11:15:03 +0000 Subject: [PATCH 01/11] handle cookies correctly on every request path --- .../fingerprint_suite/_header_generator.py | 9 + src/crawlee/http_clients/_httpx.py | 115 +++++--- .../test_header_generator.py | 4 +- tests/unit/http_clients/test_http_clients.py | 250 ++++++++++++++++++ tests/unit/http_clients/test_httpx.py | 64 ++++- tests/unit/http_clients/test_impit.py | 161 ----------- 6 files changed, 391 insertions(+), 212 deletions(-) diff --git a/src/crawlee/fingerprint_suite/_header_generator.py b/src/crawlee/fingerprint_suite/_header_generator.py index 1c7111db57..8d41d6c955 100644 --- a/src/crawlee/fingerprint_suite/_header_generator.py +++ b/src/crawlee/fingerprint_suite/_header_generator.py @@ -1,5 +1,6 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Literal from crawlee._types import HttpHeaders @@ -48,9 +49,17 @@ def get_specific_headers( def get_common_headers(self) -> HttpHeaders: """Get common HTTP headers ("Accept", "Accept-Language"). + Deprecated, use `get_specific_headers` instead. + We do not modify the "Accept-Encoding", "Connection" and other headers. They should be included and handled by the HTTP client or browser. """ + warnings.warn( + '`HeaderGenerator.get_common_headers` is deprecated and will be removed in v2.0.0. ' + 'Use `get_specific_headers` instead.', + DeprecationWarning, + stacklevel=2, + ) all_headers = self._generator.generate() return self._select_specific_headers(all_headers, header_names={'Accept', 'Accept-Language'}) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index ec3705a016..498d7d662e 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -62,20 +62,39 @@ async def read_stream(self) -> AsyncIterator[bytes]: yield chunk +def _same_origin(url: httpx.URL, other: httpx.URL) -> bool: + """Check whether two URLs share an origin.""" + return url.scheme == other.scheme and url.host == other.host and url.port == other.port + + class _HttpxTransport(httpx.AsyncHTTPTransport): - """HTTP transport adapter that stores response cookies in a `Session`. + """HTTP transport adapter that keeps cookies in a `Session` instead of in the `HTTPX` client. - This transport adapter modifies the handling of HTTP requests to update the session cookies - based on the response cookies, ensuring that the cookies are stored in the session object - rather than the `HTTPX` client itself. + Response cookies are stored in the session and the `Cookie` header is rebuilt from it before every hop, so + one client can be shared by all sessions. A `Cookie` header passed by the caller wins for as long as the + redirect chain stays on its origin. """ + def __init__(self, *args: Any, persist_cookies_per_session: bool, **kwargs: Any) -> None: + """Initialize a new instance. Extra arguments are passed to `httpx.AsyncHTTPTransport`.""" + self._persist_cookies_per_session = persist_cookies_per_session + super().__init__(*args, **kwargs) + @override async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + session = cast('Session | None', request.extensions.get('crawlee_session')) + original_url, user_cookie = request.extensions.get('crawlee_caller_cookie', (None, None)) + + # `httpx` drops the `Cookie` header on every redirect, so it is set here before every hop. + if user_cookie is not None and _same_origin(original_url, request.url): + request.headers['cookie'] = user_cookie + elif session and (cookies := session.cookies.get_cookie_string(str(request.url))): + request.headers['cookie'] = cookies + response = await super().handle_async_request(request) response.request = request - if session := cast('Session', request.extensions.get('crawlee_session')): + if self._persist_cookies_per_session and session: session.cookies.store_cookies(list(response.cookies.jar)) if 'Set-Cookie' in response.headers: @@ -124,7 +143,9 @@ def __init__( http2: Whether to enable HTTP/2 support. verify: SSL certificates used to verify the identity of requested hosts. header_generator: Header generator instance to use for generating common headers. - async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. + async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. The `proxy` argument is + ignored, proxies are configured through `ProxyConfiguration`. The `limits` argument applies per + proxy, because every proxy gets a connection pool of its own. """ super().__init__( persist_cookies_per_session=persist_cookies_per_session, @@ -138,13 +159,14 @@ def __init__( self._http1 = http1 self._http2 = http2 + # A `proxy=` kwarg would mount a transport of its own and bypass the cookie handling. + async_client_kwargs.pop('proxy', None) + self._async_client_kwargs = async_client_kwargs self._header_generator = header_generator self._ssl_context = httpx.create_ssl_context(verify=verify) - self._transport: _HttpxTransport | None = None - self._client_by_proxy_url = dict[str | None, httpx.AsyncClient]() @override @@ -158,16 +180,15 @@ async def crawl( timeout: timedelta | None = None, ) -> HttpCrawlingResult: client = self._get_client(proxy_info.url if proxy_info else None) - headers = self._combine_headers(request.headers) - http_request = client.build_request( + http_request = self._build_request( + client=client, + session=session, url=request.url, method=request.method, - headers=headers, - content=request.payload, - cookies=session.cookies.jar if session else None, - extensions={'crawlee_session': session if self._persist_cookies_per_session else None}, - timeout=timeout.total_seconds() if timeout is not None else httpx.USE_CLIENT_DEFAULT, + headers=request.headers, + payload=request.payload, + timeout=httpx.Timeout(timeout.total_seconds()) if timeout is not None else None, ) try: @@ -279,12 +300,19 @@ def _build_request( headers = self._combine_headers(headers) + extensions: dict[str, Any] = {'crawlee_session': session} + + # `httpx` drops the `Cookie` header on every redirect but keeps the extensions, so the header of the caller + # travels there. An empty header is kept as well, it means the caller wants no cookies sent at all. + if (caller_cookie := headers.get('cookie')) is not None: + extensions['crawlee_caller_cookie'] = (httpx.URL(url), caller_cookie) + return client.build_request( url=url, method=method, headers=dict(headers) if headers else None, content=payload, - extensions={'crawlee_session': session if self._persist_cookies_per_session else None}, + extensions=extensions, timeout=timeout or httpx.USE_CLIENT_DEFAULT, ) @@ -293,23 +321,25 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: If a client for the specified proxy URL does not exist, create and store a new one. """ - if not self._transport: - # Configure connection pool limits and keep-alive connections for transport - limits = self._async_client_kwargs.get( - 'limits', httpx.Limits(max_connections=1000, max_keepalive_connections=200) - ) + if proxy_url not in self._client_by_proxy_url: + # A client built with `proxy=` mounts a transport of its own for proxied URLs and never calls the one + # passed in `transport=`, so the proxy has to be handled by the transport to keep the cookie handling. + transport_kwargs: dict[str, Any] = { + 'http1': self._http1, + 'http2': self._http2, + 'verify': self._ssl_context, + 'proxy': proxy_url, + 'persist_cookies_per_session': self._persist_cookies_per_session, + } - self._transport = _HttpxTransport( - http1=self._http1, - http2=self._http2, - verify=self._ssl_context, - limits=limits, - ) + # Every proxy gets a pool of its own, so the `httpx` limits are left at their defaults. + if 'limits' in self._async_client_kwargs: + transport_kwargs['limits'] = self._async_client_kwargs['limits'] + + transport = _HttpxTransport(**transport_kwargs) - if proxy_url not in self._client_by_proxy_url: # Prepare a default kwargs for the new client. kwargs: dict[str, Any] = { - 'proxy': proxy_url, 'http1': self._http1, 'http2': self._http2, 'follow_redirects': True, @@ -320,7 +350,7 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: kwargs.update( { - 'transport': self._transport, + 'transport': transport, 'verify': self._ssl_context, } ) @@ -330,19 +360,21 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: return self._client_by_proxy_url[proxy_url] - def _combine_headers(self, explicit_headers: HttpHeaders | None) -> HttpHeaders | None: - """Merge default headers with explicit headers for an HTTP request. + def _combine_headers(self, explicit_headers: HttpHeaders | None) -> HttpHeaders: + """Merge generated headers with explicit headers for an HTTP request. - Generate a final set of request headers by combining default headers, a random User-Agent header, - and any explicitly provided headers. + The generated headers come from a single browser profile, so that the set stays consistent. Explicit + headers win over the generated ones. """ - common_headers = self._header_generator.get_common_headers() if self._header_generator else HttpHeaders() - user_agent_header = ( - self._header_generator.get_random_user_agent_header() if self._header_generator else HttpHeaders() - ) + if self._header_generator: + generated_headers = self._header_generator.get_specific_headers( + header_names={'Accept', 'Accept-Language', 'User-Agent'}, + ) + else: + generated_headers = HttpHeaders() + explicit_headers = explicit_headers or HttpHeaders() - headers = common_headers | user_agent_header | explicit_headers - return headers or None + return generated_headers | explicit_headers @staticmethod def _is_proxy_error(error: httpx.TransportError) -> bool: @@ -363,6 +395,3 @@ async def cleanup(self) -> None: for client in self._client_by_proxy_url.values(): await client.aclose() self._client_by_proxy_url.clear() - if self._transport: - await self._transport.aclose() - self._transport = None diff --git a/tests/unit/fingerprint_suite/test_header_generator.py b/tests/unit/fingerprint_suite/test_header_generator.py index ae9ab71bf0..840dcce6d3 100644 --- a/tests/unit/fingerprint_suite/test_header_generator.py +++ b/tests/unit/fingerprint_suite/test_header_generator.py @@ -16,7 +16,9 @@ def test_get_common_headers(header_network: dict) -> None: header_generator = HeaderGenerator() - headers = header_generator.get_common_headers() + + with pytest.warns(DeprecationWarning, match='get_common_headers'): + headers = header_generator.get_common_headers() assert 'Accept' in headers assert headers['Accept'] in get_available_header_values(header_network, {'Accept', 'accept'}) diff --git a/tests/unit/http_clients/test_http_clients.py b/tests/unit/http_clients/test_http_clients.py index aa95e1f62e..bd80242eba 100644 --- a/tests/unit/http_clients/test_http_clients.py +++ b/tests/unit/http_clients/test_http_clients.py @@ -2,6 +2,7 @@ import asyncio import importlib +import json import os import sys from typing import TYPE_CHECKING @@ -14,6 +15,7 @@ from crawlee import Request from crawlee.errors import ProxyError from crawlee.http_clients import CurlImpersonateHttpClient, HttpClient, HttpxHttpClient, ImpitHttpClient +from crawlee.sessions import CookieParam, Session from crawlee.statistics import Statistics from tests.unit.server import generate_file_content from tests.unit.server_endpoints import HELLO_WORLD @@ -24,6 +26,7 @@ from _pytest.fixtures import SubRequest from yarl import URL + from crawlee.http_clients import HttpResponse from crawlee.proxy_configuration import ProxyInfo @@ -39,6 +42,10 @@ async def custom_http_client(request: SubRequest) -> AsyncGenerator[HttpClient]: yield _ +async def read_json(response: HttpResponse) -> dict: + return json.loads((await response.read()).decode()) + + async def test_http_1(http_client: HttpClient, server_url: URL) -> None: response = await http_client.send_request(str(server_url)) assert response.http_version == 'HTTP/1.1' @@ -115,6 +122,42 @@ async def test_send_request_with_proxy_disabled( await http_client.send_request(url, proxy_info=disabled_proxy) +@pytest.mark.skipif(os.name == 'nt', reason='Skipped on Windows') +async def test_session_cookies_sent_through_proxy( + http_client: HttpClient, + proxy: ProxyInfo, + server_url: URL, +) -> None: + """Test that requests going through a proxy carry the session cookies and stay isolated per session.""" + session = Session(cookies=[CookieParam(name='jar', value='1', domain=server_url.host or '')]) + + request = Request.from_url(str(server_url / 'cookies')) + crawling_result = await http_client.crawl(request, session=session, proxy_info=proxy) + + assert (await read_json(crawling_result.http_response))['cookies'] == {'jar': '1'} + + response = await http_client.send_request(str(server_url / 'cookies'), session=session, proxy_info=proxy) + + assert (await read_json(response))['cookies'] == {'jar': '1'} + + await http_client.send_request( + str((server_url / 'set_cookies').with_query(a='1')), + session=session, + proxy_info=proxy, + ) + + assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'jar', 'a'} + + other_session = Session() + other_response = await http_client.send_request( + str(server_url / 'cookies'), + session=other_session, + proxy_info=proxy, + ) + + assert (await read_json(other_response))['cookies'] == {} + + async def test_crawl_allow_redirects_by_default(http_client: HttpClient, server_url: URL) -> None: target_url = str(server_url / 'status/200') redirect_url = str((server_url / 'redirect').update_query(url=target_url)) @@ -323,3 +366,210 @@ def test_import_error_handled(optional_module_name: str, import_path: str) -> No sys.modules.pop(mod_name, None) with pytest.raises(ImportError): importlib.import_module(import_path) + + +async def test_sessions_share_one_client(http_client: HttpClient, server_url: URL) -> None: + """Test that requests of different sessions are served by a single underlying client.""" + for _ in range(3): + await http_client.send_request(str(server_url / 'cookies'), session=Session()) + + assert len(http_client._client_by_proxy_url) == 1 # ty: ignore[unresolved-attribute] + + +async def test_cookies_isolated_per_session(http_client: HttpClient, server_url: URL) -> None: + """Test that sessions sharing a client don't see cookies of each other.""" + first_session = Session() + second_session = Session() + + await http_client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=first_session) + await http_client.send_request(str((server_url / 'set_cookies').with_query(b='2')), session=second_session) + + assert {item['name'] for item in first_session.cookies.get_cookies_as_dicts()} == {'a'} + assert {item['name'] for item in second_session.cookies.get_cookies_as_dicts()} == {'b'} + + first_response = await http_client.send_request(str(server_url / 'cookies'), session=first_session) + second_response = await http_client.send_request(str(server_url / 'cookies'), session=second_session) + + assert (await read_json(first_response))['cookies'] == {'a': '1'} + assert (await read_json(second_response))['cookies'] == {'b': '2'} + + +async def test_cookies_collected_on_redirect(http_client: HttpClient, server_url: URL) -> None: + """Test that a cookie set by a redirecting response is sent on the following hop.""" + session = Session() + + response = await http_client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=session) + + assert (await read_json(response))['cookies'] == {'a': '1'} + + +@pytest.mark.parametrize( + 'custom_http_client', + [ + pytest.param(lambda: CurlImpersonateHttpClient(persist_cookies_per_session=False), id='curl'), + pytest.param(lambda: HttpxHttpClient(persist_cookies_per_session=False), id='httpx'), + pytest.param(lambda: ImpitHttpClient(persist_cookies_per_session=False), id='impit'), + ], + indirect=['custom_http_client'], +) +async def test_cookies_not_persisted(custom_http_client: HttpClient, server_url: URL) -> None: + """Test that `persist_cookies_per_session` keeps the session jar untouched.""" + session = Session() + + await custom_http_client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=session) + + assert session.cookies.get_cookies_as_dicts() == [] + + +@pytest.mark.parametrize( + 'custom_http_client', + [ + pytest.param(lambda: CurlImpersonateHttpClient(persist_cookies_per_session=False), id='curl'), + pytest.param(lambda: HttpxHttpClient(persist_cookies_per_session=False), id='httpx'), + pytest.param(lambda: ImpitHttpClient(persist_cookies_per_session=False), id='impit'), + ], + indirect=['custom_http_client'], +) +async def test_cookies_sent_when_not_persisted(custom_http_client: HttpClient, server_url: URL) -> None: + """Test that `persist_cookies_per_session` gates storing the response cookies, not sending the session ones.""" + session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) + + response = await custom_http_client.send_request(str(server_url / 'cookies'), session=session) + + assert (await read_json(response))['cookies'] == {'from_jar': '1'} + + +async def test_cookie_header_rebuilt_per_hop(http_client: HttpClient, server_url: URL) -> None: + """Test that the `Cookie` header of one hop does not reach a hop whose URL the cookie does not match.""" + session = Session( + cookies=[CookieParam(name='scoped', value='value', domain=server_url.host or '', path='/redirect')] + ) + + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) + response = await http_client.send_request(str(redirect_url), session=session) + + assert (await read_json(response))['cookies'] == {} + assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'scoped'} + + +async def test_cookie_header_wins_over_session(http_client: HttpClient, server_url: URL) -> None: + """Test that a `Cookie` header passed by the caller replaces the cookies of the session.""" + session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) + + response = await http_client.send_request( + str(server_url / 'cookies'), + session=session, + headers={'cookie': 'manual=value'}, + ) + + assert (await read_json(response))['cookies'] == {'manual': 'value'} + + +async def test_cookie_header_kept_same_origin(http_client: HttpClient, server_url: URL) -> None: + """Test that a `Cookie` header set by the caller survives a redirect within the origin.""" + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) + + response = await http_client.send_request(str(redirect_url), headers={'cookie': 'manual=value'}) + + assert (await read_json(response))['cookies'] == {'manual': 'value'} + + +async def test_cookie_header_dropped_cross_origin( + http_client: HttpClient, + server_url: URL, + redirect_server_url: URL, +) -> None: + """Test that a `Cookie` header set by the caller is dropped once a redirect leaves the origin.""" + redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'cookies'), status=302) + + response = await http_client.send_request(str(redirect_url), headers={'cookie': 'manual=value'}) + + assert (await read_json(response))['cookies'] == {} + + +async def test_cookie_header_wins_over_session_on_redirect(http_client: HttpClient, server_url: URL) -> None: + """Test that a `Cookie` header of the caller keeps beating the session cookies after a redirect.""" + session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) + + response = await http_client.send_request( + str(redirect_url), + session=session, + headers={'cookie': 'manual=value'}, + ) + + assert (await read_json(response))['cookies'] == {'manual': 'value'} + + +async def test_empty_cookie_header_suppresses_session_cookies(http_client: HttpClient, server_url: URL) -> None: + """Test that an empty `Cookie` header of the caller keeps the session cookies out of every hop.""" + session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) + + direct = await http_client.send_request(str(server_url / 'cookies'), session=session, headers={'cookie': ''}) + redirected = await http_client.send_request(str(redirect_url), session=session, headers={'cookie': ''}) + + assert (await read_json(direct))['cookies'] == {} + assert (await read_json(redirected))['cookies'] == {} + + +async def test_auth_kept_same_origin(http_client: HttpClient, server_url: URL) -> None: + """Test that credentials survive a redirect that stays on the same origin.""" + redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'headers'), status=302) + + response = await http_client.send_request(str(redirect_url), headers={'authorization': 'Bearer token'}) + headers = await read_json(response) + + assert headers['authorization'] == 'Bearer token' + + +async def test_auth_dropped_cross_origin( + http_client: HttpClient, + server_url: URL, + redirect_server_url: URL, +) -> None: + """Test that credentials are dropped as soon as a redirect leaves the origin.""" + redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'headers'), status=302) + + response = await http_client.send_request( + str(redirect_url), + headers={'authorization': 'Bearer token', 'x-custom': 'kept'}, + ) + headers = await read_json(response) + + assert 'authorization' not in headers + assert headers['x-custom'] == 'kept' + + +async def test_stream_follows_redirects(http_client: HttpClient, server_url: URL) -> None: + """Test that streamed requests follow redirects and carry session cookies along.""" + session = Session() + stream_url = (server_url / 'set_cookies').with_query(a='1') + + async with http_client.stream(str(stream_url), session=session) as response: + content = b'' + async for chunk in response.read_stream(): + content += chunk + + assert json.loads(content.decode())['cookies'] == {'a': '1'} + assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'a'} + + +async def test_crawl_keeps_cookies_and_encoding(http_client: HttpClient, server_url: URL) -> None: + """Test that `crawl` carries session cookies through a redirect and sends signed URLs without re-encoding.""" + session = Session(cookies=[CookieParam(name='preset', value='value', domain=server_url.host or '')]) + + signed_query = 'X-Amz-Credential=AKIA%2F20240101%2Fus-east-1&X-Amz-Date=2024-01-01T00%3A00%3A00Z' + target_url = f'{server_url / "cookies"}?{signed_query}' + + direct_request = Request.from_url(target_url) + direct_result = await http_client.crawl(direct_request, session=session) + + assert (await read_json(direct_result.http_response))['cookies'] == {'preset': 'value'} + assert direct_request.loaded_url == target_url + + redirected_request = Request.from_url(str((server_url / 'redirect').with_query(url=target_url, status=302))) + redirected_result = await http_client.crawl(redirected_request, session=session) + + assert (await read_json(redirected_result.http_response))['cookies'] == {'preset': 'value'} + assert redirected_request.loaded_url == target_url diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index c98ca4bbf7..590fe74d2f 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -3,25 +3,49 @@ import json import logging from typing import TYPE_CHECKING +from unittest.mock import Mock +import httpx import pytest +from crawlee import HttpHeaders +from crawlee.fingerprint_suite import HeaderGenerator from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values from crawlee.fingerprint_suite._consts import COMMON_ACCEPT_LANGUAGE from crawlee.http_clients import HttpxHttpClient +from crawlee.http_clients._httpx import _same_origin if TYPE_CHECKING: - from collections.abc import AsyncGenerator - from yarl import URL - from crawlee.http_clients import HttpClient + from crawlee.http_clients import HttpResponse + + +async def read_json(response: HttpResponse) -> dict: + return json.loads((await response.read()).decode()) + +@pytest.mark.parametrize( + ('url', 'other', 'expected'), + [ + pytest.param('http://a.com/x', 'http://a.com/x', True, id='same'), + pytest.param('http://a.com/x', 'https://a.com/x', False, id='different-scheme'), + pytest.param('http://a.com/x', 'http://b.com/x', False, id='different-host'), + pytest.param('http://a.com/x', 'http://a.com:8080/x', False, id='different-port'), + pytest.param('http://a.com:80/x', 'http://a.com/y', True, id='explicit-default-port'), + pytest.param('http://a.com/x', 'http://a.com/y', True, id='different-path'), + ], +) +def test_same_origin(url: str, other: str, *, expected: bool) -> None: + """Test that two URLs share an origin only when their scheme, host and port match.""" + assert _same_origin(httpx.URL(url), httpx.URL(other)) is expected -@pytest.fixture -async def http_client() -> AsyncGenerator[HttpClient]: - async with HttpxHttpClient(http2=False) as client: - yield client + +def test_proxy_kwarg_does_not_reach_the_client() -> None: + """Test that a `proxy` kwarg cannot mount a transport that would bypass the cookie handling.""" + client = HttpxHttpClient(proxy='http://user:password@127.0.0.1:8888') + + assert client._get_client(None)._mounts == {} def test_silences_httpx_request_logging() -> None: @@ -54,3 +78,29 @@ async def test_common_headers_and_user_agent(server_url: URL, header_network: di assert 'user-agent' in response_headers assert 'python-httpx' not in response_headers['user-agent'] assert response_headers['user-agent'] in get_available_header_values(header_network, {'User-Agent', 'user-agent'}) + + +async def test_headers_come_from_one_sample(server_url: URL) -> None: + """Test that the impersonated headers are sampled from a single browser profile.""" + generator = Mock(spec=HeaderGenerator) + generator.get_specific_headers.return_value = HttpHeaders( + {'Accept': 'text/html', 'Accept-Language': 'en-GB', 'User-Agent': 'Mozilla/5.0 (Test)'} + ) + + async with HttpxHttpClient(header_generator=generator) as client: + response = await client.send_request(str(server_url / 'headers')) + headers = await read_json(response) + + assert headers['accept'] == 'text/html' + assert headers['accept-language'] == 'en-GB' + assert headers['user-agent'] == 'Mozilla/5.0 (Test)' + generator.get_specific_headers.assert_called_once_with(header_names={'Accept', 'Accept-Language', 'User-Agent'}) + + +async def test_no_headers_without_generator(server_url: URL) -> None: + """Test that no browser-like headers are sent once the header generator is turned off.""" + async with HttpxHttpClient(header_generator=None) as client: + response = await client.send_request(str(server_url / 'headers')) + headers = await read_json(response) + + assert 'python-httpx' in headers['user-agent'] diff --git a/tests/unit/http_clients/test_impit.py b/tests/unit/http_clients/test_impit.py index 657d94f5db..9e2fd31c8b 100644 --- a/tests/unit/http_clients/test_impit.py +++ b/tests/unit/http_clients/test_impit.py @@ -6,9 +6,7 @@ import pytest from impit import TooManyRedirects -from crawlee import Request from crawlee.http_clients import ImpitHttpClient -from crawlee.sessions import CookieParam, Session if TYPE_CHECKING: from collections.abc import AsyncGenerator @@ -27,58 +25,9 @@ async def http_client() -> AsyncGenerator[ImpitHttpClient]: async def read_json(response: HttpResponse) -> dict: - """Read the body of an HTTP response and decode it as JSON.""" return json.loads((await response.read()).decode()) -async def test_sessions_share_one_client(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that requests of different sessions are served by a single underlying client.""" - for _ in range(3): - await http_client.send_request(str(server_url / 'cookies'), session=Session()) - - assert len(http_client._client_by_proxy_url) == 1 - - -async def test_cookies_isolated_per_session(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that sessions sharing a client don't see cookies of each other.""" - first_session = Session() - second_session = Session() - - await http_client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=first_session) - await http_client.send_request(str((server_url / 'set_cookies').with_query(b='2')), session=second_session) - - assert {item['name'] for item in first_session.cookies.get_cookies_as_dicts()} == {'a'} - assert {item['name'] for item in second_session.cookies.get_cookies_as_dicts()} == {'b'} - - first_response = await http_client.send_request(str(server_url / 'cookies'), session=first_session) - second_response = await http_client.send_request(str(server_url / 'cookies'), session=second_session) - - assert (await read_json(first_response))['cookies'] == {'a': '1'} - assert (await read_json(second_response))['cookies'] == {'b': '2'} - - -async def test_cookies_collected_on_redirect(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that a cookie set by a redirecting response is sent on the following hop.""" - session = Session() - - response = await http_client.send_request( - str((server_url / 'set_cookies').with_query(a='1')), - session=session, - ) - - assert (await read_json(response))['cookies'] == {'a': '1'} - - -async def test_cookies_not_persisted(server_url: URL) -> None: - """Test that `persist_cookies_per_session` keeps the session jar untouched.""" - session = Session() - - async with ImpitHttpClient(persist_cookies_per_session=False) as client: - await client.send_request(str((server_url / 'set_cookies').with_query(a='1')), session=session) - - assert session.cookies.get_cookies_as_dicts() == [] - - @pytest.mark.parametrize( ('status_code', 'method', 'expected_method', 'expected_body'), [ @@ -133,118 +82,8 @@ async def test_body_headers_dropped(http_client: ImpitHttpClient, server_url: UR assert headers['x-custom'] == 'kept' -async def test_auth_kept_same_origin(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that credentials survive a redirect that stays on the same origin.""" - redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'headers'), status=302) - - response = await http_client.send_request(str(redirect_url), headers={'authorization': 'Bearer token'}) - headers = await read_json(response) - - assert headers['authorization'] == 'Bearer token' - - -async def test_auth_dropped_cross_origin( - http_client: ImpitHttpClient, - server_url: URL, - redirect_server_url: URL, -) -> None: - """Test that credentials are dropped as soon as a redirect leaves the origin.""" - redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'headers'), status=302) - - response = await http_client.send_request( - str(redirect_url), - headers={'authorization': 'Bearer token', 'x-custom': 'kept'}, - ) - headers = await read_json(response) - - assert 'authorization' not in headers - assert headers['x-custom'] == 'kept' - - -async def test_cookie_header_kept_same_origin(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that a `Cookie` header set by the caller survives a redirect within the origin.""" - redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) - - response = await http_client.send_request(str(redirect_url), headers={'cookie': 'manual=value'}) - - assert (await read_json(response))['cookies'] == {'manual': 'value'} - - -async def test_cookie_header_rebuilt_per_hop(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that the `Cookie` header of one hop does not reach a hop whose URL the cookie does not match.""" - session = Session( - cookies=[CookieParam(name='scoped', value='value', domain=server_url.host or '', path='/redirect')] - ) - - redirect_url = (server_url / 'redirect').with_query(url=str(server_url / 'cookies'), status=302) - response = await http_client.send_request(str(redirect_url), session=session) - - assert (await read_json(response))['cookies'] == {} - assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'scoped'} - - -async def test_cookie_header_wins_over_session(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that a `Cookie` header passed by the caller replaces the cookies of the session, as `impit` does.""" - session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) - - response = await http_client.send_request( - str(server_url / 'cookies'), - session=session, - headers={'cookie': 'manual=value'}, - ) - - assert (await read_json(response))['cookies'] == {'manual': 'value'} - - -async def test_cookie_header_dropped_cross_origin( - http_client: ImpitHttpClient, - server_url: URL, - redirect_server_url: URL, -) -> None: - """Test that a `Cookie` header set by the caller is dropped once a redirect leaves the origin.""" - redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'cookies'), status=302) - - response = await http_client.send_request(str(redirect_url), headers={'cookie': 'manual=value'}) - - assert (await read_json(response))['cookies'] == {} - - async def test_too_many_redirects(server_url: URL) -> None: """Test that an endless redirect chain is cut off by `max_redirects`.""" async with ImpitHttpClient(max_redirects=2) as client: with pytest.raises(TooManyRedirects, match='limit of 2 redirects'): await client.send_request(str(server_url / 'redirect_loop')) - - -async def test_stream_follows_redirects(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that streamed requests follow redirects and carry session cookies along.""" - session = Session() - stream_url = (server_url / 'set_cookies').with_query(a='1') - - async with http_client.stream(str(stream_url), session=session) as response: - content = b'' - async for chunk in response.read_stream(): - content += chunk - - assert json.loads(content.decode())['cookies'] == {'a': '1'} - assert {item['name'] for item in session.cookies.get_cookies_as_dicts()} == {'a'} - - -async def test_crawl_keeps_cookies_and_encoding(http_client: ImpitHttpClient, server_url: URL) -> None: - """Test that `crawl` carries session cookies through a redirect and sends signed URLs without re-encoding.""" - session = Session(cookies=[CookieParam(name='preset', value='value', domain=server_url.host or '')]) - - signed_query = 'X-Amz-Credential=AKIA%2F20240101%2Fus-east-1&X-Amz-Date=2024-01-01T00%3A00%3A00Z' - target_url = f'{server_url / "cookies"}?{signed_query}' - - direct_request = Request.from_url(target_url) - direct_result = await http_client.crawl(direct_request, session=session) - - assert json.loads((await direct_result.http_response.read()).decode())['cookies'] == {'preset': 'value'} - assert direct_request.loaded_url == target_url - - redirected_request = Request.from_url(str((server_url / 'redirect').with_query(url=target_url, status=302))) - redirected_result = await http_client.crawl(redirected_request, session=session) - - assert json.loads((await redirected_result.http_response.read()).decode())['cookies'] == {'preset': 'value'} - assert redirected_request.loaded_url == target_url From 3e1d4b1b3bb07a2f1e8a210b050c84770b62aa11 Mon Sep 17 00:00:00 2001 From: Max Bohomolov <34358312+Mantisus@users.noreply.github.com> Date: Thu, 20 Aug 2026 14:35:11 +0300 Subject: [PATCH 02/11] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- tests/unit/http_clients/test_httpx.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index 590fe74d2f..ef9317e310 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -41,11 +41,10 @@ def test_same_origin(url: str, other: str, *, expected: bool) -> None: assert _same_origin(httpx.URL(url), httpx.URL(other)) is expected -def test_proxy_kwarg_does_not_reach_the_client() -> None: +async def test_proxy_kwarg_does_not_reach_the_client() -> None: """Test that a `proxy` kwarg cannot mount a transport that would bypass the cookie handling.""" - client = HttpxHttpClient(proxy='http://user:password@127.0.0.1:8888') - - assert client._get_client(None)._mounts == {} + async with HttpxHttpClient(proxy='http://user:password@127.0.0.1:8888') as client: + assert client._get_client(None)._mounts == {} def test_silences_httpx_request_logging() -> None: From cd9ed0bef1196d5850d2badc340a6eb1e5bdf6e6 Mon Sep 17 00:00:00 2001 From: Max Bohomolov Date: Thu, 20 Aug 2026 11:50:03 +0000 Subject: [PATCH 03/11] fix --- src/crawlee/http_clients/_httpx.py | 5 +++-- tests/unit/http_clients/test_httpx.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index 498d7d662e..ba44d87e13 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -86,7 +86,7 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: original_url, user_cookie = request.extensions.get('crawlee_caller_cookie', (None, None)) # `httpx` drops the `Cookie` header on every redirect, so it is set here before every hop. - if user_cookie is not None and _same_origin(original_url, request.url): + if original_url is not None and _same_origin(original_url, request.url): request.headers['cookie'] = user_cookie elif session and (cookies := session.cookies.get_cookie_string(str(request.url))): request.headers['cookie'] = cookies @@ -303,7 +303,8 @@ def _build_request( extensions: dict[str, Any] = {'crawlee_session': session} # `httpx` drops the `Cookie` header on every redirect but keeps the extensions, so the header of the caller - # travels there. An empty header is kept as well, it means the caller wants no cookies sent at all. + # travels there. An empty header is kept as well, it suppresses the session cookies while the chain stays + # on the origin the header was meant for. if (caller_cookie := headers.get('cookie')) is not None: extensions['crawlee_caller_cookie'] = (httpx.URL(url), caller_cookie) diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index ef9317e310..30b0d9c2d3 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -44,7 +44,7 @@ def test_same_origin(url: str, other: str, *, expected: bool) -> None: async def test_proxy_kwarg_does_not_reach_the_client() -> None: """Test that a `proxy` kwarg cannot mount a transport that would bypass the cookie handling.""" async with HttpxHttpClient(proxy='http://user:password@127.0.0.1:8888') as client: - assert client._get_client(None)._mounts == {} + assert client._get_client(None)._mounts == {} # ty: ignore[unresolved-attribute] def test_silences_httpx_request_logging() -> None: From 8ce43504d0047e8442914a944a1f2350ba87898b Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:32:59 +0200 Subject: [PATCH 04/11] fix(httpx): let the transport own the `Cookie` header on every hop --- src/crawlee/http_clients/_httpx.py | 5 ++++- tests/unit/http_clients/test_httpx.py | 12 ++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index ba44d87e13..c45a3eb4bb 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -85,11 +85,14 @@ async def handle_async_request(self, request: httpx.Request) -> httpx.Response: session = cast('Session | None', request.extensions.get('crawlee_session')) original_url, user_cookie = request.extensions.get('crawlee_caller_cookie', (None, None)) - # `httpx` drops the `Cookie` header on every redirect, so it is set here before every hop. + # The transport owns the `Cookie` header. Anything already on the request came from the `httpx` jar, + # which is scoped to no session and no origin, so it is always replaced or dropped. if original_url is not None and _same_origin(original_url, request.url): request.headers['cookie'] = user_cookie elif session and (cookies := session.cookies.get_cookie_string(str(request.url))): request.headers['cookie'] = cookies + else: + request.headers.pop('cookie', None) response = await super().handle_async_request(request) response.request = request diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index 30b0d9c2d3..5a08d89395 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -96,6 +96,18 @@ async def test_headers_come_from_one_sample(server_url: URL) -> None: generator.get_specific_headers.assert_called_once_with(header_names={'Accept', 'Accept-Language', 'User-Agent'}) +async def test_client_cookies_dropped_cross_origin(server_url: URL, redirect_server_url: URL) -> None: + """Test that cookies of the underlying client reach their origin but not the target of a cross-origin redirect.""" + redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'cookies'), status=302) + + async with HttpxHttpClient(cookies={'from_client': '1'}) as client: + direct = await client.send_request(str(server_url / 'cookies')) + redirected = await client.send_request(str(redirect_url)) + + assert (await read_json(direct))['cookies'] == {'from_client': '1'} + assert (await read_json(redirected))['cookies'] == {} + + async def test_no_headers_without_generator(server_url: URL) -> None: """Test that no browser-like headers are sent once the header generator is turned off.""" async with HttpxHttpClient(header_generator=None) as client: From b528cf900e15577ebe2eb34d4140040bb9873d70 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:33:00 +0200 Subject: [PATCH 05/11] fix(httpx): capture the caller `Cookie` from the built request --- src/crawlee/http_clients/_httpx.py | 19 +++++++++---------- tests/unit/http_clients/test_httpx.py | 11 +++++++++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index c45a3eb4bb..6bbb83f40f 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -303,23 +303,22 @@ def _build_request( headers = self._combine_headers(headers) - extensions: dict[str, Any] = {'crawlee_session': session} - - # `httpx` drops the `Cookie` header on every redirect but keeps the extensions, so the header of the caller - # travels there. An empty header is kept as well, it suppresses the session cookies while the chain stays - # on the origin the header was meant for. - if (caller_cookie := headers.get('cookie')) is not None: - extensions['crawlee_caller_cookie'] = (httpx.URL(url), caller_cookie) - - return client.build_request( + request = client.build_request( url=url, method=method, headers=dict(headers) if headers else None, content=payload, - extensions=extensions, + extensions={'crawlee_session': session}, timeout=timeout or httpx.USE_CLIENT_DEFAULT, ) + # Extensions survive a redirect, the `Cookie` header does not, so the caller's value rides along there. + # An empty value is kept too: it means "no cookies" and outranks the session on the origin it came from. + if (caller_cookie := request.headers.get('cookie')) is not None: + request.extensions['crawlee_caller_cookie'] = (request.url, caller_cookie) + + return request + def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: """Retrieve or create an HTTP client for the given proxy URL. diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index 5a08d89395..c4ad1991b2 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -14,6 +14,7 @@ from crawlee.fingerprint_suite._consts import COMMON_ACCEPT_LANGUAGE from crawlee.http_clients import HttpxHttpClient from crawlee.http_clients._httpx import _same_origin +from crawlee.sessions import CookieParam, Session if TYPE_CHECKING: from yarl import URL @@ -96,6 +97,16 @@ async def test_headers_come_from_one_sample(server_url: URL) -> None: generator.get_specific_headers.assert_called_once_with(header_names={'Accept', 'Accept-Language', 'User-Agent'}) +async def test_client_cookie_header_wins_over_session(server_url: URL) -> None: + """Test that a `Cookie` header set on the underlying client replaces the cookies of the session.""" + session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) + + async with HttpxHttpClient(headers={'cookie': 'from_client=1'}) as client: + response = await client.send_request(str(server_url / 'cookies'), session=session) + + assert (await read_json(response))['cookies'] == {'from_client': '1'} + + async def test_client_cookies_dropped_cross_origin(server_url: URL, redirect_server_url: URL) -> None: """Test that cookies of the underlying client reach their origin but not the target of a cross-origin redirect.""" redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'cookies'), status=302) From 363918abc808d144022578ba2475843ad7c99ea0 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:33:01 +0200 Subject: [PATCH 06/11] fix(httpx): warn when the `proxy`, `mounts` or `transport` kwargs are ignored --- src/crawlee/http_clients/_httpx.py | 26 +++++++++++++++++++++----- tests/unit/http_clients/test_httpx.py | 24 ++++++++++++++++++------ 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index 6bbb83f40f..b22683ab8f 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import warnings from contextlib import asynccontextmanager from logging import DEBUG, WARNING, getLogger from typing import TYPE_CHECKING, Any, cast @@ -146,9 +147,9 @@ def __init__( http2: Whether to enable HTTP/2 support. verify: SSL certificates used to verify the identity of requested hosts. header_generator: Header generator instance to use for generating common headers. - async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. The `proxy` argument is - ignored, proxies are configured through `ProxyConfiguration`. The `limits` argument applies per - proxy, because every proxy gets a connection pool of its own. + async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. The `proxy`, `mounts` and + `transport` arguments are ignored, proxies are configured through `ProxyConfiguration`. The `limits` + argument applies per proxy, because every proxy gets a connection pool of its own. """ super().__init__( persist_cookies_per_session=persist_cookies_per_session, @@ -162,8 +163,23 @@ def __init__( self._http1 = http1 self._http2 = http2 - # A `proxy=` kwarg would mount a transport of its own and bypass the cookie handling. - async_client_kwargs.pop('proxy', None) + # Each of these kwargs would put a transport of its own in front of the one that handles the cookies. + if async_client_kwargs.pop('proxy', None) is not None: + warnings.warn( + 'The `proxy` argument of `HttpxHttpClient` is ignored, it does not route any request. ' + 'Configure proxies through `ProxyConfiguration`.', + UserWarning, + stacklevel=2, + ) + + for ignored_kwarg in ('mounts', 'transport'): + if async_client_kwargs.pop(ignored_kwarg, None) is not None: + warnings.warn( + f'The `{ignored_kwarg}` argument of `HttpxHttpClient` is ignored, requests are sent through ' + 'the transport that handles the cookies.', + UserWarning, + stacklevel=2, + ) self._async_client_kwargs = async_client_kwargs self._header_generator = header_generator diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index c4ad1991b2..2dd9bf6fa9 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -2,7 +2,7 @@ import json import logging -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Any from unittest.mock import Mock import httpx @@ -13,7 +13,7 @@ from crawlee.fingerprint_suite._browserforge_adapter import get_available_header_values from crawlee.fingerprint_suite._consts import COMMON_ACCEPT_LANGUAGE from crawlee.http_clients import HttpxHttpClient -from crawlee.http_clients._httpx import _same_origin +from crawlee.http_clients._httpx import _HttpxTransport, _same_origin from crawlee.sessions import CookieParam, Session if TYPE_CHECKING: @@ -42,10 +42,22 @@ def test_same_origin(url: str, other: str, *, expected: bool) -> None: assert _same_origin(httpx.URL(url), httpx.URL(other)) is expected -async def test_proxy_kwarg_does_not_reach_the_client() -> None: - """Test that a `proxy` kwarg cannot mount a transport that would bypass the cookie handling.""" - async with HttpxHttpClient(proxy='http://user:password@127.0.0.1:8888') as client: - assert client._get_client(None)._mounts == {} # ty: ignore[unresolved-attribute] +@pytest.mark.parametrize( + ('client_kwargs', 'expected_warning'), + [ + pytest.param({'proxy': 'http://user:password@127.0.0.1:8888'}, '`proxy` argument', id='proxy'), + pytest.param({'mounts': {'all://': httpx.AsyncHTTPTransport()}}, '`mounts` argument', id='mounts'), + pytest.param({'transport': httpx.AsyncHTTPTransport()}, '`transport` argument', id='transport'), + ], +) +async def test_transport_kwargs_do_not_reach_the_client(client_kwargs: dict[str, Any], expected_warning: str) -> None: + """Test that kwargs mounting a transport of their own are rejected with a warning, so the cookies keep working.""" + with pytest.warns(UserWarning, match=expected_warning): + client = HttpxHttpClient(**client_kwargs) + + async with client: + assert client._get_client(None)._mounts == {} + assert isinstance(client._get_client(None)._transport, _HttpxTransport) def test_silences_httpx_request_logging() -> None: From 23c3b7fe763828ca24acf3e61d045d9356899e80 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:33:02 +0200 Subject: [PATCH 07/11] fix(httpx): restore the larger connection pool limits on the transport --- src/crawlee/http_clients/_httpx.py | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index b22683ab8f..fdda3ec27e 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -341,21 +341,20 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: If a client for the specified proxy URL does not exist, create and store a new one. """ if proxy_url not in self._client_by_proxy_url: - # A client built with `proxy=` mounts a transport of its own for proxied URLs and never calls the one - # passed in `transport=`, so the proxy has to be handled by the transport to keep the cookie handling. - transport_kwargs: dict[str, Any] = { - 'http1': self._http1, - 'http2': self._http2, - 'verify': self._ssl_context, - 'proxy': proxy_url, - 'persist_cookies_per_session': self._persist_cookies_per_session, - } - - # Every proxy gets a pool of its own, so the `httpx` limits are left at their defaults. - if 'limits' in self._async_client_kwargs: - transport_kwargs['limits'] = self._async_client_kwargs['limits'] - - transport = _HttpxTransport(**transport_kwargs) + # A client built with `proxy=` mounts its own transport and never calls the one given to `transport=`, + # so the proxy has to go on the transport for the cookie handling to run. + transport = _HttpxTransport( + http1=self._http1, + http2=self._http2, + verify=self._ssl_context, + proxy=proxy_url, + persist_cookies_per_session=self._persist_cookies_per_session, + # Above the `httpx` default of 20 kept-alive connections every request pays a TCP and TLS handshake. + limits=self._async_client_kwargs.get( + 'limits', + httpx.Limits(max_connections=1000, max_keepalive_connections=200), + ), + ) # Prepare a default kwargs for the new client. kwargs: dict[str, Any] = { From c7b811d19579ca950817b15592ab1f40c34d0d56 Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:33:03 +0200 Subject: [PATCH 08/11] refactor(httpx): drop the unused positional passthrough on the transport --- src/crawlee/http_clients/_httpx.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index fdda3ec27e..fad26fdfee 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -76,10 +76,10 @@ class _HttpxTransport(httpx.AsyncHTTPTransport): redirect chain stays on its origin. """ - def __init__(self, *args: Any, persist_cookies_per_session: bool, **kwargs: Any) -> None: + def __init__(self, *, persist_cookies_per_session: bool, **kwargs: Any) -> None: """Initialize a new instance. Extra arguments are passed to `httpx.AsyncHTTPTransport`.""" self._persist_cookies_per_session = persist_cookies_per_session - super().__init__(*args, **kwargs) + super().__init__(**kwargs) @override async def handle_async_request(self, request: httpx.Request) -> httpx.Response: From 42c517da0921a20bf8df259a2e0f703bf533c50a Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:33:04 +0200 Subject: [PATCH 09/11] docs(httpx): correct the transport and header generator docstrings --- src/crawlee/http_clients/_httpx.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index fad26fdfee..48a926c1ca 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -71,9 +71,9 @@ def _same_origin(url: httpx.URL, other: httpx.URL) -> bool: class _HttpxTransport(httpx.AsyncHTTPTransport): """HTTP transport adapter that keeps cookies in a `Session` instead of in the `HTTPX` client. - Response cookies are stored in the session and the `Cookie` header is rebuilt from it before every hop, so - one client can be shared by all sessions. A `Cookie` header passed by the caller wins for as long as the - redirect chain stays on its origin. + Response cookies are stored in the session when `persist_cookies_per_session` is enabled, and the `Cookie` + header is rebuilt from it before every hop, so one client can be shared by all sessions. A `Cookie` header + passed by the caller wins for as long as the redirect chain stays on its origin. """ def __init__(self, *, persist_cookies_per_session: bool, **kwargs: Any) -> None: @@ -146,7 +146,7 @@ def __init__( http1: Whether to enable HTTP/1.1 support. http2: Whether to enable HTTP/2 support. verify: SSL certificates used to verify the identity of requested hosts. - header_generator: Header generator instance to use for generating common headers. + header_generator: Header generator instance to use for generating browser-like headers. async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. The `proxy`, `mounts` and `transport` arguments are ignored, proxies are configured through `ProxyConfiguration`. The `limits` argument applies per proxy, because every proxy gets a connection pool of its own. From ce0813bb0e3281fff271b35cc6543468139747ba Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:33:05 +0200 Subject: [PATCH 10/11] test(httpx): cover the session cookie takeover and the disabled header generator --- tests/unit/http_clients/test_http_clients.py | 18 ++++++++++++++++++ tests/unit/http_clients/test_httpx.py | 2 ++ 2 files changed, 20 insertions(+) diff --git a/tests/unit/http_clients/test_http_clients.py b/tests/unit/http_clients/test_http_clients.py index bd80242eba..dc1e54261a 100644 --- a/tests/unit/http_clients/test_http_clients.py +++ b/tests/unit/http_clients/test_http_clients.py @@ -487,6 +487,24 @@ async def test_cookie_header_dropped_cross_origin( assert (await read_json(response))['cookies'] == {} +async def test_session_cookies_take_over_cross_origin( + http_client: HttpClient, + server_url: URL, + redirect_server_url: URL, +) -> None: + """Test that the session cookies of the new origin take over once a redirect leaves the origin of the caller.""" + session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=redirect_server_url.host or '')]) + redirect_url = (server_url / 'redirect').with_query(url=str(redirect_server_url / 'cookies'), status=302) + + response = await http_client.send_request( + str(redirect_url), + session=session, + headers={'cookie': 'manual=value'}, + ) + + assert (await read_json(response))['cookies'] == {'from_jar': '1'} + + async def test_cookie_header_wins_over_session_on_redirect(http_client: HttpClient, server_url: URL) -> None: """Test that a `Cookie` header of the caller keeps beating the session cookies after a redirect.""" session = Session(cookies=[CookieParam(name='from_jar', value='1', domain=server_url.host or '')]) diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index 2dd9bf6fa9..247ba327ad 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -138,3 +138,5 @@ async def test_no_headers_without_generator(server_url: URL) -> None: headers = await read_json(response) assert 'python-httpx' in headers['user-agent'] + assert headers['accept'] == '*/*' + assert 'accept-language' not in headers From 11230731087408d480c6ed14cc37632b1d00fa9d Mon Sep 17 00:00:00 2001 From: Vlada Dusek Date: Fri, 21 Aug 2026 12:48:41 +0200 Subject: [PATCH 11/11] fix(httpx): keep the `proxy` kwarg working through the transport --- src/crawlee/http_clients/_httpx.py | 19 ++++++++----------- tests/unit/http_clients/test_httpx.py | 27 ++++++++++++++++++++++++++- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/src/crawlee/http_clients/_httpx.py b/src/crawlee/http_clients/_httpx.py index 48a926c1ca..b2aaaa09c3 100644 --- a/src/crawlee/http_clients/_httpx.py +++ b/src/crawlee/http_clients/_httpx.py @@ -147,8 +147,9 @@ def __init__( http2: Whether to enable HTTP/2 support. verify: SSL certificates used to verify the identity of requested hosts. header_generator: Header generator instance to use for generating browser-like headers. - async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. The `proxy`, `mounts` and - `transport` arguments are ignored, proxies are configured through `ProxyConfiguration`. The `limits` + async_client_kwargs: Additional keyword arguments for `httpx.AsyncClient`. The `mounts` and `transport` + arguments are ignored, they would bypass the cookie handling. The `proxy` argument covers only the + requests made without a `ProxyInfo`, a `ProxyConfiguration` takes precedence over it. The `limits` argument applies per proxy, because every proxy gets a connection pool of its own. """ super().__init__( @@ -163,15 +164,11 @@ def __init__( self._http1 = http1 self._http2 = http2 - # Each of these kwargs would put a transport of its own in front of the one that handles the cookies. - if async_client_kwargs.pop('proxy', None) is not None: - warnings.warn( - 'The `proxy` argument of `HttpxHttpClient` is ignored, it does not route any request. ' - 'Configure proxies through `ProxyConfiguration`.', - UserWarning, - stacklevel=2, - ) + # `httpx.AsyncClient` turns a `proxy` into a mount that bypasses the cookie handling, so it is handed to + # the transport instead. It covers the requests that carry no `ProxyInfo` of their own. + self._proxy = async_client_kwargs.pop('proxy', None) + # These two would put a transport of their own in front of the one that handles the cookies. for ignored_kwarg in ('mounts', 'transport'): if async_client_kwargs.pop(ignored_kwarg, None) is not None: warnings.warn( @@ -347,7 +344,7 @@ def _get_client(self, proxy_url: str | None) -> httpx.AsyncClient: http1=self._http1, http2=self._http2, verify=self._ssl_context, - proxy=proxy_url, + proxy=proxy_url or self._proxy, persist_cookies_per_session=self._persist_cookies_per_session, # Above the `httpx` default of 20 kept-alive connections every request pays a TCP and TLS handshake. limits=self._async_client_kwargs.get( diff --git a/tests/unit/http_clients/test_httpx.py b/tests/unit/http_clients/test_httpx.py index 247ba327ad..4f6f79f738 100644 --- a/tests/unit/http_clients/test_httpx.py +++ b/tests/unit/http_clients/test_httpx.py @@ -20,6 +20,7 @@ from yarl import URL from crawlee.http_clients import HttpResponse + from crawlee.proxy_configuration import ProxyInfo async def read_json(response: HttpResponse) -> dict: @@ -45,7 +46,6 @@ def test_same_origin(url: str, other: str, *, expected: bool) -> None: @pytest.mark.parametrize( ('client_kwargs', 'expected_warning'), [ - pytest.param({'proxy': 'http://user:password@127.0.0.1:8888'}, '`proxy` argument', id='proxy'), pytest.param({'mounts': {'all://': httpx.AsyncHTTPTransport()}}, '`mounts` argument', id='mounts'), pytest.param({'transport': httpx.AsyncHTTPTransport()}, '`transport` argument', id='transport'), ], @@ -60,6 +60,31 @@ async def test_transport_kwargs_do_not_reach_the_client(client_kwargs: dict[str, assert isinstance(client._get_client(None)._transport, _HttpxTransport) +async def test_proxy_kwarg_routes_requests(server_url: URL) -> None: + """Test that a `proxy` kwarg routes the requests that carry no `ProxyInfo` of their own.""" + # Port 1 refuses every connection, so the request can only reach the server if the proxy is not used. + async with HttpxHttpClient(proxy='http://127.0.0.1:1') as client: + with pytest.raises(httpx.ConnectError): + await client.send_request(str(server_url / 'status/222')) + + +async def test_proxy_kwarg_works_against_a_real_proxy(proxy: ProxyInfo, server_url: URL) -> None: + """Test that a `proxy` kwarg still gets the response through once the proxy accepts the connection.""" + async with HttpxHttpClient(proxy=proxy.url) as client: + response = await client.send_request(str(server_url / 'status/222')) + + assert response.status_code == 222 + + +async def test_proxy_info_wins_over_the_proxy_kwarg(proxy: ProxyInfo, server_url: URL) -> None: + """Test that the `ProxyInfo` of a request takes precedence over the `proxy` kwarg of the client.""" + # Port 1 refuses every connection, so the request only succeeds if the `ProxyInfo` is the one being used. + async with HttpxHttpClient(proxy='http://127.0.0.1:1') as client: + response = await client.send_request(str(server_url / 'status/222'), proxy_info=proxy) + + assert response.status_code == 222 + + def test_silences_httpx_request_logging() -> None: """Instantiating the client lowers the noisy per-request `httpx` INFO logs to WARNING by default.""" httpx_logger = logging.getLogger('httpx')