diff --git a/crawl4ai/async_configs.py b/crawl4ai/async_configs.py index 27320fd4b..8cbc213b2 100644 --- a/crawl4ai/async_configs.py +++ b/crawl4ai/async_configs.py @@ -1436,6 +1436,14 @@ class CrawlerRunConfig(): Default: "domcontentloaded". page_timeout (int): Timeout in ms for page operations like navigation. Default: 60000 (60 seconds). + total_timeout (int or None): Hard ceiling in ms for the whole fetch phase of a + single arun() — every anti-bot attempt and proxy retry + share this one budget. page_timeout only bounds + navigation and the wait_* family; a page that keeps + renavigating can wedge an untimed protocol call and + burn an unbounded amount of wall clock. Set this when + the caller has a real deadline. None disables it. + Default: None. wait_for (str or None): A CSS selector or JS condition to wait for before extracting content. Default: None. wait_for_timeout (int or None): Specific timeout in ms for the wait_for condition. @@ -1627,6 +1635,7 @@ def __init__( # Page Navigation and Timing Parameters wait_until: str = "domcontentloaded", page_timeout: int = PAGE_TIMEOUT, + total_timeout: Optional[int] = None, wait_for: str = None, wait_for_timeout: int = None, wait_for_images: bool = False, @@ -1756,6 +1765,7 @@ def __init__( # Page Navigation and Timing Parameters self.wait_until = wait_until self.page_timeout = page_timeout + self.total_timeout = total_timeout self.wait_for = wait_for self.wait_for_timeout = wait_for_timeout self.wait_for_images = wait_for_images @@ -2126,6 +2136,7 @@ def to_dict(self): "shared_data": self.shared_data, "wait_until": self.wait_until, "page_timeout": self.page_timeout, + "total_timeout": self.total_timeout, "wait_for": self.wait_for, "wait_for_timeout": self.wait_for_timeout, "wait_for_images": self.wait_for_images, diff --git a/crawl4ai/async_crawler_strategy.py b/crawl4ai/async_crawler_strategy.py index 265c376e9..7ef7ca3bf 100644 --- a/crawl4ai/async_crawler_strategy.py +++ b/crawl4ai/async_crawler_strategy.py @@ -22,7 +22,12 @@ from .ssl_certificate import SSLCertificate from .user_agent_generator import ValidUAGenerator, UAGen from .browser_manager import BrowserManager -from .browser_adapter import BrowserAdapter, PlaywrightAdapter, UndetectedAdapter +from .browser_adapter import ( + EVALUATE_TIMEOUT_S, + BrowserAdapter, + PlaywrightAdapter, + UndetectedAdapter, +) import aiofiles import aiohttp @@ -33,6 +38,22 @@ import contextlib from functools import partial +# --- Bounds for Playwright calls that carry no protocol timeout ------------ +# page.content() and page.evaluate() are sent without a `timeout` field, so the +# driver arms no timer and they can only end when the target replies or closes. +# See AsyncPlaywrightCrawlerStrategy._capture_html and browser_adapter. +HTML_CAPTURE_TIMEOUT_S: Final[float] = 15.0 # per page.content() attempt +HTML_CAPTURE_TOTAL_TIMEOUT_S: Final[float] = 25.0 # across all attempts +HTML_CAPTURE_SETTLE_TIMEOUT_S: Final[float] = 5.0 # wait for the next document +HTML_CAPTURE_ATTEMPTS: Final[int] = 3 +PAGE_CLOSE_TIMEOUT_S: Final[float] = 10.0 # page.close() is unbounded too +VIRTUAL_SCROLL_TIMEOUT_S: Final[float] = 300.0 # in-page scroll loop, legitimately slow +# Cosmetic DOM steps that already degrade gracefully (image dimensions, consent +# and overlay removal). They are worth a few seconds, never worth a minute, so +# they get a tighter ceiling than the adapter default. +OPTIONAL_DOM_STEP_TIMEOUT_S: Final[float] = 10.0 + + class AsyncCrawlerStrategy(ABC): """ Abstract base class for crawler strategies. @@ -333,7 +354,12 @@ async def csp_compliant_wait( """ try: - result = await self.adapter.evaluate(page, wrapper_js) + # The polling loop above enforces `timeout` itself; the adapter + # bound only has to catch a page whose execution context never + # settles, so give it headroom rather than racing the JS. + result = await self.adapter.evaluate( + page, wrapper_js, timeout=timeout / 1000.0 + EVALUATE_TIMEOUT_S + ) return result except Exception as e: if "Error evaluating condition" in str(e): @@ -511,6 +537,66 @@ async def crawl( "URL must start with 'http://', 'https://', 'file://', or 'raw:'" ) + async def _capture_html(self, page: Page, attempts: int = None) -> str: + """Capture the page HTML, tolerating a page that is still navigating. + + `page.content()` carries no protocol timeout — the Python client sends + no timeout field, so the driver arms no timer and the call waits on the + frame's execution-context promise. On a page that keeps committing + navigations that promise is repeatedly replaced, giving two failure + modes for the same race: + + * the context dies *during* the call -> Playwright raises + "Unable to retrieve content because the page is navigating and + changing the content"; + * the context is already gone *at* the call -> it blocks forever + (`page_timeout` does not cover this; only an external deadline does). + + Both are transient. Wait for the new document to reach + domcontentloaded and capture again, which is the documented remedy and + is far cheaper than re-running the whole crawl. + """ + attempts = attempts or HTML_CAPTURE_ATTEMPTS + # A recoverable race raises immediately, so retries are nearly free; a + # wedged page burns a full timeout each time. Bound the retries as a + # group so only the wedged case pays, and it pays once. + deadline = time.perf_counter() + HTML_CAPTURE_TOTAL_TIMEOUT_S + last_err: Optional[BaseException] = None + for _i in range(attempts): + budget = min(HTML_CAPTURE_TIMEOUT_S, deadline - time.perf_counter()) + if budget <= 0: + break + try: + return await asyncio.wait_for(page.content(), budget) + except asyncio.TimeoutError: + last_err = PlaywrightTimeoutError( + f"page.content() did not return within {budget:.0f}s " + f"— the page never stopped navigating" + ) + except Error as e: + last_err = e + if _i >= attempts - 1: + break + self.logger.debug( + message="HTML capture attempt {n} failed ({err}) — letting the page settle", + tag="SCRAPE", + params={"n": _i + 1, "err": str(last_err)[:120]}, + ) + try: + await page.wait_for_load_state( + "domcontentloaded", + timeout=HTML_CAPTURE_SETTLE_TIMEOUT_S * 1000, + ) + except Exception: + # The page cannot even reach domcontentloaded, so it is not + # between documents — it is stuck. Another capture attempt + # would only buy another full timeout. Give up now. + break + raise last_err or PlaywrightTimeoutError( + f"page.content() could not be captured within " + f"{HTML_CAPTURE_TOTAL_TIMEOUT_S:g}s" + ) + async def _crawl_web( self, url: str, config: CrawlerRunConfig ) -> AsyncCrawlResponse: @@ -1029,7 +1115,10 @@ async def handle_request_failed_capture(request): await page.wait_for_load_state("domcontentloaded", timeout=5) except PlaywrightTimeoutError: pass - await self.adapter.evaluate(page, update_image_dimensions_js) + await self.adapter.evaluate( + page, update_image_dimensions_js, + timeout=OPTIONAL_DOM_STEP_TIMEOUT_S, + ) except Exception as e: self.logger.error( message="Error updating image dimensions: {error}", @@ -1061,7 +1150,7 @@ async def handle_request_failed_capture(request): message="Shadow DOM flattening returned no content, falling back to page.content()", tag="SCRAPE", ) - html = await page.content() + html = await self._capture_html(page) elif config.css_selector: try: selectors = [s.strip() for s in config.css_selector.split(',')] @@ -1082,7 +1171,7 @@ async def handle_request_failed_capture(request): except Error as e: raise RuntimeError(f"Failed to extract HTML content: {str(e)}") else: - html = await page.content() + html = await self._capture_html(page) await self.execute_hook( "before_return_html", page=page, html=html, context=context, config=config @@ -1126,7 +1215,7 @@ async def get_delayed_content(delay: float = 5.0) -> str: params={"delay": delay, "url": url}, ) await asyncio.sleep(delay) - return await page.content() + return await self._capture_html(page) # For undetected browsers, retrieve console messages before returning if config.capture_console_messages and hasattr(self.adapter, 'retrieve_console_messages'): @@ -1196,7 +1285,11 @@ async def get_delayed_content(delay: float = 5.0) -> str: all_contexts = page.context.browser.contexts total_pages = sum(len(context.pages) for context in all_contexts) if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)): - await page.close() + # page.close() is also sent without a timeout and waits + # on the target's closed-promise, so a wedged renderer + # can block cleanup indefinitely — including while this + # coroutine is being cancelled by an outer deadline. + await asyncio.wait_for(page.close(), PAGE_CLOSE_TIMEOUT_S) except Exception: pass @@ -1428,8 +1521,13 @@ async def _handle_virtual_scroll(self, page: Page, config: "VirtualScrollConfig" } """ - # Execute virtual scroll capture - result = await self.adapter.evaluate(page, virtual_scroll_js, config.to_dict()) + # Execute virtual scroll capture. Unlike the other evaluates this + # one legitimately runs a long scroll loop inside the page, so it + # gets its own generous ceiling rather than the adapter default. + result = await self.adapter.evaluate( + page, virtual_scroll_js, config.to_dict(), + timeout=VIRTUAL_SCROLL_TIMEOUT_S, + ) if result.get("replaced", False): self.logger.success( @@ -1532,7 +1630,8 @@ async def remove_overlay_elements(self, page: Page) -> None: }}; }} }})() - """ + """, + timeout=OPTIONAL_DOM_STEP_TIMEOUT_S, ) await page.wait_for_timeout(500) # Wait for any animations to complete except Exception as e: @@ -1576,7 +1675,8 @@ async def remove_consent_popups(self, page: Page) -> None: }}; }} }})() - """ + """, + timeout=OPTIONAL_DOM_STEP_TIMEOUT_S, ) await page.wait_for_timeout(500) # Wait for any animations to complete except Exception as e: diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 8216d19bc..8876d24c6 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -403,6 +403,18 @@ async def arun( _is_raw_url = url.startswith("raw:") or url.startswith("raw://") _max_attempts = 1 + getattr(config, "max_retries", 0) + # One shared budget for the whole fetch phase (every attempt + # and every proxy). page_timeout only bounds navigation and + # the wait_* family; Playwright calls sent without a timeout + # (page.content, page.evaluate, page.close) are not covered by + # anything, so without this a single wedged page can consume + # the caller's entire deadline in silence. + _total_timeout = getattr(config, "total_timeout", None) + _fetch_deadline = ( + time.perf_counter() + _total_timeout / 1000.0 + if _total_timeout + else None + ) _proxy_list = config._get_proxy_list() _original_proxy_config = config.proxy_config _block_reason = "" @@ -456,8 +468,25 @@ async def arun( self.crawler_strategy.update_user_agent( config.user_agent) - async_response = await self.crawler_strategy.crawl( - url, config=config) + _remaining = None + if _fetch_deadline is not None: + _remaining = _fetch_deadline - time.perf_counter() + if _remaining <= 0: + raise TimeoutError( + f"Fetch budget of {_total_timeout} ms exhausted " + f"before attempt {_attempt + 1}/{_max_attempts}") + try: + async_response = await asyncio.wait_for( + self.crawler_strategy.crawl(url, config=config), + timeout=_remaining, + ) + except asyncio.TimeoutError: + # asyncio.TimeoutError carries no message — + # name the budget so the failure is attributable. + raise TimeoutError( + f"Crawl attempt exceeded the {_total_timeout} ms " + f"fetch budget ({_remaining:.1f}s remained for " + f"attempt {_attempt + 1}/{_max_attempts})") from None html = sanitize_input_encode(async_response.html) screenshot_data = async_response.screenshot diff --git a/crawl4ai/browser_adapter.py b/crawl4ai/browser_adapter.py index 2f4c15827..819e60ed7 100644 --- a/crawl4ai/browser_adapter.py +++ b/crawl4ai/browser_adapter.py @@ -6,14 +6,17 @@ from abc import ABC, abstractmethod from typing import List, Dict, Any, Optional, Callable +import asyncio import time import json # Import both, but use conditionally try: from playwright.async_api import Page + from playwright.async_api import TimeoutError as PlaywrightTimeoutError except ImportError: Page = Any + PlaywrightTimeoutError = TimeoutError try: from patchright.async_api import Page as UndetectedPage @@ -21,11 +24,51 @@ UndetectedPage = Any +# --------------------------------------------------------------------------- +# Why evaluate() needs an explicit bound +# +# Playwright's evaluate() takes no `timeout` argument. The Python client sends +# no timeout field, so the driver arms no timer at all — the call can only ever +# end when it gets a reply or the target is closed. What it waits on is the +# frame's execution-context promise, and every navigation *replaces* that +# promise with a fresh, unresolved one. So an evaluate issued while the main +# frame is between documents never returns: `page_timeout` does not cover it +# (that only reaches page.goto and the wait_* family), and because most call +# sites wrap evaluate in a swallow-all try/except, nothing is logged either. +# +# Observed in production 2026-07-30: a WordPress + Turnstile site kept +# committing navigations, an evaluate stalled, and the request burned the whole +# 180 s wall-clock fence in total silence. Bound it so a wedged renderer costs +# seconds instead of the caller's entire budget. +# --------------------------------------------------------------------------- +EVALUATE_TIMEOUT_S = 30.0 + + +async def bounded_evaluate(awaitable, timeout: float, what: str = "page.evaluate"): + """Await a Playwright evaluate with a hard ceiling. + + Raises Playwright's TimeoutError — the same type Playwright itself raises + for the operations that *do* carry a timeout, so existing + `except Error` / `except Exception` handlers keep working, and it stays + distinguishable from an outer asyncio deadline. + """ + try: + return await asyncio.wait_for(awaitable, timeout) + except asyncio.TimeoutError: + raise PlaywrightTimeoutError( + f"{what} did not return within {timeout:g}s — the page is most " + f"likely navigating and its execution context never settled" + ) from None + + class BrowserAdapter(ABC): """Abstract adapter for browser-specific operations""" @abstractmethod - async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any: + async def evaluate( + self, page: Page, expression: str, arg: Any = None, + timeout: Optional[float] = None, + ) -> Any: """Execute JavaScript in the page""" pass @@ -58,11 +101,13 @@ def get_imports(self) -> tuple: class PlaywrightAdapter(BrowserAdapter): """Adapter for standard Playwright""" - async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any: - """Standard Playwright evaluate""" - if arg is not None: - return await page.evaluate(expression, arg) - return await page.evaluate(expression) + async def evaluate( + self, page: Page, expression: str, arg: Any = None, + timeout: Optional[float] = None, + ) -> Any: + """Standard Playwright evaluate, bounded (see bounded_evaluate)""" + coro = page.evaluate(expression, arg) if arg is not None else page.evaluate(expression) + return await bounded_evaluate(coro, timeout or EVALUATE_TIMEOUT_S) async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: """Setup console capture using Playwright's event system""" @@ -175,11 +220,13 @@ async def apply_stealth(self, page: Page): # Fail silently or log error depending on requirements pass - async def evaluate(self, page: Page, expression: str, arg: Any = None) -> Any: - """Standard Playwright evaluate with stealth applied""" - if arg is not None: - return await page.evaluate(expression, arg) - return await page.evaluate(expression) + async def evaluate( + self, page: Page, expression: str, arg: Any = None, + timeout: Optional[float] = None, + ) -> Any: + """Standard Playwright evaluate with stealth applied, bounded""" + coro = page.evaluate(expression, arg) if arg is not None else page.evaluate(expression) + return await bounded_evaluate(coro, timeout or EVALUATE_TIMEOUT_S) async def setup_console_capture(self, page: Page, captured_console: List[Dict]) -> Optional[Callable]: """Setup console capture using Playwright's event system with stealth""" @@ -274,8 +321,11 @@ class UndetectedAdapter(BrowserAdapter): def __init__(self): self._console_script_injected = {} - async def evaluate(self, page: UndetectedPage, expression: str, arg: Any = None) -> Any: - """Undetected browser evaluate with isolated context""" + async def evaluate( + self, page: UndetectedPage, expression: str, arg: Any = None, + timeout: Optional[float] = None, + ) -> Any: + """Undetected browser evaluate with isolated context, bounded""" # For most evaluations, use isolated context for stealth # Only use non-isolated when we need to access our injected console capture isolated = not ( @@ -285,9 +335,12 @@ async def evaluate(self, page: UndetectedPage, expression: str, arg: Any = None) "window.__" in expression ) - if arg is not None: - return await page.evaluate(expression, arg, isolated_context=isolated) - return await page.evaluate(expression, isolated_context=isolated) + coro = ( + page.evaluate(expression, arg, isolated_context=isolated) + if arg is not None + else page.evaluate(expression, isolated_context=isolated) + ) + return await bounded_evaluate(coro, timeout or EVALUATE_TIMEOUT_S) async def setup_console_capture(self, page: UndetectedPage, captured_console: List[Dict]) -> Optional[Callable]: """Setup console capture using JavaScript injection for undetected browsers""" diff --git a/docs/md_v2/advanced/anti-bot-and-fallback.md b/docs/md_v2/advanced/anti-bot-and-fallback.md index 68ed03de8..d40ac5324 100644 --- a/docs/md_v2/advanced/anti-bot-and-fallback.md +++ b/docs/md_v2/advanced/anti-bot-and-fallback.md @@ -23,6 +23,7 @@ All anti-bot retry options live on `CrawlerRunConfig`: |---|---|---|---| | `proxy_config` | `ProxyConfig`, `list[ProxyConfig]`, or `None` | `None` | Single proxy or ordered list of proxies to try. Each retry round iterates through the full list. Use `"direct"` or `ProxyConfig.DIRECT` in a list to explicitly try without a proxy. | | `max_retries` | `int` | `0` | Number of retry rounds when blocking is detected. `0` = no retries. | +| `total_timeout` | `int` or `None` (ms) | `None` | Hard ceiling for the whole escalation chain below — every round and every proxy share this one budget. An attempt that would exceed it is cancelled, and a round that cannot start within it is skipped. Recommended whenever the caller has a deadline: `page_timeout` bounds only navigation and the `wait_*` family, so without `total_timeout` the worst case is unbounded. | | `fallback_fetch_function` | `async (str) -> str` | `None` | Async function called as last resort. Takes URL, returns raw HTML. | ## Escalation Chain diff --git a/docs/md_v2/api/parameters.md b/docs/md_v2/api/parameters.md index 568e14c30..f987061bc 100644 --- a/docs/md_v2/api/parameters.md +++ b/docs/md_v2/api/parameters.md @@ -140,6 +140,7 @@ Use these for controlling whether you read or write from a local content cache. |----------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------| | **`wait_until`** | `str` (domcontentloaded)| Condition for navigation to "complete". Often `"networkidle"` or `"domcontentloaded"`. | | **`page_timeout`** | `int` (60000 ms) | Timeout for page navigation or JS steps. Increase for slow sites. | +| **`total_timeout`** | `int or None` (None) | Hard ceiling in ms for the whole fetch phase of one `arun()` — every anti-bot attempt and proxy retry share this one budget. `page_timeout` bounds only navigation and the `wait_*` family, so set this when you have a real deadline. `None` disables it. | | **`wait_for`** | `str or None` | Wait for a CSS (`"css:selector"`) or JS (`"js:() => bool"`) condition before content extraction. | | **`wait_for_timeout`** | `int or None` (None) | Specific timeout in ms for the `wait_for` condition. If None, uses `page_timeout`. | | **`wait_for_images`** | `bool` (False) | Wait for images to load before finishing. Slows down if you only want text. | diff --git a/tests/async/test_render_call_bounds.py b/tests/async/test_render_call_bounds.py new file mode 100644 index 000000000..58b125013 --- /dev/null +++ b/tests/async/test_render_call_bounds.py @@ -0,0 +1,300 @@ +""" +Playwright calls that carry no protocol timeout must still be bounded. + +`page.content()` and `page.evaluate()` are sent to the driver with **no** +`timeout` field, so no timer is armed server-side and they can only end when +they get a reply or the target closes. What they wait on is the frame's +execution-context promise, which every navigation replaces with a fresh, +unresolved one. A page that keeps committing navigations therefore wedges them +forever: `page_timeout` does not cover it (that reaches only `page.goto` and the +`wait_*` family), and because the call sites wrap them in swallow-all +`try/except`, nothing is logged either. + +Three bounds are pinned here, outermost last: + 1. `bounded_evaluate` — every adapter-mediated `page.evaluate` + 2. `_capture_html` — `page.content()`, with settle-and-retry + 3. `total_timeout` — one budget shared by every attempt in `arun()` + +No browser and no network needed. + + pytest tests/async/test_render_call_bounds.py -q +""" + +import asyncio +import time + +import pytest +from playwright.async_api import Error as PlaywrightError +from playwright.async_api import TimeoutError as PlaywrightTimeoutError + +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai import async_crawler_strategy as acs +from crawl4ai.async_crawler_strategy import AsyncPlaywrightCrawlerStrategy +from crawl4ai.browser_adapter import ( + EVALUATE_TIMEOUT_S, + PlaywrightAdapter, + bounded_evaluate, +) +from crawl4ai.models import AsyncCrawlResponse + +# The exact message Playwright raises when the context dies mid-capture. +NAVIGATING = ( + "Page.content: Unable to retrieve content because the page is navigating " + "and changing the content." +) + +PAGE_HTML = ( + "Example Co" + "

Example Co

Contact: info@example.com

" + "

Phone +1 555 0100. Address: 1 Example Street, Springfield.

" + "" + "

Example Co is a company that does example things for example people.

" + "" +) + + +def _run(coro): + return asyncio.get_event_loop_policy().new_event_loop().run_until_complete(coro) + + +async def _never(): + """Models an untimed protocol call issued against a wedged page.""" + await asyncio.Event().wait() + + +class _Logger: + def debug(self, *a, **kw): + pass + + def warning(self, *a, **kw): + pass + + def error(self, *a, **kw): + pass + + +class _FakePage: + def __init__(self, content_behaviour, settle_ok=True): + # each entry is "hang", "navigating", or the HTML to return + self.behaviour = list(content_behaviour) + self.settle_ok = settle_ok + self.content_calls = 0 + self.settle_calls = 0 + + async def content(self): + self.content_calls += 1 + step = self.behaviour.pop(0) if self.behaviour else PAGE_HTML + if step == "hang": + await _never() + if step == "navigating": + raise PlaywrightError(NAVIGATING) + return step + + async def wait_for_load_state(self, state, timeout=None): + self.settle_calls += 1 + if not self.settle_ok: + raise PlaywrightTimeoutError(f"Timeout {timeout}ms exceeded") + + +def _strategy(): + """_capture_html needs only .logger — build without __init__ so the test + never touches BrowserManager or Playwright.""" + s = AsyncPlaywrightCrawlerStrategy.__new__(AsyncPlaywrightCrawlerStrategy) + s.logger = _Logger() + return s + + +# --- 1. bounded_evaluate --------------------------------------------------- + + +def test_bounded_evaluate_returns_the_value(): + async def main(): + async def ok(): + return {"width": 100} + + assert await bounded_evaluate(ok(), 5) == {"width": 100} + + _run(main()) + + +def test_bounded_evaluate_raises_instead_of_hanging(): + async def main(): + t0 = time.perf_counter() + with pytest.raises(PlaywrightTimeoutError) as exc: + await bounded_evaluate(_never(), 0.2) + assert time.perf_counter() - t0 < 3 + assert "navigating" in str(exc.value) + + _run(main()) + + +def test_bounded_evaluate_error_is_catchable_by_existing_handlers(): + """Call sites wrap evaluate in `except Exception` / `except Error`. The + bound must not escape either, or a cosmetic skip becomes a hard failure.""" + + async def main(): + try: + await bounded_evaluate(_never(), 0.1) + except Exception as e: + assert isinstance(e, PlaywrightError) + return + raise AssertionError("no exception raised") + + _run(main()) + + +def test_adapter_evaluate_honours_an_explicit_timeout(): + class HangingPage: + async def evaluate(self, expression, *a, **kw): + await _never() + + async def main(): + t0 = time.perf_counter() + with pytest.raises(PlaywrightTimeoutError): + await PlaywrightAdapter().evaluate(HangingPage(), "1+1", timeout=0.2) + assert time.perf_counter() - t0 < 3 + + _run(main()) + + +def test_adapter_default_timeout_is_finite(): + assert 0 < EVALUATE_TIMEOUT_S < 120 + assert 0 < acs.OPTIONAL_DOM_STEP_TIMEOUT_S <= EVALUATE_TIMEOUT_S + + +# --- 2. _capture_html ------------------------------------------------------ + + +def test_capture_succeeds_first_try(): + async def main(): + page = _FakePage([PAGE_HTML]) + assert await _strategy()._capture_html(page) == PAGE_HTML + assert page.content_calls == 1 + assert page.settle_calls == 0 + + _run(main()) + + +def test_capture_retries_after_the_navigation_race_and_succeeds(): + """The documented remedy for "the page is navigating and changing the + content" is to capture again once the navigation settles, not to throw the + whole crawl away.""" + + async def main(): + page = _FakePage(["navigating", PAGE_HTML]) + assert await _strategy()._capture_html(page) == PAGE_HTML + assert page.content_calls == 2 + assert page.settle_calls == 1 + + _run(main()) + + +def test_capture_gives_up_after_the_configured_attempts(): + async def main(): + page = _FakePage(["navigating"] * 10) + with pytest.raises(PlaywrightError): + await _strategy()._capture_html(page) + assert page.content_calls == acs.HTML_CAPTURE_ATTEMPTS + + _run(main()) + + +def test_capture_bails_out_when_the_page_cannot_settle(): + """A page that cannot even reach domcontentloaded is stuck, not between + documents — another attempt would only buy another full timeout.""" + + async def main(): + page = _FakePage(["navigating"] * 10, settle_ok=False) + with pytest.raises(PlaywrightError): + await _strategy()._capture_html(page) + assert page.content_calls == 1 + + _run(main()) + + +def test_capture_bounds_a_hanging_content_call(monkeypatch): + monkeypatch.setattr(acs, "HTML_CAPTURE_TIMEOUT_S", 0.2) + monkeypatch.setattr(acs, "HTML_CAPTURE_TOTAL_TIMEOUT_S", 0.5) + monkeypatch.setattr(acs, "HTML_CAPTURE_SETTLE_TIMEOUT_S", 0.1) + + async def main(): + page = _FakePage(["hang"] * 10) + t0 = time.perf_counter() + with pytest.raises(PlaywrightTimeoutError): + await _strategy()._capture_html(page) + # Bounded by the GROUP budget, not attempts x per-call timeout. + assert time.perf_counter() - t0 < 2.0 + + _run(main()) + + +# --- 3. total_timeout ------------------------------------------------------ + + +class _SlowStrategy: + def __init__(self, delay): + self.delay = delay + self.calls = 0 + + def update_user_agent(self, ua): + pass + + async def crawl(self, url, config=None, **kwargs): + self.calls += 1 + await asyncio.sleep(self.delay) + return AsyncCrawlResponse( + html=PAGE_HTML, + response_headers={}, + status_code=200, + redirected_status_code=200, + ) + + +async def _arun(strategy, **cfg): + crawler = AsyncWebCrawler(crawler_strategy=strategy) + crawler.ready = True + return await crawler.arun("https://example.com", config=CrawlerRunConfig(**cfg)) + + +def test_total_timeout_bounds_a_slow_attempt(): + async def main(): + strategy = _SlowStrategy(delay=5) + t0 = time.perf_counter() + result = await _arun(strategy, max_retries=1, total_timeout=400) + assert time.perf_counter() - t0 < 3 + assert result.success is False + # Attributable: the message names the budget, not just "failed". + assert "400 ms" in result.error_message + + _run(main()) + + +def test_total_timeout_is_shared_across_attempts_not_per_attempt(): + """Otherwise max_retries silently multiplies the caller's deadline.""" + + async def main(): + strategy = _SlowStrategy(delay=5) + t0 = time.perf_counter() + await _arun(strategy, max_retries=3, total_timeout=400) + assert time.perf_counter() - t0 < 3 + assert strategy.calls <= 2 + + _run(main()) + + +def test_total_timeout_defaults_to_off(): + async def main(): + assert CrawlerRunConfig().total_timeout is None + strategy = _SlowStrategy(delay=0.2) + result = await _arun(strategy, max_retries=0) + assert result.success is True + assert strategy.calls == 1 + + _run(main()) + + +def test_total_timeout_survives_clone_and_to_dict(): + cfg = CrawlerRunConfig(total_timeout=100000) + assert cfg.to_dict()["total_timeout"] == 100000 + assert cfg.clone().total_timeout == 100000 + assert CrawlerRunConfig.from_kwargs({"total_timeout": 123}).total_timeout == 123