Skip to content

fix(crawler): bound the Playwright calls that carry no protocol timeout #2113

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
Aitosoft wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

fix(crawler): bound the Playwright calls that carry no protocol timeout #2113

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions crawl4ai/async_configs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1436,6 +1436,14 @@ class CrawlerRunConfig():
Default: "domcontentloaded".
page_timeout (int): Timeout in ms for page operations like navigation.
Default: 60000 (60 seconds).
total_timeout (int or None): Hard ceiling in ms for the whole fetch phase of a
single arun() — every anti-bot attempt and proxy retry
share this one budget. page_timeout only bounds
navigation and the wait_* family; a page that keeps
renavigating can wedge an untimed protocol call and
burn an unbounded amount of wall clock. Set this when
the caller has a real deadline. None disables it.
Default: None.
wait_for (str or None): A CSS selector or JS condition to wait for before extracting content.
Default: None.
wait_for_timeout (int or None): Specific timeout in ms for the wait_for condition.
Expand Down Expand Up @@ -1627,6 +1635,7 @@ def __init__(
# Page Navigation and Timing Parameters
wait_until: str = "domcontentloaded",
page_timeout: int = PAGE_TIMEOUT,
total_timeout: Optional[int] = None,
wait_for: str = None,
wait_for_timeout: int = None,
wait_for_images: bool = False,
Expand Down Expand Up @@ -1756,6 +1765,7 @@ def __init__(
# Page Navigation and Timing Parameters
self.wait_until = wait_until
self.page_timeout = page_timeout
self.total_timeout = total_timeout
self.wait_for = wait_for
self.wait_for_timeout = wait_for_timeout
self.wait_for_images = wait_for_images
Expand Down Expand Up @@ -2126,6 +2136,7 @@ def to_dict(self):
"shared_data": self.shared_data,
"wait_until": self.wait_until,
"page_timeout": self.page_timeout,
"total_timeout": self.total_timeout,
"wait_for": self.wait_for,
"wait_for_timeout": self.wait_for_timeout,
"wait_for_images": self.wait_for_images,
Expand Down
122 changes: 111 additions & 11 deletions crawl4ai/async_crawler_strategy.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,12 @@
from .ssl_certificate import SSLCertificate
from .user_agent_generator import ValidUAGenerator, UAGen
from .browser_manager import BrowserManager
from .browser_adapter import BrowserAdapter, PlaywrightAdapter, UndetectedAdapter
from .browser_adapter import (
EVALUATE_TIMEOUT_S,
BrowserAdapter,
PlaywrightAdapter,
UndetectedAdapter,
)

import aiofiles
import aiohttp
Expand All @@ -33,6 +38,22 @@
import contextlib
from functools import partial

# --- Bounds for Playwright calls that carry no protocol timeout ------------
# page.content() and page.evaluate() are sent without a `timeout` field, so the
# driver arms no timer and they can only end when the target replies or closes.
# See AsyncPlaywrightCrawlerStrategy._capture_html and browser_adapter.
HTML_CAPTURE_TIMEOUT_S: Final[float] = 15.0 # per page.content() attempt
HTML_CAPTURE_TOTAL_TIMEOUT_S: Final[float] = 25.0 # across all attempts
HTML_CAPTURE_SETTLE_TIMEOUT_S: Final[float] = 5.0 # wait for the next document
HTML_CAPTURE_ATTEMPTS: Final[int] = 3
PAGE_CLOSE_TIMEOUT_S: Final[float] = 10.0 # page.close() is unbounded too
VIRTUAL_SCROLL_TIMEOUT_S: Final[float] = 300.0 # in-page scroll loop, legitimately slow
# Cosmetic DOM steps that already degrade gracefully (image dimensions, consent
# and overlay removal). They are worth a few seconds, never worth a minute, so
# they get a tighter ceiling than the adapter default.
OPTIONAL_DOM_STEP_TIMEOUT_S: Final[float] = 10.0


class AsyncCrawlerStrategy(ABC):
"""
Abstract base class for crawler strategies.
Expand Down Expand Up @@ -333,7 +354,12 @@ async def csp_compliant_wait(
"""

try:
result = await self.adapter.evaluate(page, wrapper_js)
# The polling loop above enforces `timeout` itself; the adapter
# bound only has to catch a page whose execution context never
# settles, so give it headroom rather than racing the JS.
result = await self.adapter.evaluate(
page, wrapper_js, timeout=timeout / 1000.0 + EVALUATE_TIMEOUT_S
)
return result
except Exception as e:
if "Error evaluating condition" in str(e):
Expand Down Expand Up @@ -511,6 +537,66 @@ async def crawl(
"URL must start with 'http://', 'https://', 'file://', or 'raw:'"
)

async def _capture_html(self, page: Page, attempts: int = None) -> str:
"""Capture the page HTML, tolerating a page that is still navigating.

`page.content()` carries no protocol timeout — the Python client sends
no timeout field, so the driver arms no timer and the call waits on the
frame's execution-context promise. On a page that keeps committing
navigations that promise is repeatedly replaced, giving two failure
modes for the same race:

* the context dies *during* the call -> Playwright raises
"Unable to retrieve content because the page is navigating and
changing the content";
* the context is already gone *at* the call -> it blocks forever
(`page_timeout` does not cover this; only an external deadline does).

Both are transient. Wait for the new document to reach
domcontentloaded and capture again, which is the documented remedy and
is far cheaper than re-running the whole crawl.
"""
attempts = attempts or HTML_CAPTURE_ATTEMPTS
# A recoverable race raises immediately, so retries are nearly free; a
# wedged page burns a full timeout each time. Bound the retries as a
# group so only the wedged case pays, and it pays once.
deadline = time.perf_counter() + HTML_CAPTURE_TOTAL_TIMEOUT_S
last_err: Optional[BaseException] = None
for _i in range(attempts):
budget = min(HTML_CAPTURE_TIMEOUT_S, deadline - time.perf_counter())
if budget <= 0:
break
try:
return await asyncio.wait_for(page.content(), budget)
except asyncio.TimeoutError:
last_err = PlaywrightTimeoutError(
f"page.content() did not return within {budget:.0f}s "
f"— the page never stopped navigating"
)
except Error as e:
last_err = e
if _i >= attempts - 1:
break
self.logger.debug(
message="HTML capture attempt {n} failed ({err}) — letting the page settle",
tag="SCRAPE",
params={"n": _i + 1, "err": str(last_err)[:120]},
)
try:
await page.wait_for_load_state(
"domcontentloaded",
timeout=HTML_CAPTURE_SETTLE_TIMEOUT_S * 1000,
)
except Exception:
# The page cannot even reach domcontentloaded, so it is not
# between documents — it is stuck. Another capture attempt
# would only buy another full timeout. Give up now.
break
raise last_err or PlaywrightTimeoutError(
f"page.content() could not be captured within "
f"{HTML_CAPTURE_TOTAL_TIMEOUT_S:g}s"
)

async def _crawl_web(
self, url: str, config: CrawlerRunConfig
) -> AsyncCrawlResponse:
Expand Down Expand Up @@ -1029,7 +1115,10 @@ async def handle_request_failed_capture(request):
await page.wait_for_load_state("domcontentloaded", timeout=5)
except PlaywrightTimeoutError:
pass
await self.adapter.evaluate(page, update_image_dimensions_js)
await self.adapter.evaluate(
page, update_image_dimensions_js,
timeout=OPTIONAL_DOM_STEP_TIMEOUT_S,
)
except Exception as e:
self.logger.error(
message="Error updating image dimensions: {error}",
Expand Down Expand Up @@ -1061,7 +1150,7 @@ async def handle_request_failed_capture(request):
message="Shadow DOM flattening returned no content, falling back to page.content()",
tag="SCRAPE",
)
html = await page.content()
html = await self._capture_html(page)
elif config.css_selector:
try:
selectors = [s.strip() for s in config.css_selector.split(',')]
Expand All @@ -1082,7 +1171,7 @@ async def handle_request_failed_capture(request):
except Error as e:
raise RuntimeError(f"Failed to extract HTML content: {str(e)}")
else:
html = await page.content()
html = await self._capture_html(page)

await self.execute_hook(
"before_return_html", page=page, html=html, context=context, config=config
Expand Down Expand Up @@ -1126,7 +1215,7 @@ async def get_delayed_content(delay: float = 5.0) -> str:
params={"delay": delay, "url": url},
)
await asyncio.sleep(delay)
return await page.content()
return await self._capture_html(page)

# For undetected browsers, retrieve console messages before returning
if config.capture_console_messages and hasattr(self.adapter, 'retrieve_console_messages'):
Expand Down Expand Up @@ -1196,7 +1285,11 @@ async def get_delayed_content(delay: float = 5.0) -> str:
all_contexts = page.context.browser.contexts
total_pages = sum(len(context.pages) for context in all_contexts)
if not (total_pages <= 1 and (self.browser_config.use_managed_browser or self.browser_config.headless)):
await page.close()
# page.close() is also sent without a timeout and waits
# on the target's closed-promise, so a wedged renderer
# can block cleanup indefinitely — including while this
# coroutine is being cancelled by an outer deadline.
await asyncio.wait_for(page.close(), PAGE_CLOSE_TIMEOUT_S)
except Exception:
pass

Expand Down Expand Up @@ -1428,8 +1521,13 @@ async def _handle_virtual_scroll(self, page: Page, config: "VirtualScrollConfig"
}
"""

# Execute virtual scroll capture
result = await self.adapter.evaluate(page, virtual_scroll_js, config.to_dict())
# Execute virtual scroll capture. Unlike the other evaluates this
# one legitimately runs a long scroll loop inside the page, so it
# gets its own generous ceiling rather than the adapter default.
result = await self.adapter.evaluate(
page, virtual_scroll_js, config.to_dict(),
timeout=VIRTUAL_SCROLL_TIMEOUT_S,
)

if result.get("replaced", False):
self.logger.success(
Expand Down Expand Up @@ -1532,7 +1630,8 @@ async def remove_overlay_elements(self, page: Page) -> None:
}};
}}
}})()
"""
""",
timeout=OPTIONAL_DOM_STEP_TIMEOUT_S,
)
await page.wait_for_timeout(500) # Wait for any animations to complete
except Exception as e:
Expand Down Expand Up @@ -1576,7 +1675,8 @@ async def remove_consent_popups(self, page: Page) -> None:
}};
}}
}})()
"""
""",
timeout=OPTIONAL_DOM_STEP_TIMEOUT_S,
)
await page.wait_for_timeout(500) # Wait for any animations to complete
except Exception as e:
Expand Down
33 changes: 31 additions & 2 deletions crawl4ai/async_webcrawler.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -403,6 +403,18 @@ async def arun(
_is_raw_url = url.startswith("raw:") or url.startswith("raw://")

_max_attempts = 1 + getattr(config, "max_retries", 0)
# One shared budget for the whole fetch phase (every attempt
# and every proxy). page_timeout only bounds navigation and
# the wait_* family; Playwright calls sent without a timeout
# (page.content, page.evaluate, page.close) are not covered by
# anything, so without this a single wedged page can consume
# the caller's entire deadline in silence.
_total_timeout = getattr(config, "total_timeout", None)
_fetch_deadline = (
time.perf_counter() + _total_timeout / 1000.0
if _total_timeout
else None
)
_proxy_list = config._get_proxy_list()
_original_proxy_config = config.proxy_config
_block_reason = ""
Expand Down Expand Up @@ -456,8 +468,25 @@ async def arun(
self.crawler_strategy.update_user_agent(
config.user_agent)

async_response = await self.crawler_strategy.crawl(
url, config=config)
_remaining = None
if _fetch_deadline is not None:
_remaining = _fetch_deadline - time.perf_counter()
if _remaining <= 0:
raise TimeoutError(
f"Fetch budget of {_total_timeout} ms exhausted "
f"before attempt {_attempt + 1}/{_max_attempts}")
try:
async_response = await asyncio.wait_for(
self.crawler_strategy.crawl(url, config=config),
timeout=_remaining,
)
except asyncio.TimeoutError:
# asyncio.TimeoutError carries no message —
# name the budget so the failure is attributable.
raise TimeoutError(
f"Crawl attempt exceeded the {_total_timeout} ms "
f"fetch budget ({_remaining:.1f}s remained for "
f"attempt {_attempt + 1}/{_max_attempts})") from None

html = sanitize_input_encode(async_response.html)
screenshot_data = async_response.screenshot
Expand Down
Loading