From 315bcd5500118bd51243f821db006ef5656b752b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tero=20Vaalam=C3=A4ki?= Date: Thu, 30 Jul 2026 16:42:31 +0000 Subject: [PATCH 1/2] fix(antibot): detect blocks behind a redirect chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `CrawlResult.status_code` deliberately carries the FIRST hop of a redirect chain (#660, documented in docs/md_v2/api/crawl-result.md §1.3), while the HTML always comes from the LAST hop, kept separately as `redirected_status_code` (#1435, §1.4). `async_webcrawler` passes the first hop to `antibot_detector.is_blocked()` at all three call sites, so a 301 is judged instead of the 403 it led to — and no status rule in `is_blocked` can fire on a 3xx (429 / 403 / 503 / >=400 / ==200 all miss). Effect: any site that redirects (apex -> www, http -> https) and then serves a 4xx/5xx block page is returned as `success: True` with the block page as its content. Live example, unmodified v0.9.2: {"url": "https://konecranes.com", "success": true, "status_code": 301, "redirected_status_code": 403, "redirected_url": "https://www.konecranes.com/", "error_message": "", "markdown": "# Error 403 Forbidden ... Varnish cache server"} Two things make this a clear regression by omission rather than stale code: * `redirected_status_code` was merged 2026-02-06 (#1435); `antibot_detector.py` and this retry loop were added 8 days later (72b546c) still wired to `status_code`. The right field already existed. * `AsyncHTTPCrawlerStrategy` sets `status_code` to the post-redirect final status, so today the same URL yields a different block verdict depending on which crawler strategy runs. Fix: `antibot_detector.effective_status(status_code, redirected_status_code)`, used at the three `is_blocked` call sites. `status_code` itself is untouched — that is deliberate upstream semantics, pinned by tests/test_pr_1435_redirected_status_code.py, and callers may branch on it. `redirected_status_code` is None on non-HTTP paths (raw:, file://, js_only), so the helper falls back to `status_code` there. The change is strictly additive: redirect hops are 3xx by construction and no status rule matches a 3xx, so it can only add blocked verdicts, never remove one. Without a redirect, `status_code == redirected_status_code` and it is a literal no-op. Note this touches the same three lines as #2088 (`check_blocked` opt-out). The two are orthogonal — that PR gates the calls, this one fixes their argument — and I am happy to rebase on whichever lands first. --- crawl4ai/antibot_detector.py | 28 ++++ crawl4ai/async_webcrawler.py | 28 +++- tests/proxy/test_antibot_redirect_status.py | 153 ++++++++++++++++++++ 3 files changed, 205 insertions(+), 4 deletions(-) create mode 100644 tests/proxy/test_antibot_redirect_status.py diff --git a/crawl4ai/antibot_detector.py b/crawl4ai/antibot_detector.py index 228c1b258..a1dc57b12 100644 --- a/crawl4ai/antibot_detector.py +++ b/crawl4ai/antibot_detector.py @@ -188,6 +188,34 @@ def _structural_integrity_check(html: str) -> Tuple[bool, str]: return False, "" +def effective_status( + status_code: Optional[int], + redirected_status_code: Optional[int] = None, +) -> Optional[int]: + """ + The status code that actually produced the HTML we are holding. + + `AsyncCrawlResponse.status_code` / `CrawlResult.status_code` deliberately + carry the **first** hop of a redirect chain (a 3xx), while the body always + comes from the **last** hop; the final status is kept separately as + `redirected_status_code`. Block detection must be given the last hop — + otherwise a site that redirects apex -> www and then serves a 403 block + page is judged on its 301, no status rule fires, and the block page is + returned as successful content. + + `redirected_status_code` is None on non-HTTP paths (raw:, file://, + js_only), so fall back to `status_code` there. + + Args: + status_code: First hop of the redirect chain (or the only response). + redirected_status_code: Final hop, when the request was redirected. + + Returns: + The status code to judge the response body by. + """ + return redirected_status_code or status_code + + def is_blocked( status_code: Optional[int], html: str, diff --git a/crawl4ai/async_webcrawler.py b/crawl4ai/async_webcrawler.py index 8216d19bc..99d5befad 100644 --- a/crawl4ai/async_webcrawler.py +++ b/crawl4ai/async_webcrawler.py @@ -52,7 +52,7 @@ compute_head_fingerprint, ) from .cache_validator import CacheValidator, CacheValidationResult -from .antibot_detector import is_blocked +from .antibot_detector import is_blocked, effective_status class AsyncWebCrawler: @@ -509,8 +509,17 @@ async def arun( _blocked = False _block_reason = "" else: + # Judge the body by the status that produced + # it — the LAST hop. status_code holds the + # first hop (a 3xx on any redirect), which no + # status rule in is_blocked() can ever match. _blocked, _block_reason = is_blocked( - async_response.status_code, html) + effective_status( + async_response.status_code, + async_response.redirected_status_code, + ), + html, + ) _crawl_stats["proxies_used"].append({ "proxy": _proxy.server if _proxy else None, @@ -554,7 +563,13 @@ async def arun( if _fallback_fn and not _done and not _is_raw_url: _needs_fallback = ( crawl_result is None # All proxies threw exceptions - or is_blocked(crawl_result.status_code, crawl_result.html or "")[0] + or is_blocked( + effective_status( + crawl_result.status_code, + crawl_result.redirected_status_code, + ), + crawl_result.html or "", + )[0] ) if _needs_fallback: self.logger.warning( @@ -627,7 +642,12 @@ async def arun( _has_download = bool(getattr(crawl_result, "downloaded_files", None)) if not _fallback_succeeded and not _is_raw_url and not _has_download: _blocked, _block_reason = is_blocked( - crawl_result.status_code, crawl_result.html or "") + effective_status( + crawl_result.status_code, + crawl_result.redirected_status_code, + ), + crawl_result.html or "", + ) if _blocked: crawl_result.success = False crawl_result.error_message = f"Blocked by anti-bot protection: {_block_reason}" diff --git a/tests/proxy/test_antibot_redirect_status.py b/tests/proxy/test_antibot_redirect_status.py new file mode 100644 index 000000000..9a29412bd --- /dev/null +++ b/tests/proxy/test_antibot_redirect_status.py @@ -0,0 +1,153 @@ +""" +Block detection must judge the FINAL hop of a redirect chain. + +`CrawlResult.status_code` deliberately carries the first hop of a redirect +chain (issue #660, documented in docs/md_v2/api/crawl-result.md §1.3), while +the HTML always comes from the last hop, kept separately as +`redirected_status_code` (PR #1435, §1.4). `async_webcrawler` used to hand the +first hop to `antibot_detector.is_blocked()`, so a 301 was judged instead of +the 403 it led to — and no status rule in `is_blocked` can fire on a 3xx. + +Effect: any site that redirects (apex -> www, http -> https) and then serves a +4xx/5xx block page was returned as `success: True` with the block page as its +content. Note `AsyncHTTPCrawlerStrategy` already sets `status_code` to the +post-redirect status, so the same URL produced different verdicts depending on +which crawler strategy ran. + +No browser or network needed. + + pytest tests/proxy/test_antibot_redirect_status.py -q +""" + +import asyncio + +from crawl4ai import AsyncWebCrawler, CrawlerRunConfig +from crawl4ai.antibot_detector import effective_status, is_blocked +from crawl4ai.models import AsyncCrawlResponse + +BLOCK_PAGE = ( + "403 Forbidden" + "

Error 403 Forbidden

Forbidden

Error 54113

" + "

Details: cache-ams-eham8680049-AMS

Varnish cache server

" + "" +) + +REAL_PAGE = ( + "Example Co" + "

Example Co

Contact: info@example.com

" + "

Phone +1 555 0100. Address: 1 Example Street, Springfield.

" + "" + "

Example Co is a company that does example things for example people.

" + "" +) + + +class _FakeStrategy: + """Returns a canned AsyncCrawlResponse; counts page loads.""" + + def __init__(self, response): + self.response = response + self.calls = 0 + + def update_user_agent(self, ua): + pass + + async def crawl(self, url, config=None, **kwargs): + self.calls += 1 + return self.response + + +def _run(coro): + return asyncio.get_event_loop_policy().new_event_loop().run_until_complete(coro) + + +def _response(html, status_code, redirected_status_code=None, redirected_url=None): + return AsyncCrawlResponse( + html=html, + response_headers={}, + status_code=status_code, + redirected_status_code=redirected_status_code, + redirected_url=redirected_url, + ) + + +async def _crawl(response, url="https://example.com", max_retries=1): + strategy = _FakeStrategy(response) + crawler = AsyncWebCrawler(crawler_strategy=strategy) + crawler.ready = True # skip start(): no browser is launched + result = await crawler.arun(url, config=CrawlerRunConfig(max_retries=max_retries)) + return result, strategy + + +def test_effective_status_prefers_the_final_hop(): + assert effective_status(301, 403) == 403 + assert effective_status(302, 200) == 200 + + +def test_effective_status_falls_back_without_a_redirect(): + # raw:, file:// and js_only paths leave redirected_status_code as None. + assert effective_status(200, None) == 200 + assert effective_status(None, None) is None + assert effective_status(403, 403) == 403 + + +def test_block_page_behind_a_redirect_is_detected(): + async def main(): + result, _ = await _crawl( + _response(BLOCK_PAGE, 301, 403, "https://www.example.com/") + ) + assert result.success is False + assert result.error_message.startswith("Blocked by anti-bot protection:") + assert "403" in result.error_message + # status_code semantics are unchanged: still the first hop. + assert result.status_code == 301 + assert result.redirected_status_code == 403 + + _run(main()) + + +def test_block_page_without_a_redirect_still_detected(): + async def main(): + result, _ = await _crawl(_response(BLOCK_PAGE, 403, 403)) + assert result.success is False + assert "Blocked by anti-bot protection:" in result.error_message + + _run(main()) + + +def test_benign_redirect_is_not_a_false_positive(): + async def main(): + result, strategy = await _crawl( + _response(REAL_PAGE, 301, 200, "https://www.example.com/"), + url="http://example.com", + ) + assert result.success is True + assert not result.error_message + assert strategy.calls == 1 # no retry burned on a good page + + _run(main()) + + +def test_no_redirect_is_a_no_op(): + async def main(): + result, strategy = await _crawl(_response(REAL_PAGE, 200, 200)) + assert result.success is True + assert strategy.calls == 1 + + _run(main()) + + +def test_raw_url_still_skips_block_detection(): + async def main(): + result, _ = await _crawl( + _response(BLOCK_PAGE, 200, None), url="raw:" + BLOCK_PAGE + ) + assert result.success is True + + _run(main()) + + +def test_detector_itself_is_unchanged(): + # Only the argument changed; is_blocked's own rules are untouched. + assert is_blocked(301, BLOCK_PAGE)[0] is False + assert is_blocked(403, BLOCK_PAGE)[0] is True From 03b51a0c2e7730228032a951ded59d6d65db34bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tero=20Vaalam=C3=A4ki?= Date: Thu, 30 Jul 2026 16:43:35 +0000 Subject: [PATCH 2/2] docs: note that block detection uses the final redirect hop --- docs/md_v2/advanced/anti-bot-and-fallback.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/md_v2/advanced/anti-bot-and-fallback.md b/docs/md_v2/advanced/anti-bot-and-fallback.md index 68ed03de8..d498266ca 100644 --- a/docs/md_v2/advanced/anti-bot-and-fallback.md +++ b/docs/md_v2/advanced/anti-bot-and-fallback.md @@ -13,6 +13,8 @@ After each crawl attempt, Crawl4AI inspects the HTTP status code and HTML conten Detection uses structural HTML markers (specific element IDs, script sources, form actions) rather than generic keywords to minimize false positives. A normal page that happens to mention "CAPTCHA" or "Cloudflare" in its content will not be flagged. +**Redirects:** the status code used for detection is the **final** hop of the redirect chain — `redirected_status_code`, falling back to `status_code` when there was no redirect. `CrawlResult.status_code` deliberately reports the *first* hop (see [CrawlResult](../api/crawl-result.md) §1.3/1.4), so a site that redirects `example.com → www.example.com` and then serves a `403` block page is correctly flagged even though `status_code` is `301`. + When all attempts fail and blocking is still detected, the result is returned with `success=False` and `error_message` describing the block reason. ## Configuration Options