From 55adcac3e694da08498525bd6db43bb33abf32ce Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tero=20Vaalam=C3=A4ki?= <tero.vaalamaki@aitosoft.com>
Date: Thu, 30 Jul 2026 17:57:29 +0000
Subject: [PATCH] fix(scraping): a nested <noscript> discards the entire page
 body
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

`<noscript>` may not nest. With scripting enabled its content is raw text, so
the parser never sees an inner `<noscript>` as an element and the OUTER element
is left unclosed — libxml2 then swallows every following node into it. Since
`LXMLWebScrapingStrategy._scrap` hands the raw serialised DOM straight to
`lhtml.document_fromstring`, the damage is done before there is a tree to fix.

A single WordPress lazy-load plugin that wraps the Google Tag Manager block in
a second `<noscript>` is enough to lose a whole site:

    312,628 B rendered HTML  ->  97 B cleaned_html  ->  1 B markdown

with `status_code: 200` and `success: True`, so nothing downstream can tell.
`cleaned_html` is literally `<html><head><title>…</title></head></html>` — the
`<body>` is gone. I found this across 70 hosts in one corpus; it reproduced
byte-identically on the same URLs 3.5 months apart, because the plugin markup is
stable.

Six-line reproduction on unmodified v0.9.2:

    from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy
    MIN = '''<html><head><title>T</title></head><body>
    <noscript><iframe src="about:blank"></iframe><noscript><iframe src="about:blank"></iframe></noscript>
    <h1>Contact</h1><p>Phone 010 123 4567</p>
    <div><p>A second paragraph.</p></div>
    </body></html>'''
    LXMLWebScrapingStrategy().scrap('https://x/', MIN, word_count_threshold=1).cleaned_html
    # nested:            42 B  -> '<html><head><title>T</title></head></html>'
    # single <noscript>: 187 B  (well-formed noscript is harmless)
    # no <noscript>:     188 B

Fix: `strip_noscript()`, applied at the top of `_scrap()` immediately before
parsing. Removal rather than unwrapping, because Crawl4AI renders with
JavaScript enabled — by definition `<noscript>` content is not what the page
displayed, and its scripted equivalent is already in the DOM. Two passes:
well-formed elements first (a non-greedy match ends at the first `</noscript>`,
which is where the browser's parser ends the outer element too), then any
unpaired tag left by truncated markup, so an unclosed `<noscript>` cannot
swallow a body either. Pages without the substring return the same object
unparsed, so the cost is one `in` check.

`grep noscript crawl4ai/content_scraping_strategy.py crawl4ai/utils.py` returns
nothing on develop — there is no existing `<noscript>` handling to conflict with.

Tests: tests/general/test_noscript_body_loss.py, 11 passed. Covers nested,
well-formed, unclosed, uppercase and attributed shapes, asserts nested and
well-formed become indistinguishable, and pins that a page whose only content
is inside `<noscript>` correctly stays empty.

    pytest tests/general/test_noscript_body_loss.py -q          # 11 passed
    pytest tests/test_pr_1435_redirected_status_code.py -q      #  7 passed
---
 crawl4ai/content_scraping_strategy.py    |  39 +++++++++
 tests/general/test_noscript_body_loss.py | 100 +++++++++++++++++++++++
 2 files changed, 139 insertions(+)
 create mode 100644 tests/general/test_noscript_body_loss.py
diff --git a/crawl4ai/content_scraping_strategy.py b/crawl4ai/content_scraping_strategy.py
index 67e87250d..8884be344 100644
--- a/crawl4ai/content_scraping_strategy.py
+++ b/crawl4ai/content_scraping_strategy.py
@@ -37,6 +37,41 @@
 TWITTER_REGEX = re.compile(r"^twitter:")
 DIMENSION_REGEX = re.compile(r"(\d+)(\D*)")
 
+# <noscript> removal — see strip_noscript() below.
+NOSCRIPT_ELEMENT_REGEX = re.compile(
+    r"<noscript\b[^>]*>[\s\S]*?</noscript\s*>", re.IGNORECASE
+)
+NOSCRIPT_TAG_REGEX = re.compile(r"</?noscript\b[^>]*>", re.IGNORECASE)
+
+
+def strip_noscript(html: str) -> str:
+    """Remove ``<noscript>`` elements from raw HTML *before* it is parsed.
+
+    Why this has to happen pre-parse: ``<noscript>`` may not nest. With
+    scripting enabled its content is raw text, so a parser never sees an inner
+    ``<noscript>`` as an element and the *outer* element is left unclosed —
+    every following node is swallowed into it and dropped. A single WordPress
+    lazy-load plugin that wraps the Google Tag Manager block in a second
+    ``<noscript>`` therefore costs the entire page body:
+
+        312,628 B of rendered HTML -> 97 B of cleaned_html -> 1 B of markdown
+
+    measured on https://www.kiertopakkaus.fi/ (2026-07-30). The damage is done
+    by the time lxml returns a tree, so it cannot be repaired downstream.
+
+    Removal (rather than unwrapping) is correct because Crawl4AI renders with
+    JavaScript enabled: by definition ``<noscript>`` content is *not* what the
+    page showed, and its scripted equivalent is already in the DOM.
+
+    Two passes: well-formed elements first (a non-greedy match stops at the
+    first ``</noscript>``, which is exactly where the browser's parser ends the
+    outer element too), then any unpaired tag left over by truncated or
+    malformed markup, so an unclosed ``<noscript>`` can never swallow a body.
+    """
+    if not html or "<noscript" not in html.lower():
+        return html
+    return NOSCRIPT_TAG_REGEX.sub("", NOSCRIPT_ELEMENT_REGEX.sub("", html))
+
 
 # Function to parse srcset
 def parse_srcset(s: str) -> List[Dict]:
@@ -624,6 +659,10 @@ def _scrap(
         if not html:
             return None
 
+        # Malformed/nested <noscript> makes libxml2 swallow the rest of the
+        # document. Must be removed before parsing — see strip_noscript().
+        html = strip_noscript(html)
+
         success = True
         try:
             doc = lhtml.document_fromstring(html)
diff --git a/tests/general/test_noscript_body_loss.py b/tests/general/test_noscript_body_loss.py
new file mode 100644
index 000000000..b5cd9e168
--- /dev/null
+++ b/tests/general/test_noscript_body_loss.py
@@ -0,0 +1,100 @@
+"""Regression tests for nested/malformed <noscript> discarding the page body.
+
+`<noscript>` may not nest. With scripting enabled its content is raw text, so
+a parser never sees an inner `<noscript>` as an element and the *outer* one is
+left unclosed — libxml2 then swallows every following node. A WordPress
+lazy-load plugin that wraps the Google Tag Manager block in a second
+`<noscript>` is enough to lose an entire page:
+
+    312,628 B of rendered HTML -> 97 B of cleaned_html -> 1 B of markdown
+
+at HTTP 200 with success=True, which makes it silent.
+
+Run:  pytest tests/general/test_noscript_body_loss.py -q
+"""
+
+import pytest
+
+from crawl4ai.content_scraping_strategy import LXMLWebScrapingStrategy, strip_noscript
+
+PAGE = """<html><head><title>T</title></head><body>
+{ns}
+<h1>Contact</h1><p>Phone 010 123 4567, email info@example.com</p>
+<div><p>A second paragraph of real content.</p></div>
+</body></html>"""
+
+IFRAME = '<iframe src="about:blank"></iframe>'
+
+# The shape emitted by the plugin: the inner element consumes the closing tag,
+# so the outer <noscript> is never closed.
+NESTED = f"<noscript>{IFRAME}<noscript>{IFRAME}</noscript>"
+SINGLE = f"<noscript>{IFRAME}</noscript>"
+UNCLOSED = f"<noscript>{IFRAME}"
+UPPERCASE = f"<NOSCRIPT>{IFRAME}<NOSCRIPT>{IFRAME}</NOSCRIPT>"
+WITH_ATTRS = f'<noscript data-lazyloaded="1">{IFRAME}<noscript>{IFRAME}</noscript>'
+
+
+def cleaned(html):
+    return (
+        LXMLWebScrapingStrategy()
+        .scrap("https://example.com/", html, word_count_threshold=1)
+        .cleaned_html
+    )
+
+
+@pytest.mark.parametrize(
+    "region",
+    [NESTED, SINGLE, UNCLOSED, UPPERCASE, WITH_ATTRS, ""],
+    ids=["nested", "single", "unclosed", "uppercase", "with-attrs", "absent"],
+)
+def test_body_survives_every_noscript_shape(region):
+    out = cleaned(PAGE.format(ns=region))
+    assert "Contact" in out
+    assert "info@example.com" in out
+    assert "A second paragraph" in out
+
+
+def test_nested_and_single_are_indistinguishable():
+    """Before the fix, `nested` produced
+    '<html><head><title>T</title></head></html>' (42 B) while `single`
+    produced the full 187 B document."""
+    assert cleaned(PAGE.format(ns=NESTED)) == cleaned(PAGE.format(ns=SINGLE))
+
+
+def test_noscript_only_page_is_still_empty():
+    """Not a regression: Crawl4AI renders with JavaScript enabled, so
+    <noscript> content is by definition not what the page displayed."""
+    out = cleaned(
+        "<html><head><title>T</title></head><body>"
+        "<noscript><p>Enable JavaScript</p></noscript></body></html>"
+    )
+    assert "Enable JavaScript" not in out
+
+
+def test_strip_noscript_is_a_noop_without_noscript():
+    html = "<html><body><p>hello</p></body></html>"
+    assert strip_noscript(html) is html
+    assert strip_noscript("") == ""
+
+
+def test_strip_noscript_removes_element_and_contents():
+    assert strip_noscript("<p>a</p><noscript><img src=x></noscript><p>b</p>") == (
+        "<p>a</p><p>b</p>"
+    )
+
+
+def test_realistic_gtm_lazyload_markup():
+    html = (
+        "<html><head><title>Home</title></head><body>"
+        '<noscript><iframe data-lazyloaded="1" src="about:blank" '
+        'data-src="https://www.googletagmanager.com/ns.html?id=GTM-XXXX"></iframe>'
+        '<noscript><iframe src="https://www.googletagmanager.com/ns.html?id=GTM-XXXX">'
+        "</iframe></noscript>"
+        '<a class="skip-link" href="#content">Skip to content</a>'
+        '<div id="page"><h1>Example Oy</h1><p>We sell packaging. Call 010 000 0000.</p>'
+        "</div></body></html>"
+    )
+    out = cleaned(html)
+    assert "Example Oy" in out
+    assert "Skip to content" in out
+    assert "010 000 0000" in out
