diff --git a/crawl4ai/content_filter_strategy.py b/crawl4ai/content_filter_strategy.py index ab99a793c..a34637fd3 100644 --- a/crawl4ai/content_filter_strategy.py +++ b/crawl4ai/content_filter_strategy.py @@ -712,6 +712,14 @@ def _prune_tree(self, node): if self._is_preserved(node): return + # Skip pruning inside 
/ blocks where whitespace is significant.
+        # Syntax highlighters wrap every token in  (including whitespace-only
+        # spans like  ) which score far below any threshold
+        # and get decomposed, corrupting code. Mirrors the scraper-side guard in
+        # WebScrapingStrategy.remove_empty_elements_fast (#1181).
+        if node.name in ("pre", "code"):
+            return
+
         text_len = len(node.get_text(strip=True))
         tag_len = len(node.encode_contents().decode("utf-8"))
         link_text_len = sum(
diff --git a/tests/test_pruning_code_whitespace.py b/tests/test_pruning_code_whitespace.py
new file mode 100644
index 000000000..c5e8f1223
--- /dev/null
+++ b/tests/test_pruning_code_whitespace.py
@@ -0,0 +1,78 @@
+"""
+PruningContentFilter must not prune inside 
/ blocks.
+
+Syntax highlighters (pygments, prism, rouge) wrap every token in ,
+including whitespace-only spans like  . Those spans
+score ~0.26-0.36 against the default 0.48 threshold and were decomposed,
+collapsing `import torch` to `importtorch` — the fit_markdown counterpart of
+the scraper-side bug fixed in #1181. Heavily-highlighted 
 blocks (huge
+tag_len, low text density) could also be dropped wholesale.
+"""
+import pytest
+from crawl4ai.content_filter_strategy import PruningContentFilter
+
+
+# Pygments-style highlighting: every token in a span, whitespace in class="w".
+HIGHLIGHTED_CODE_HTML = """
+
+

+    
Using PyTorch

+    
This paragraph carries enough real prose to pass the pruning threshold
+    comfortably. It explains how to import the library and configure a device
+    before running the model, so the surrounding article content is retained
+    by the filter as fully relevant text.

+    
import torch
+import triton.language as tl
+FROM golang:1.21 AS builder

+    
Another paragraph of real prose after the code block, long enough to be
+    kept on its own merits, describing what the snippet above actually does in
+    the larger training pipeline.

+

+
+
+"""
+
+
+def _filtered_html(html: str, **kwargs) -> str:
+    chunks = PruningContentFilter(**kwargs).filter_content(html)
+    return "\n".join(chunks)
+
+
+def test_whitespace_spans_inside_code_survive():
+    out = _filtered_html(HIGHLIGHTED_CODE_HTML)
+    # The whitespace-only spans must still separate the tokens.
+    assert ' ' in out
+    assert "importtorch" not in out.replace("", "").replace(
+        '', ""
+    ).replace('', "").replace('', "")
+
+
+def test_highlighted_pre_block_not_dropped():
+    out = _filtered_html(HIGHLIGHTED_CODE_HTML)
+    # The heavily-tagged 
 (low text density) must be kept wholesale.
+    assert "
" in out
+    assert "triton.language" in out
+
+
+def test_code_guard_does_not_disable_normal_pruning():
+    out = _filtered_html(HIGHLIGHTED_CODE_HTML)
+    # Boilerplate outside code blocks is still pruned.
+    assert "/docs" not in out
+
+
+def test_rendered_text_keeps_token_spacing():
+    from bs4 import BeautifulSoup
+
+    out = _filtered_html(HIGHLIGHTED_CODE_HTML)
+    pre = BeautifulSoup(out, "html.parser").find("pre")
+    assert pre is not None
+    text = pre.get_text()
+    assert "import torch" in text
+    assert "FROM golang:1.21 AS builder" in text
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])