diff --git a/contexts/design/flow/news.md b/contexts/design/flow/news.md index a6f0edd..b04e2c0 100644 --- a/contexts/design/flow/news.md +++ b/contexts/design/flow/news.md @@ -31,7 +31,7 @@ The primary requirement is that any caller can request a complete, one-shot coll ## Design Principles 1. **Ask for news, not fetch details.** Callers choose a source and time window. They do not choose RSS, listing pages, how to move between pages, or article parsing rules. -2. **Return every source row.** QuantMind does not silently remove duplicate source rows. Repeated rows remain repeated output records, and may share the same stable ID. +2. **Return every source row for complete windows.** QuantMind does not silently remove duplicate source rows. Repeated rows remain repeated output records, and may share the same stable ID. An incomplete listing scan returns discovery evidence without fetching partial article bodies. 3. **Show partial failures.** The returned batch includes item failures. Invalid inputs still raise before network work starts. 4. **Hide source implementation details.** PR Newswire may later use a public mechanism other than listing pages without changing the public function. 5. **List supported sources explicitly.** The first version selects from a closed source list. It does not expose a provider plugin API or registry. @@ -97,6 +97,7 @@ NewsWindow -> select the source collector -> scan public listing pages from newest to oldest -> keep rows inside [start, end) + -> stop before article fetches when listing coverage is incomplete -> fetch each linked article -> convert HTML to Markdown -> NewsDocument or NewsFailure @@ -124,6 +125,8 @@ After collection begins, one item failure does not stop independent items. Each - a listing page could not be fetched or parsed; - the listing scan stopped before crossing the window start. +An incomplete listing scan returns its discovery failures and observed row count without fetching article bodies. Its documents are empty because the caller cannot treat the partial listing as complete coverage; a completeness-preserving caller can retry narrower windows without downloading the same parent-window articles again. + Article failures stay in `failures` but do not change whether all listing rows were found. A caller can distinguish "the full time window was scanned" from "every found article was processed." It can store successful records and retry failed articles separately. An empty batch is complete only when the listing scan crossed the requested start. ## Who Owns What diff --git a/quantmind/preprocess/pr_newswire.py b/quantmind/preprocess/pr_newswire.py index 38ed0af..ee8e111 100644 --- a/quantmind/preprocess/pr_newswire.py +++ b/quantmind/preprocess/pr_newswire.py @@ -279,6 +279,12 @@ async def _collect_pr_newswire( end=end, fetcher=fetcher, ) + if not discovery.complete: + return NewsBatch( + failures=discovery.failures, + observed_count=discovery.observed_count, + complete=False, + ) outcomes = await asyncio.gather( *( _collect_observation( diff --git a/tests/preprocess/test_pr_newswire.py b/tests/preprocess/test_pr_newswire.py index d7fca23..d90e0a2 100644 --- a/tests/preprocess/test_pr_newswire.py +++ b/tests/preprocess/test_pr_newswire.py @@ -2,7 +2,7 @@ import unittest from datetime import datetime, timezone from pathlib import Path -from unittest.mock import patch +from unittest.mock import AsyncMock, patch import httpx import respx @@ -337,6 +337,36 @@ async def test_rejects_invalid_windows_before_network_work(self) -> None: class CollectPRNewswireTests(unittest.IsolatedAsyncioTestCase): + async def test_incomplete_discovery_skips_article_collection(self) -> None: + with ( + patch( + "quantmind.preprocess.pr_newswire._DEFAULT_FETCH_POLICY", + _TEST_POLICY, + ), + patch("quantmind.preprocess.pr_newswire._MAX_PAGES", 1), + patch( + "quantmind.preprocess.pr_newswire._collect_observation", + new=AsyncMock(), + ) as collect_observation, + respx.mock(assert_all_called=True) as router, + ): + router.get(url__regex=_LISTING_RE).mock( + side_effect=lambda request: _listing_response( + _fixture("listing_short_page_1.html"), request + ) + ) + result = await _collect_pr_newswire( + start=datetime(2026, 7, 12, tzinfo=timezone.utc), + end=datetime(2026, 7, 14, 4, 30, tzinfo=timezone.utc), + retain_raw_html=False, + ) + + self.assertFalse(result.complete) + self.assertGreater(result.observed_count, 0) + self.assertEqual(result.documents, ()) + self.assertIn("exceeded 1 pages", result.failures[0].message) + collect_observation.assert_not_awaited() + async def test_collects_article_and_discards_raw_html_by_default( self, ) -> None: