From 4cc46970e0a1f04624fb84d9275213571a636b41 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 19:58:27 +0000 Subject: [PATCH 1/5] feat(reporting): persist authorized frozen reporting feeds --- .github/workflows/ci.yml | 163 ++++++ .github/workflows/pr-title-check.yml | 2 +- docs/reporting-frozen-feed.md | 167 ++++++ examples/reporting_receipt_ingress.py | 6 + pyproject.toml | 1 + src/adcp/reporting/feed/__init__.py | 29 + src/adcp/reporting/feed/_errors.py | 35 ++ src/adcp/reporting/feed/errors.py | 31 ++ src/adcp/reporting/feed/memory.py | 140 +++++ src/adcp/reporting/feed/pg.py | 184 +++++++ src/adcp/reporting/feed/projection.py | 514 ++++++++++++++++++ src/adcp/reporting/feed/request.py | 173 ++++++ src/adcp/reporting/feed/required_schema.json | 134 +++++ src/adcp/reporting/feed/schema.py | 27 + src/adcp/reporting/feed/snapshot.py | 270 +++++++++ src/adcp/reporting/feed/store.py | 30 + src/adcp/reporting/ledger/reporting_feed.sql | 43 ++ src/adcp/reporting/receipts/handler.py | 86 ++- src/adcp/reporting/receipts/transport.py | 24 +- src/adcp/server/a2a_server.py | 24 +- src/adcp/server/idempotency/store.py | 8 + src/adcp/server/mcp_tools.py | 43 +- src/adcp/server/serve.py | 24 +- .../reporting/_feed_installed_base.py | 79 +++ tests/conformance/reporting/_feed_process.py | 198 +++++++ tests/conformance/reporting/_feed_support.py | 183 +++++++ .../conformance/reporting/_receipt_frozen.py | 12 +- .../reporting/test_reporting_feed_contract.py | 334 ++++++++++++ .../reporting/test_reporting_feed_history.py | 464 ++++++++++++++++ .../test_reporting_feed_installed_pg.py | 225 ++++++++ .../test_reporting_feed_migration.py | 194 +++++++ .../test_reporting_feed_packaging.py | 142 +++++ .../test_reporting_feed_positions.py | 216 ++++++++ .../reporting/test_reporting_feed_process.py | 86 +++ .../reporting/test_reporting_feed_rolling.py | 321 +++++++++++ .../test_reporting_feed_transactions.py | 329 +++++++++++ .../test_reporting_feed_transports.py | 507 +++++++++++++++++ tests/type_checks/reporting_frozen_feed.py | 55 ++ 38 files changed, 5476 insertions(+), 27 deletions(-) create mode 100644 docs/reporting-frozen-feed.md create mode 100644 src/adcp/reporting/feed/__init__.py create mode 100644 src/adcp/reporting/feed/_errors.py create mode 100644 src/adcp/reporting/feed/errors.py create mode 100644 src/adcp/reporting/feed/memory.py create mode 100644 src/adcp/reporting/feed/pg.py create mode 100644 src/adcp/reporting/feed/projection.py create mode 100644 src/adcp/reporting/feed/request.py create mode 100644 src/adcp/reporting/feed/required_schema.json create mode 100644 src/adcp/reporting/feed/schema.py create mode 100644 src/adcp/reporting/feed/snapshot.py create mode 100644 src/adcp/reporting/feed/store.py create mode 100644 src/adcp/reporting/ledger/reporting_feed.sql create mode 100644 tests/conformance/reporting/_feed_installed_base.py create mode 100644 tests/conformance/reporting/_feed_process.py create mode 100644 tests/conformance/reporting/_feed_support.py create mode 100644 tests/conformance/reporting/test_reporting_feed_contract.py create mode 100644 tests/conformance/reporting/test_reporting_feed_history.py create mode 100644 tests/conformance/reporting/test_reporting_feed_installed_pg.py create mode 100644 tests/conformance/reporting/test_reporting_feed_migration.py create mode 100644 tests/conformance/reporting/test_reporting_feed_packaging.py create mode 100644 tests/conformance/reporting/test_reporting_feed_positions.py create mode 100644 tests/conformance/reporting/test_reporting_feed_process.py create mode 100644 tests/conformance/reporting/test_reporting_feed_rolling.py create mode 100644 tests/conformance/reporting/test_reporting_feed_transactions.py create mode 100644 tests/conformance/reporting/test_reporting_feed_transports.py create mode 100644 tests/type_checks/reporting_frozen_feed.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee326fa65..32267a6e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,6 +10,7 @@ on: - conductor/reporting-status-notifications-1168c - conductor/1167b1-materializer-contracts - conductor/1167b2-durable-managed-reporting + - conductor/reporting-receipt-ingress-b22 # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -194,6 +195,7 @@ jobs: tests/conformance/reporting/ \ --ignore-glob='tests/conformance/reporting/test_reporting_status_*.py' \ --ignore-glob='tests/conformance/reporting/test_reporting_receipt_*.py' \ + --ignore-glob='tests/conformance/reporting/test_reporting_feed_*.py' \ --ignore=tests/conformance/reporting/test_reporting_materializer_rolling.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_process.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_migration.py \ @@ -439,6 +441,167 @@ jobs: path: pg-reporting-receipt-compatibility-evidence.log if-no-files-found: error + pg-reporting-feed: + name: Frozen authorized feed and mounted isolation (Postgres 16) + runs-on: ubuntu-latest + # Crash processes and shared memory/SQL vectors have their own database + # and bounded suite; setup and cleanup have five minutes of headroom. + timeout-minutes: 25 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_feed_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run frozen feeds, checkpoints, mounts, transaction faults and migrations + shell: bash + timeout-minutes: 20 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_feed_test + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_feed_contract.py \ + tests/conformance/reporting/test_reporting_feed_positions.py \ + tests/conformance/reporting/test_reporting_feed_history.py \ + tests/conformance/reporting/test_reporting_feed_transports.py \ + tests/conformance/reporting/test_reporting_feed_transactions.py \ + tests/conformance/reporting/test_reporting_feed_migration.py \ + tests/conformance/reporting/test_reporting_feed_process.py \ + -v -s -ra | tee pg-reporting-feed-evidence.log + - name: Preserve exact public, private, restart and fault evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-feed-evidence-${{ github.run_attempt }} + path: pg-reporting-feed-evidence.log + if-no-files-found: error + + pg-reporting-feed-compatibility: + name: Feed rolling compatibility (nine actual artifacts) + runs-on: ubuntu-latest + # Actual B2.1 produces the artifact and actual B2.2 mounts receipts on + # both sides of migration for every historical reader/writer cell. + timeout-minutes: 50 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_feed_rolling_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - name: Fetch all nine exact approved historical artifacts + timeout-minutes: 2 + run: | + git fetch --no-tags --depth=1 origin \ + 3e76aa54623529a3dda01cd690b8a5c287c75641 \ + 3c405a21f978ed9d3208611bb4a7a8434a056933 \ + 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ + 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 \ + 198d50e61c74fb82aedbf2c77e06a0e200b91db6 \ + ea150fabd5ad90e3abf93f89729d2919f1c61798 \ + 1c91311ec28d25506d5db43f59d0c34936ecb8f7 \ + 8e18ca12b9a0c3750f80aa822058c02982ab3e52 \ + 74b338d81653f4f65bc1520e638da10225f199f3 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run historical binaries before and after the isolated feed migration + shell: bash + timeout-minutes: 45 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_feed_rolling_test + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_feed_rolling.py \ + -v -s -ra | tee pg-reporting-feed-compatibility-evidence.log + - name: Preserve exact artifact origins, catalogs and original replay responses + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-feed-compatibility-evidence-${{ github.run_attempt }} + path: pg-reporting-feed-compatibility-evidence.log + if-no-files-found: error + + pg-reporting-feed-installed: + name: Installed frozen feed (Python 3.10 VCS and sdist) + runs-on: ubuntu-latest + timeout-minutes: 25 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_feed_installed_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + id: feed-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run installed base and PostgreSQL restart cells + shell: bash + timeout-minutes: 20 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_feed_installed_test + ADCP_PYTHON310: ${{ steps.feed-python310.outputs.python-path }} + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_feed_packaging.py \ + tests/conformance/reporting/test_reporting_feed_installed_pg.py \ + -v -s -ra | tee pg-reporting-feed-installed-evidence.log + - name: Preserve installed origins, SQL, strict adopter and cold page evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-feed-installed-evidence-${{ github.run_attempt }} + path: pg-reporting-feed-installed-evidence.log + if-no-files-found: error + conventional-commits: name: Validate conventional commit format runs-on: ubuntu-latest diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index ce86531ea..c989316c2 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -3,7 +3,7 @@ name: PR Title Check on: pull_request: types: [opened, edited, synchronize, reopened] - branches: [main, conductor/1167b2-durable-managed-reporting] + branches: [main, conductor/1167b2-durable-managed-reporting, conductor/reporting-receipt-ingress-b22] permissions: contents: read diff --git a/docs/reporting-frozen-feed.md b/docs/reporting-frozen-feed.md new file mode 100644 index 000000000..489e7efde --- /dev/null +++ b/docs/reporting-frozen-feed.md @@ -0,0 +1,167 @@ +# Frozen authorized reporting feeds + +`PgReportingFeedStore` adds persisted `get_reporting_status(view="periods")` +walks to the existing receipt/materializer store. Pass it to +`ReportingReceiptHandler`, using the same trusted identity adapters and fresh +account resolver as receipt ingress. The existing +[`reporting_receipt_ingress.py`](../examples/reporting_receipt_ingress.py) +composition checks the actual optional feed protocol and schema. A +`PgReportingReceiptStore` instance continues to mount receipts only; both +handler instances can coexist in either construction order. Existing Core +handlers and low-level buyer complete-history inputs keep their behavior. + +This is B2.3 of 4 within B2 of B1/B2. It does not activate Managed/Reconciled +status, the public revision-ownership extension, or higher-tier capabilities. +The new-store notification advertisement veto remains in place, including +Core `reporting.ledger_changed` advertisement. Core polling remains available; +existing eligible reviewed stores keep their supported Core notifications. + +## Reading and consuming a walk + +The handler resolves the canonical account and consumer on **every** request, +including continuation and checkpoint replay. A production RequestContext's +opaque account-cache key and transport tenant never identify the consumer. +API/OAuth/registry and signed-agent adapter identities must agree. Revocation +can deny an old page; it never reconstructs that page from current records. +The SDK's generic idempotency cache is bypassed for this optional mounted read. +Do not supply an `idempotency_key` on a feed request. + +The public record set is Core obligations, revisions, adjustments and the +caller's consumer statements, plus the caller's terminal materializations, +revision receipts and adjustment receipts. Destination bindings, obligation +deliveries, pending attempts, checks and private captured inputs stay private. +Consumer privacy applies with `consumer_status_enabled` both off and on. +Core adjustments appear once. + +The first page acquires the account lock, captures Core and caller histories, +both visible maxima and database `as_of`, and persists the **actual wire +records** and private historical inputs on that same connection. It never +settles an issue, acknowledges work, changes receipt replay or mutates a +notification queue. Partial capture, insertion or page assembly rolls back +the entire snapshot transaction. Memory follows the same unconditional rollback +boundary, including newly initialized collections and sequence heads. + +Ordering is `(domain_rank, sequence, record_kind, record_id)`, with Core rank 0 +and caller reconciliation rank 1. Each domain retains its original committed +sequence; unrelated sequence spaces are never collapsed into one maximum. +The persisted ordering is total and reproducible. Wire identity is the pair +`(record_kind, record_id)` within the authenticated account/consumer snapshot. +Visibility precedes maxima, counts and page limits. Foreign private writes +cannot change another consumer's visible vector or already-open walk. + +Dependency closure replays an affected current owning obligation once and its +complete revision history, including older snapshot restatements and an +unlinked terminal official. Receipt/materialization/adjustment references +retain their exact targets even below `changes_after`. A current official +without its own materialization is never assigned an older snapshot's artifact. +Missing, conflicting or corrupt retained dependencies fail closed. Identical +scope metadata is never used to guess ownership. `pagination.total_count` counts +the final deduplicated wire set **after** closure; it stays identical on all pages. + +Use the same semantic filters throughout a walk. `pagination.max_results` may +change (1–100), and `context` is echoed independently on each request. All +semantic fields, including vendor `ext`, bind the snapshot. Ordinary finite JSON +numbers are supported in context and vendor filters. Account aliases are bound +through the authenticated canonical account, not their spelling in the request. + +Each page supplies one identical `changes_checkpoint`. Persist or advance it +**only after** `pagination.has_more` is false and all pages have been consumed, +including an empty final projected page. On interruption, resume the saved +cursor or replay the previous exhausted checkpoint; do not advance to the new +checkpoint simply because its bytes appeared on page one. Server-issued tokens +cannot prove that a buyer durably applied pages. This consumption rule and its +reference walk tests are B2.3's contract; new durable buyer submission automation +remains separately owned. + +`rpf1` cursor/checkpoint tokens use a versioned compact body and a per-snapshot +HMAC key stored with the snapshot. Their binding hashes the canonical account, +canonical consumer, semantic filters, both after/through coordinates, `as_of`, +wire total, snapshot identity, complete frozen representation and last global +sort key. Principals and filter bodies are not embedded in the token. Generated +client round trips cover maximum supported 2,048-character URL principals and +enforce a 2,048-character token bound in actual MCP/A2A requests and MCP inventory +schemas, including pinned and fallback definitions. Legacy unbound checkpoints +and unavailable snapshots return `INVALID_CHECKPOINT`; restart a complete walk. +Frozen positions are valid only for the periods view. Changing to a summary or +revision view rejects them before consulting the legacy current-state projector. + +## Versioned private inputs and B2.4 continuation + +Version 1 stores the original Core configuration/obligation/revision/issue and +consumer-status snapshot, all caller reconciliation history and sequences, the +retained B2.1 materializer and B2.2 receipt boundaries, exact revision ownership, +complete selections/history, readability at capture, seed membership and exact +dependency edges. Trusted destination references remain private. Capture rejects +conflicting owners or changed immutable records across retained boundaries; +readability may evolve without rewriting earlier evidence. Accepted receipts +remain attached to their original artifacts after later success, failure, +unavailability, corruption or expiry. Rejected receipt evidence and its exact +replacement chain are retained without triggering a materializer retry. +Receipt admission uses its committed caller-feed prefix, so a later check with +an older observation timestamp cannot rewrite acceptance. Current readability +is captured independently from the complete history. A legacy Core consumer +statement that names a revision but omits its obligation ID retains its original +wire bytes; closure follows that exact revision's authoritative owner. + +`representation_version=1`, private `projection_version=1` and +`ownership_mode="absent"` identify this feed representation. They are not a +status activation or production admission certificate. Every existing snapshot +retains its wire bytes, membership, ordering, counts, checkpoint, private inputs +and absent ownership mode through later activation. The SQL table forbids +updates/deletes and version retagging; no public ownership extension is emitted, +including on empty pages. B2.4 must retain this decoder and serving path and may +use isolated additive objects for a new representation on new snapshots. It must +prove continuation using the actual approved B2.3 binary/pages through migration +and restart. It cannot alter objects in this version's exact readiness manifest. + +B2.4 owns opt-in public ownership serialization and complete extension-walk +validation, versioned reconciliation status/projector fences, tier-correct +definitive/count/retention views, offering-specific capability admission and the +final integrated Core → Managed → Reconciled lifecycle. These remain closed. +The final seller acceptance stays inside complete B2; the separately owned buyer +automation/facade and pinned cross-language program remain #1172 prerequisites. + +## Migration, rollout and recovery + +1. Back up retained reporting history and keep the approved B2.2 + [receipt-ingress operations contract](reporting-receipt-ingress.md) and + [materializer recovery contract](reporting-durable-materializer.md). + Drain legacy autonomous materialization writers before activating the durable + worker; ordinary old readers/writers in a controlled rollout do not authorize + competing legacy autonomous workers. Preserve explicit legacy-pending import + and original external effect identity. B2.4 will additionally own incompatible + projector/sweeper drain and activation. +2. During deployment, call `await store.create_schema()` with the deployment + connection or pool **before** starting workers or mounting the new feed. + The same additive bootstrap retains the private turn-primary fairness objects + `adcp_reporting_configuration_lease_turns`, outside the enumerated catalog. + An unmigrated first lease fails loudly; a generation without a turn is treated + as not yet leased. Do not add lease-turn columns to the frozen configuration shape. + Leave the production application-clock override unset for DB-timed workers. +3. Check `await store.reporting_feed_ready()` and + `await store.receipt_ingestion_ready()` in the typed composition. The isolated + feed manifest contains 33 objects. It adds to the preserved 453 ledger objects, + 187 materializer objects and 102 receipt objects: 775 enumerated objects when + those features are installed. Private fairness objects remain outside these + counts. No prior manifest acquires feed objects. Missing/partial/mismatched + objects refuse new snapshots and continuation; no silent legacy fallback. +4. Mount the same authenticated handler on MCP/A2A. New snapshots use the new + store; receipt-only handlers retain their prior inventory and unsupported + feed behavior. Schema installation is transactional, serialized, repeatable + and safe to retry after interruption. It preserves pending work, lease turns, + original receipt ordinal/final responses and server timestamps, captured + boundaries, immutable outcomes and already-open snapshots. +5. On application rollback, leave additive objects and retained data installed. + Actual frozen historical binaries exercise their permitted ordinary reads + and writes on the new schema. They do not serve B2.3 cursor tokens; route + continuations to a compatible feed reader until walks finish. A missing or + damaged snapshot requires operator investigation and conservative full replay, + not a reconstructed projection. This slice does not prune snapshots or + accepted evidence; plan storage capacity for retained histories. + +The materializer finish and receipt ordinal transactions are unchanged: snapshot +reads consume committed history under the same account-lock ordering. Permanently +quarantined epoch-zero readiness records remain quarantined after restart, +pending resume, replay, migration and rollback. Disabled notifications enqueue +nothing; an enabled path retains its original atomic enqueue/rollback contract. +Feed readiness does not release these higher-tier activation gates. diff --git a/examples/reporting_receipt_ingress.py b/examples/reporting_receipt_ingress.py index b0ac3e1c5..9d1917791 100644 --- a/examples/reporting_receipt_ingress.py +++ b/examples/reporting_receipt_ingress.py @@ -8,6 +8,7 @@ from __future__ import annotations from adcp.decisioning.registry import BuyerAgentRegistry +from adcp.reporting.feed import ReportingFeedStore from adcp.reporting.receipts import ( PgReportingReceiptStore, ReceiptAccountResolver, @@ -36,6 +37,11 @@ async def compose_receipt_ingress( switch is needed. Schema installation is an explicit deployment operation. """ await store.receipt_ingestion_ready() + # PgReportingFeedStore is an additive subtype of the same receipt and + # materializer store. Its actual schema enables the periods read route; + # no adopter-maintained capability boolean or second facade is needed. + if isinstance(store, ReportingFeedStore): + await store.reporting_feed_ready() return ReportingReceiptHandler( store, resolve_account=resolve_account, buyer_agents=buyer_agents ) diff --git a/pyproject.toml b/pyproject.toml index d0f4556b3..b6ce77fc6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -194,6 +194,7 @@ adcp = [ "reporting/materializer/assets/*.json", "reporting/materializer/*.json", "reporting/receipts/*.json", + "reporting/feed/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/src/adcp/reporting/feed/__init__.py b/src/adcp/reporting/feed/__init__.py new file mode 100644 index 000000000..b623d4394 --- /dev/null +++ b/src/adcp/reporting/feed/__init__.py @@ -0,0 +1,29 @@ +"""Frozen authorized reporting feeds; PostgreSQL remains an optional lazy import.""" + +from typing import TYPE_CHECKING, Any + +from adcp.reporting.feed.errors import FeedErrorCode, ReportingFeedError +from adcp.reporting.feed.memory import InMemoryReportingFeedStore +from adcp.reporting.feed.snapshot import ReportingFeedRecord, ReportingFeedSnapshot +from adcp.reporting.feed.store import ReportingFeedStore + +if TYPE_CHECKING: + from adcp.reporting.feed.pg import PgReportingFeedStore + +__all__ = [ + "FeedErrorCode", + "InMemoryReportingFeedStore", + "PgReportingFeedStore", + "ReportingFeedError", + "ReportingFeedRecord", + "ReportingFeedSnapshot", + "ReportingFeedStore", +] + + +def __getattr__(name: str) -> Any: + if name == "PgReportingFeedStore": + from adcp.reporting.feed.pg import PgReportingFeedStore + + return PgReportingFeedStore + raise AttributeError(name) diff --git a/src/adcp/reporting/feed/_errors.py b/src/adcp/reporting/feed/_errors.py new file mode 100644 index 000000000..c87d3b8ac --- /dev/null +++ b/src/adcp/reporting/feed/_errors.py @@ -0,0 +1,35 @@ +"""The same closed, redacted storage boundary on both implementations.""" + +from __future__ import annotations + +from collections.abc import Callable, Coroutine +from functools import wraps +from typing import Any, ParamSpec, TypeVar + +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.receipts.errors import ReportingReceiptError + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def storage_errors( + fn: Callable[_P, Coroutine[Any, Any, _R]], +) -> Callable[_P, Coroutine[Any, Any, _R]]: + @wraps(fn) + async def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _R: + try: + return await fn(*args, **kwargs) + except ReportingFeedError: + raise + except (LedgerConflictError, ReportingNotificationError, ReportingReceiptError): + error = ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + except Exception: + error = ReportingFeedError("REPORTING_FEED_STORAGE_UNAVAILABLE") + # Outside the exception scope: provider bodies and identifiers must not + # survive in an implicit exception chain, repr, or transport error. + raise error + + return wrapped diff --git a/src/adcp/reporting/feed/errors.py b/src/adcp/reporting/feed/errors.py new file mode 100644 index 000000000..bd165c1b2 --- /dev/null +++ b/src/adcp/reporting/feed/errors.py @@ -0,0 +1,31 @@ +"""Closed errors for authorized frozen reporting reads.""" + +from __future__ import annotations + +from typing import Literal + +FeedErrorCode = Literal[ + "INVALID_REQUEST", + "INVALID_CHECKPOINT", + "UNAUTHORIZED", + "REPORTING_FEED_SCHEMA_UNREADY", + "REPORTING_FEED_HISTORY_CORRUPT", + "REPORTING_FEED_STORAGE_UNAVAILABLE", +] + +_MESSAGES: dict[FeedErrorCode, str] = { + "INVALID_REQUEST": "supply a periods request with valid reporting filters and pagination", + "INVALID_CHECKPOINT": "restart the reporting walk; this position is unavailable for this scope", + "UNAUTHORIZED": "the reporting account or authenticated consumer is unavailable", + "REPORTING_FEED_SCHEMA_UNREADY": "install and verify the isolated reporting feed schema", + "REPORTING_FEED_HISTORY_CORRUPT": "retained reporting feed evidence requires operator repair", + "REPORTING_FEED_STORAGE_UNAVAILABLE": ( + "reporting feed storage is unavailable; retry the request" + ), +} + + +class ReportingFeedError(RuntimeError): + def __init__(self, code: FeedErrorCode) -> None: + self.code = code + super().__init__(_MESSAGES[code]) diff --git a/src/adcp/reporting/feed/memory.py b/src/adcp/reporting/feed/memory.py new file mode 100644 index 000000000..99e649ea8 --- /dev/null +++ b/src/adcp/reporting/feed/memory.py @@ -0,0 +1,140 @@ +"""Reference feed participant sharing the approved memory rollback boundary.""" + +from __future__ import annotations + +import hashlib +import json +import secrets +from typing import Any + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed._errors import storage_errors +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.feed.projection import capture_feed +from adcp.reporting.feed.request import FeedRequest +from adcp.reporting.feed.snapshot import ( + ReportingFeedSnapshot, + StoredFeedSnapshot, + decode_snapshot, + token_position, +) +from adcp.reporting.ledger._delivery_state import principal +from adcp.reporting.ledger.delivery_changes import ReportingReconciliationChange +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.status_snapshot import memory_snapshot +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.receipts.memory import InMemoryReportingReceiptStore +from adcp.server.helpers import inject_context + + +class InMemoryReportingFeedStore(InMemoryReportingReceiptStore): + """Persisted for this store's lifetime; production uses PgReportingFeedStore.""" + + _reporting_feed_snapshots: dict[str, tuple[bytes, str, bytes]] + + def _feed_snapshot( + self, snapshot_id: str, caller: ReportingDeliveryPrincipal + ) -> StoredFeedSnapshot | None: + value = getattr(self, "_reporting_feed_snapshots", {}).get(snapshot_id) + if value is None: + return None + document, digest, key = value + if hashlib.sha256(document).hexdigest() != digest or len(key) != 32: + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + snapshot = decode_snapshot(json.loads(document)) + if snapshot.snapshot_id != snapshot_id: + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + if snapshot.caller != caller: + return None + return StoredFeedSnapshot(snapshot, key) + + def _save_feed_snapshot(self, stored: StoredFeedSnapshot) -> None: + document = canonical_json_utf8_v1(stored.snapshot.to_storage()) + decode_snapshot(json.loads(document)) + if not hasattr(self, "_reporting_feed_snapshots"): + self._reporting_feed_snapshots = {} + if stored.snapshot.snapshot_id in self._reporting_feed_snapshots: + raise ReportingFeedError("REPORTING_FEED_STORAGE_UNAVAILABLE") + self._reporting_feed_snapshots[stored.snapshot.snapshot_id] = ( + document, + hashlib.sha256(document).hexdigest(), + stored.signing_key, + ) + + def _capture_feed( + self, + caller: ReportingDeliveryPrincipal, + request: FeedRequest, + after: tuple[int, int], + consumer_status_enabled: bool, + ) -> ReportingFeedSnapshot: + owned = tuple( + (seq, who, record) + for seq, who, record in getattr(self, "_delivery_records", ()) + if who == caller or principal(record) == caller + ) + if any(who != caller or principal(record) != caller for _, who, record in owned): + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + changes = tuple(ReportingReconciliationChange(seq, record) for seq, _, record in owned) + return capture_feed( + memory_snapshot(self, caller.account_id), + changes, + caller=caller, + request=request, + after=after, + consumer_status_enabled=consumer_status_enabled, + materializer_boundaries=tuple( + b for b in getattr(self, "_materializer_boundaries", ()) if b.caller == caller + ), + receipt_boundaries=tuple( + b for b in getattr(self, "_receipt_boundaries", ()) if b.caller == caller + ), + ) + + @storage_errors + async def read_reporting_feed( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: + parsed = FeedRequest.parse(request) + try: + async with self._mutation(): + stored = None + offset = 0 + if parsed.cursor is not None: + stored = self._feed_snapshot(token_position(parsed.cursor)[2], caller) + if stored is None: + raise ReportingFeedError("INVALID_CHECKPOINT") + offset = stored.check(parsed.cursor, "cursor", parsed, caller) + after = (0, 0) + if parsed.changes_after is not None: + previous = self._feed_snapshot(token_position(parsed.changes_after)[2], caller) + if previous is None: + raise ReportingFeedError("INVALID_CHECKPOINT") + previous.check(parsed.changes_after, "checkpoint", parsed, caller) + after = previous.snapshot.through + if stored is not None and stored.snapshot.after != after: + raise ReportingFeedError("INVALID_CHECKPOINT") + if stored is None: + stored = StoredFeedSnapshot( + self._capture_feed(caller, parsed, after, consumer_status_enabled), + secrets.token_bytes(32), + ) + self._save_feed_snapshot(stored) + return inject_context(request, stored.page(offset, parsed.limit)) + except LedgerConflictError: + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") from None + + @storage_errors + async def read_reporting_feed_snapshot( + self, snapshot_id: str, *, caller: ReportingDeliveryPrincipal + ) -> ReportingFeedSnapshot | None: + async with self._lock: + stored = self._feed_snapshot(snapshot_id, caller) + return stored.snapshot if stored else None + + async def reporting_feed_ready(self) -> bool: + return True diff --git a/src/adcp/reporting/feed/pg.py b/src/adcp/reporting/feed/pg.py new file mode 100644 index 000000000..51aa94aff --- /dev/null +++ b/src/adcp/reporting/feed/pg.py @@ -0,0 +1,184 @@ +"""Connection-bound frozen reads over the unchanged receipt/materializer store.""" + +from __future__ import annotations + +import hashlib +import json +import secrets +from importlib.resources import files +from typing import Any + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed._errors import storage_errors +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.feed.projection import capture_feed +from adcp.reporting.feed.request import FeedRequest +from adcp.reporting.feed.schema import validate_feed_schema +from adcp.reporting.feed.snapshot import ( + ReportingFeedSnapshot, + StoredFeedSnapshot, + decode_snapshot, + token_position, +) +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.status_snapshot import read_snapshot_on +from adcp.reporting.materializer.capture import decode_materializer_boundary +from adcp.reporting.materializer.pg import _now +from adcp.reporting.receipts.capture import decode_receipt_boundary +from adcp.reporting.receipts.pg import PgReportingReceiptStore +from adcp.server.helpers import inject_context + + +class PgReportingFeedStore(PgReportingReceiptStore): + @storage_errors + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + root = files("adcp.reporting.ledger") + for name in ( + "reporting_materializer.sql", + "reporting_receipt_ingestion.sql", + "reporting_feed.sql", + ): + await connection.execute(root.joinpath(name).read_text()) + + @storage_errors + async def reporting_feed_ready(self) -> bool: + async with self._connection() as connection: + await validate_feed_schema(connection, notifications=self._notifications_enabled) + return True + + async def _feed_snapshot_on( + self, connection: Any, snapshot_id: str, caller: ReportingDeliveryPrincipal + ) -> StoredFeedSnapshot | None: + row = await ( + await connection.execute( + "SELECT document, content_sha256, signing_key FROM reporting_feed_snapshots" + " WHERE account_id=%s AND consumer_id=%s AND snapshot_id=%s", + (caller.account_id, caller.consumer_id, snapshot_id), + ) + ).fetchone() + if row is None: + return None + if hashlib.sha256(row[0].encode()).hexdigest() != row[1] or len(row[2]) != 32: + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + snapshot = decode_snapshot(json.loads(row[0])) + if snapshot.caller != caller or snapshot.snapshot_id != snapshot_id: + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + return StoredFeedSnapshot(snapshot, bytes(row[2])) + + async def _save_feed_snapshot_on(self, connection: Any, stored: StoredFeedSnapshot) -> None: + snapshot = stored.snapshot + document = canonical_json_utf8_v1(snapshot.to_storage()) + decode_snapshot(json.loads(document)) + await connection.execute( + "INSERT INTO reporting_feed_snapshots" + " (account_id,consumer_id,snapshot_id,as_of,representation_version,ownership_mode," + " document,content_sha256,signing_key) VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s)", + ( + snapshot.caller.account_id, + snapshot.caller.consumer_id, + snapshot.snapshot_id, + snapshot.as_of, + snapshot.representation_version, + snapshot.ownership_mode, + document.decode(), + hashlib.sha256(document).hexdigest(), + stored.signing_key, + ), + ) + + async def _capture_feed_on( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + request: FeedRequest, + after: tuple[int, int], + consumer_status_enabled: bool, + ) -> ReportingFeedSnapshot: + # Database time and both histories share the writer's account lock. No + # public reader, second connection, lifecycle settlement, or queue write. + as_of = await _now(connection) + core = await read_snapshot_on(connection, account_id=caller.account_id, as_of=as_of) + await self._validate_feed(connection, caller) + changes = await self._changes(connection, caller) + materializer = await ( + await connection.execute( + "SELECT input,content_sha256=reporting_payload_sha256(input)" + " FROM reporting_materializer_status_boundaries" + " WHERE account_id=%s AND consumer_id=%s ORDER BY sequence", + (caller.account_id, caller.consumer_id), + ) + ).fetchall() + receipts = await ( + await connection.execute( + "SELECT input,content_sha256=reporting_receipt_ingestion_sha256(input)" + " FROM reporting_receipt_ingestion_boundaries" + " WHERE account_id=%s AND consumer_id=%s ORDER BY sequence", + (caller.account_id, caller.consumer_id), + ) + ).fetchall() + if any(not r[1] for r in (*materializer, *receipts)): + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + return capture_feed( + core, + changes, + caller=caller, + request=request, + after=after, + consumer_status_enabled=consumer_status_enabled, + materializer_boundaries=tuple(decode_materializer_boundary(r[0]) for r in materializer), + receipt_boundaries=tuple(decode_receipt_boundary(r[0]) for r in receipts), + ) + + @storage_errors + async def read_reporting_feed( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: + parsed = FeedRequest.parse(request) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + await validate_feed_schema(connection, notifications=self._notifications_enabled) + stored = None + offset = 0 + # Restore before inspecting current projection/evidence. Authorization + # may deny a caller; no missing/old token silently opens a new walk. + if parsed.cursor is not None: + stored = await self._feed_snapshot_on( + connection, token_position(parsed.cursor)[2], caller + ) + if stored is None: + raise ReportingFeedError("INVALID_CHECKPOINT") + offset = stored.check(parsed.cursor, "cursor", parsed, caller) + after = (0, 0) + if parsed.changes_after is not None: + previous = await self._feed_snapshot_on( + connection, token_position(parsed.changes_after)[2], caller + ) + if previous is None: + raise ReportingFeedError("INVALID_CHECKPOINT") + previous.check(parsed.changes_after, "checkpoint", parsed, caller) + after = previous.snapshot.through + if stored is not None and stored.snapshot.after != after: + raise ReportingFeedError("INVALID_CHECKPOINT") + if stored is None: + snapshot = await self._capture_feed_on( + connection, caller, parsed, after, consumer_status_enabled + ) + stored = StoredFeedSnapshot(snapshot, secrets.token_bytes(32)) + await self._save_feed_snapshot_on(connection, stored) + return inject_context(request, stored.page(offset, parsed.limit)) + + @storage_errors + async def read_reporting_feed_snapshot( + self, snapshot_id: str, *, caller: ReportingDeliveryPrincipal + ) -> ReportingFeedSnapshot | None: + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + await validate_feed_schema(connection, notifications=self._notifications_enabled) + stored = await self._feed_snapshot_on(connection, snapshot_id, caller) + return stored.snapshot if stored else None diff --git a/src/adcp/reporting/feed/projection.py b/src/adcp/reporting/feed/projection.py new file mode 100644 index 000000000..86fee2429 --- /dev/null +++ b/src/adcp/reporting/feed/projection.py @@ -0,0 +1,514 @@ +"""Capture version 1 from committed inputs, with exact graph closure and no I/O.""" + +from __future__ import annotations + +from dataclasses import replace +from typing import Any, NoReturn, cast +from uuid import uuid4 + +from pydantic import TypeAdapter + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.feed.request import FeedRequest +from adcp.reporting.feed.snapshot import FeedKind, ReportingFeedRecord, ReportingFeedSnapshot +from adcp.reporting.ledger._delivery_state import ( + _verify_materialization, + _verify_receipt, + adjustment_sha256, + current_receipt, + decode_record, + payload, + principal, + record_identity, +) +from adcp.reporting.ledger.delivery import ( + ReportingMaterializationView, + materialization_to_wire, + receipt_to_wire, +) +from adcp.reporting.ledger.delivery_changes import ReportingReconciliationChange +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.notification_models import ReportingStatusScope +from adcp.reporting.ledger.status import ( + _adjustment_to_wire, + _consumer_status_to_wire, + _filters, + _obligation_to_wire, + _parse, + _revision_to_wire, + _scope_to_wire, +) +from adcp.reporting.ledger.status_projection import ( + ReportingStatusSnapshot, + StatusProjectionInput, + apply_intents_to_snapshot, + lifecycle_intents, + period_selected, + project_status_scope, +) +from adcp.reporting.materializer.capture import ReportingMaterializerBoundary, private_snapshot +from adcp.reporting.receipts.capture import ReportingReceiptBoundary +from adcp.reporting.revision_selection import select_reporting_revision + +_CORE = TypeAdapter(ReportingStatusSnapshot) +_PUBLIC = ( + ReportingMaterializationRecord, + ReportingRevisionReceiptRecord, + ReportingAdjustmentReceiptRecord, +) +Identity = tuple[FeedKind, str] + + +def _corrupt() -> NoReturn: + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + + +def capture_feed( + core: ReportingStatusSnapshot, + changes: tuple[ReportingReconciliationChange, ...], + *, + caller: ReportingDeliveryPrincipal, + request: FeedRequest, + after: tuple[int, int], + consumer_status_enabled: bool, + materializer_boundaries: tuple[ReportingMaterializerBoundary, ...] = (), + receipt_boundaries: tuple[ReportingReceiptBoundary, ...] = (), +) -> ReportingFeedSnapshot: + """Caller owns the account lock and connection until persistence completes. + + Domain rank, original domain sequence, kind, and wire ID form the total + order. The two sequence spaces are never compared or collapsed with max(). + Closure records retain their original sort keys even below changes_after. + """ + if core.account_id != caller.account_id: + _corrupt() + # Filter foreign statements before deriving maxima, projection or membership. + # IDs must resolve unambiguously; scope metadata never supplies ownership. + status_owners: dict[str, set[str]] = {} + for status in core.statuses: + status_owners.setdefault(status.reporting_status_id, set()).add(status.consumer_id) + if any(caller.consumer_id in owners and len(owners) != 1 for owners in status_owners.values()): + _corrupt() + core = private_snapshot(core, caller) + frozen_core = core + # Derive pending lifecycle intents only in the captured value. Reads never + # mutate the old issue tables, queues, captures, or notification boundaries. + for _ in range(3): + intents = lifecycle_intents(core) + if not intents: + break + core = apply_intents_to_snapshot(core, intents) + if lifecycle_intents(core): + _corrupt() + records = tuple(decode_record(payload(c.record)) for c in changes) + if any(c.sequence != i or principal(c.record) != caller for i, c in enumerate(changes, 1)): + _corrupt() + if len({record_identity(r) for r in records}) != len(records): + _corrupt() + raw: dict[Identity, Any] = {} + keys: dict[Identity, tuple[int, int, str, str]] = {} + dependencies: dict[Identity, set[Identity]] = {} + wires: dict[Identity, dict[str, Any]] = {} + owners = {o.reporting_obligation_id: o for o in core.obligations} + revisions = {r.reporting_revision_id: r for r in core.revisions} + adjustments = {a.reporting_adjustment_id: a for a in core.adjustments} + if any( + len(mapping) != len(items) + for mapping, items in ( + (owners, core.obligations), + (revisions, core.revisions), + (adjustments, core.adjustments), + ) + ): + _corrupt() + # Captured mutations are immutable historical inputs, not permission to + # infer today's owner from whichever artifact remains. Every retained + # revision/owner and reconciliation payload must agree across boundaries. + current_records = {record_identity(r): r for r in records} + account_sequences: set[int] = set() + captured_histories: tuple[ + tuple[ReportingMaterializerBoundary | ReportingReceiptBoundary, ...], ... + ] = (materializer_boundaries, receipt_boundaries) + for captured_history in captured_histories: + for sequence, boundary in enumerate(captured_history, 1): + if ( + boundary.caller != caller + or boundary.sequence != sequence + or boundary.account_sequence in account_sequences + ): + _corrupt() + account_sequences.add(boundary.account_sequence) + for historical_owner in boundary.core.obligations: + if owners.get(historical_owner.reporting_obligation_id) != historical_owner: + _corrupt() + for historical_revision in boundary.core.revisions: + current = revisions.get(historical_revision.reporting_revision_id) + if ( + current is None + or replace(historical_revision, readable=current.readable) != current + ): + _corrupt() + for historical_record in boundary.reconciliation: + if current_records.get(record_identity(historical_record)) != historical_record: + _corrupt() + selections = {} + for owning_obligation in core.obligations: + history = tuple( + r + for r in core.revisions + if r.reporting_obligation_id == owning_obligation.reporting_obligation_id + ) + selection = select_reporting_revision( + history, + account_id=caller.account_id, + reporting_obligation_id=owning_obligation.reporting_obligation_id, + required_finality=owning_obligation.required_finality, + ) + if selection.kind == "corrupt": + _corrupt() + selections[owning_obligation.reporting_obligation_id] = { + "kind": selection.kind, + "reporting_revision_id": ( + selection.revision.reporting_revision_id if selection.kind == "selected" else None + ), + "history": [r.reporting_revision_id for r in history], + } + for kind, items, field in ( + ("obligation", core.obligations, "reporting_obligation_id"), + ("revision", core.revisions, "reporting_revision_id"), + ("adjustment", core.adjustments, "reporting_adjustment_id"), + ("consumer_status", core.statuses, "reporting_status_id"), + ): + for item in items: + identity = (cast(FeedKind, kind), getattr(item, field)) + if identity in raw or item.account_id != caller.account_id: + _corrupt() + raw[identity] = item + for seq, kind, record_id, _namespace in core.changes: + if kind not in {"obligation", "revision", "adjustment", "consumer_status"}: + continue + identity = (cast(FeedKind, kind), record_id) + if type(seq) is not int or seq < 1 or identity not in raw or identity in keys: + _corrupt() + keys[identity] = (0, seq, kind, record_id) + if set(raw) != set(keys): + _corrupt() + for change in changes: + record = change.record + if isinstance(record, _PUBLIC): + kind = cast(FeedKind, record.kind) + record_id = ( + record.reporting_materialization_id + if isinstance(record, ReportingMaterializationRecord) + else record.reporting_receipt_id + ) + identity = (kind, record_id) + if identity in raw: + _corrupt() + raw[identity] = record + keys[identity] = (1, change.sequence, kind, record_id) + through = ( + max((k[1] for k in keys.values() if k[0] == 0), default=0), + max((k[1] for k in keys.values() if k[0] == 1), default=0), + ) + if any(a > t for a, t in zip(after, through)): + _corrupt() + filters = request.filters + scoped = _filters(filters) + projection = StatusProjectionInput( + core, + ReportingStatusScope( + caller.account_id, consumer_id=caller.consumer_id if consumer_status_enabled else None + ), + delivery_config_ids=tuple(scoped["delivery_config_ids"] or ()), + media_buy_ids=tuple(scoped["media_buy_ids"] or ()), + feed_purposes=tuple(scoped["feed_purposes"] or ()), + period_start=_parse(scoped["period_start"]), + period_end=_parse(scoped["period_end"]), + ) + scope_result = project_status_scope(projection) + selected_owners = {p.obligation.reporting_obligation_id for p in scope_result.obligations} + healths = set(filters["health"]) + finalities = set(filters["finality"]) + bindings = {r.generation_key: r for r in records if isinstance(r, ReportingDestinationBinding)} + deliveries = {r.scope: r for r in records if isinstance(r, ReportingObligationDeliveryRecord)} + attempts = { + r.reporting_materialization_id: r + for r in records + if isinstance(r, ReportingMaterializationAttempt) + } + outcomes = { + r.reporting_materialization_id: r + for r in records + if isinstance(r, ReportingMaterializationRecord) + } + selected: set[Identity] = set() + readable: dict[str, bool] = {} + owner_for: dict[Identity, str | None] = {} + for identity, record in raw.items(): + kind, record_id = identity + deps: set[Identity] = set() + owner_id: str | None = None + revision_id: str | None = None + if kind == "obligation": + owner_id = record_id + # Retain the complete owning history, including an unlinked snapshot + # plus official and all restatements. Buyer selectors require it. + deps.update( + ("revision", r.reporting_revision_id) + for r in core.revisions + if r.reporting_obligation_id == owner_id + ) + result = project_status_scope( + replace( + projection, + scope=ReportingStatusScope.for_obligation(record, projection.scope.consumer_id), + delivery_config_ids=(), + media_buy_ids=(), + feed_purposes=(), + period_start=None, + period_end=None, + ) + ) + projected_obligation = result.obligations[0] + wires[identity] = _obligation_to_wire( + record, + revisions=projected_obligation.revisions, + health=result.health, + production_status=projected_obligation.projection.production_status, + issues=result.issues, + statuses=projected_obligation.statuses, + ) + elif kind == "revision": + owner_id = record.reporting_obligation_id + if owner_id not in owners: + _corrupt() + revision_id = record_id + wires[identity] = _revision_to_wire(record, owners[owner_id]) + if record.supersedes_reporting_revision_id is not None: + predecessor = revisions.get(record.supersedes_reporting_revision_id) + if predecessor is None or predecessor.reporting_obligation_id != owner_id: + _corrupt() + deps.add(("revision", predecessor.reporting_revision_id)) + elif kind == "adjustment": + revision_id = record.adjusts_reporting_revision_id + target = revisions.get(revision_id) + if target is None or target.finality != "official": + _corrupt() + owner_id = target.reporting_obligation_id + wires[identity] = _adjustment_to_wire(record) + elif kind == "consumer_status": + owner_id, revision_id = record.reporting_obligation_id, record.reporting_revision_id + if owner_id is None and revision_id is not None: + # Core permits a named revision without repeating its owner. + # Resolve only that exact authenticated revision reference. + target = revisions.get(revision_id) + if target is None: + _corrupt() + owner_id = target.reporting_obligation_id + # A legacy obligation_missing statement may name no extant owner. + # Keep its explicit identity; never attach a matching scope guess. + if ( + owner_id not in owners + and revision_id is None + and record.consumer_status == "obligation_missing" + ): + owner_id = None + wires[identity] = _consumer_status_to_wire(record) + if record.supersedes_reporting_status_id is not None: + deps.add(("consumer_status", record.supersedes_reporting_status_id)) + else: + owner_id = record.scope.reporting_obligation_id + revision_id = ( + record.adjusts_reporting_revision_id + if isinstance(record, ReportingAdjustmentReceiptRecord) + else record.reporting_revision_id + ) + owner = owners.get(owner_id) + if owner is None or owner.generation_key != record.scope.generation_key: + _corrupt() + if isinstance(record, ReportingMaterializationRecord): + attempt, binding, delivery = ( + attempts.get(record_id), + bindings.get(record.scope.generation_key), + deliveries.get(record.scope), + ) + revision = revisions.get(revision_id) + if ( + attempt is None + or binding is None + or delivery is None + or revision is None + or attempt.scope != record.scope + or attempt.reporting_revision_id != revision_id + ): + _corrupt() + if record.status != "failed": + _verify_materialization(record, binding, delivery, revision, owner) + view = ReportingMaterializationView( + attempt, + binding, + record, + tuple( + r + for r in records + if isinstance(r, ReportingMaterializationCheck) + and r.reporting_materialization_id == record_id + ), + ) + wires[identity] = materialization_to_wire(view, obligation=owner) + readable[record_id] = view.readable_at(core.as_of) + else: + current_receipt(records, record) + if isinstance(record, ReportingRevisionReceiptRecord): + target = revisions.get(record.reporting_revision_id) + if target is None or record.reporting_materialization_id not in outcomes: + _corrupt() + # Admission is immutable. A later committed check can + # carry an earlier observation time; it changes current + # readability, never the evidence available at admission. + _verify_receipt(record, records[: keys[identity][1]], target) + deps.add(("materialization", record.reporting_materialization_id)) + else: + adjustment = adjustments.get(record.reporting_adjustment_id) + if ( + adjustment is None + or adjustment.adjusts_reporting_revision_id != revision_id + or ( + record.status == "accepted" + and record.observed_adjustment_sha256 != adjustment_sha256(adjustment) + ) + ): + _corrupt() + deps.add(("adjustment", record.reporting_adjustment_id)) + if record.supersedes_reporting_receipt_id is not None: + deps.add((kind, record.supersedes_reporting_receipt_id)) + wires[identity] = receipt_to_wire(record) + if revision_id is not None: + revision = revisions.get(revision_id) + if revision is None or revision.reporting_obligation_id != owner_id: + _corrupt() + deps.add(("revision", revision_id)) + if owner_id is not None: + if owner_id not in owners: + _corrupt() + deps.add(("obligation", owner_id)) + owner_for[identity] = owner_id + dependencies[identity] = deps - {identity} + for identity, record in raw.items(): + owner_id = owner_for[identity] + owner_wire = wires.get(("obligation", owner_id or "")) + matches = owner_id in selected_owners + if identity[0] == "consumer_status" and owner_id is None: + matches = ( + ( + not scoped["delivery_config_ids"] + or record.delivery_config_id in scoped["delivery_config_ids"] + ) + and not scoped["media_buy_ids"] + and not scoped["feed_purposes"] + and period_selected( + record.period_start, + record.period_end, + projection.period_start, + projection.period_end, + ) + ) + if healths and (owner_wire is None or owner_wire["health"] not in healths): + matches = False + if finalities: + revision_id = ( + record.reporting_revision_id + if identity[0] == "revision" + else getattr( + record, + "reporting_revision_id", + getattr(record, "adjusts_reporting_revision_id", None), + ) + ) + finality = ( + revisions[revision_id].finality + if revision_id in revisions + else owners[owner_id].required_finality if owner_id in owners else None + ) + if finality not in finalities: + matches = False + key = keys[identity] + if matches and key[1] > after[key[0]]: + selected.add(identity) + seeds = sorted(selected) + pending = list(selected) + while pending: + identity = pending.pop() + for dependency in dependencies[identity]: + if dependency not in raw: + _corrupt() + if dependency not in selected: + selected.add(dependency) + pending.append(dependency) + # Check every retained edge, including filtered-out records, before saving + # the private dependency history needed for later representations. + if any(d not in raw for deps in dependencies.values() for d in deps): + _corrupt() + inputs = { + "version": 1, + "projection_version": 1, + "ownership_mode": "absent", + "consumer_status_enabled": consumer_status_enabled, + "core": _CORE.dump_python(frozen_core, mode="json"), + "reconciliation": [payload(r) for r in records], + "reconciliation_sequences": [c.sequence for c in changes], + "materializer_boundaries": [b.to_storage() for b in materializer_boundaries], + "receipt_boundaries": [b.to_storage() for b in receipt_boundaries], + "revision_ownership": [ + { + "reporting_revision_id": r.reporting_revision_id, + "reporting_obligation_id": r.reporting_obligation_id, + } + for r in sorted(core.revisions, key=lambda r: r.reporting_revision_id) + ], + "selections": selections, + "readable_materializations": readable, + "seeds": [list(s) for s in seeds], + "dependency_membership": [ + {"record": list(i), "dependencies": [list(d) for d in sorted(dependencies[i])]} + for i in sorted(raw) + ], + } + common = { + "status": "completed", + "view": "periods", + "account_id": caller.account_id, + "scope": _scope_to_wire( + scope_result.configurations, + ledger_as_of=core.as_of, + request=filters, + obligations=tuple(p.obligation for p in scope_result.obligations), + ), + "health": scope_result.health, + "issues": [issue.to_wire() for issue in scope_result.issues], + } + return ReportingFeedSnapshot( + caller, + "rpfs_" + uuid4().hex, + core.as_of, + after, + through, + request.filters_json, + canonical_json_utf8_v1(common), + tuple( + ReportingFeedRecord(i[0], i[1], keys[i], canonical_json_utf8_v1(wires[i])) + for i in sorted(selected, key=keys.__getitem__) + ), + canonical_json_utf8_v1(inputs), + ) diff --git a/src/adcp/reporting/feed/request.py b/src/adcp/reporting/feed/request.py new file mode 100644 index 000000000..ce8ad6736 --- /dev/null +++ b/src/adcp/reporting/feed/request.py @@ -0,0 +1,173 @@ +"""Normalize the complete semantic request, independently of transport models.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from datetime import datetime, timezone +from decimal import Decimal +from functools import lru_cache +from typing import Any, Literal + +from jsonschema import Draft7Validator, FormatChecker + +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.receipts.wire import _IDENTITY_FIELDS +from adcp.validation.schema_loader import get_portable_schema + +TASK = "get_reporting_status" +TOKEN_LIMIT = 2048 + + +def transport_parameters(params: dict[str, Any]) -> dict[str, Any]: + """Read ordinary JSON without rounding the raw pagination integer first. + + The shared transport decoder retains decimal lexemes for financial receipt + admission. Feed context/vendor JSON uses normal finite JSON numbers, while + A2A's exact 1.0 spelling of a page limit must remain distinguishable from a + non-integer such as 1.000000000000000000001. + """ + + def convert(value: Any) -> Any: + if isinstance(value, Decimal): + number = float(value) + if not math.isfinite(number): + raise ValueError("feed parameters require finite JSON numbers") + return number + if type(value) is dict: + return {key: convert(item) for key, item in value.items()} + if type(value) is list: + return [convert(item) for item in value] + return value + + result = dict(convert(params)) + pagination = params.get("pagination") + if isinstance(pagination, dict) and isinstance(pagination.get("max_results"), Decimal): + limit = pagination["max_results"] + if not limit.is_finite() or not 1 <= limit <= 100 or limit != limit.to_integral_value(): + raise ValueError("feed page size requires an exact bounded integer") + result["pagination"]["max_results"] = int(limit) + return result + + +def feed_schema( + direction: Literal["request", "sync"], *, version: str | None = None +) -> dict[str, Any]: + """Copy the bundled schema; never change cached/generated Core schemas.""" + schema = get_portable_schema(TASK, direction, version=version) + if schema is None: + raise ReportingFeedError("REPORTING_FEED_SCHEMA_UNREADY") + properties = schema["properties"] + if direction == "request": + properties["changes_after"].update(minLength=1, maxLength=TOKEN_LIMIT) + else: + properties["changes_checkpoint"].update(minLength=1, maxLength=TOKEN_LIMIT) + # A portable pagination schema can be a local $ref. Draft 7 ignores + # siblings of $ref, so impose the bound in a separate allOf member. + pagination = properties.get("pagination") + if pagination is not None: + properties["pagination"] = { + "allOf": [ + pagination, + { + "properties": { + "cursor": {"type": "string", "minLength": 1, "maxLength": TOKEN_LIMIT} + } + }, + ] + } + return schema + + +@lru_cache(maxsize=1) +def _validator() -> Any: + return Draft7Validator(feed_schema("request"), format_checker=FormatChecker()) + + +@dataclass(frozen=True) +class FeedRequest: + filters_json: bytes + cursor: str | None + changes_after: str | None + limit: int + + @property + def filters(self) -> dict[str, Any]: + return dict(json.loads(self.filters_json)) + + @classmethod + def parse(cls, request: dict[str, Any]) -> FeedRequest: + try: + if ( + type(request) is not dict + or request.get("view") != "periods" + or _IDENTITY_FIELDS.intersection(request) + or "idempotency_key" in request + or request.get("reporting_revision_id") is not None + or not _validator().is_valid(request) + ): + raise ValueError + pagination = request.get("pagination") or {} + cursor, checkpoint = pagination.get("cursor"), request.get("changes_after") + for token in (cursor, checkpoint): + if token is not None and ( + type(token) is not str or not 1 <= len(token) <= TOKEN_LIMIT + ): + raise ValueError + limit = pagination.get("max_results", 50) + if type(limit) is not int or not 1 <= limit <= 100: + raise ValueError + # All request extensions/unknown fields are conservatively semantic. + # Account aliases resolve to the canonical caller outside this hash. + filters = { + k: v + for k, v in request.items() + if k + not in { + "account", + "pagination", + "context", + "changes_after", + "adcp_version", + "adcp_major_version", + "reporting_revision_id", + } + } + for name in ( + "delivery_config_ids", + "media_buy_ids", + "feed_purposes", + "health", + "finality", + ): + filters[name] = sorted(set(request.get(name) or [])) + period = request.get("period") + if period is not None: + dates = [ + datetime.fromisoformat(period[k].replace("Z", "+00:00")) + for k in ("start", "end") + ] + if dates[0] >= dates[1]: + raise ValueError + filters["period"] = { + k: v.astimezone(timezone.utc).isoformat() + for k, v in zip(("start", "end"), dates) + } + else: + filters.pop("period", None) + if not filters.get("ext"): + filters.pop("ext", None) + # Context and vendor filters accept ordinary JSON, including finite + # fractional numbers. The restricted financial evidence encoder is + # not their wire contract. Store deterministic JSON *bytes* as part + # of the snapshot binding, without normalizing the caller's context. + if json.loads(json.dumps(request, allow_nan=False)) != request: + raise ValueError + encoded = json.dumps( + filters, allow_nan=False, sort_keys=True, separators=(",", ":") + ).encode("ascii") + return cls(encoded, cursor, checkpoint, limit) + except (ValueError, TypeError, KeyError, OverflowError, RecursionError): + pass + raise ReportingFeedError("INVALID_REQUEST") diff --git a/src/adcp/reporting/feed/required_schema.json b/src/adcp/reporting/feed/required_schema.json new file mode 100644 index 000000000..b6635323e --- /dev/null +++ b/src/adcp/reporting/feed/required_schema.json @@ -0,0 +1,134 @@ +{ + "column:reporting_feed_snapshots.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_feed_snapshots.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_feed_snapshots.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_feed_snapshots.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_feed_snapshots.document": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_feed_snapshots.ownership_mode": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_feed_snapshots.representation_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_feed_snapshots.signing_key": { + "enabled": true, + "fingerprint": "554c34e416bd4546469b42d5773bc7c59b2104366ffa5259a2838c64cd826e57" + }, + "column:reporting_feed_snapshots.snapshot_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check": { + "enabled": true, + "fingerprint": "00c281a801bf6afa7206453af0c2ec9839dae0df4420b9b7f6a1990eeb7c358f" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check1": { + "enabled": true, + "fingerprint": "375f2992cfd0b07e7dfefe6ae5c2aaa8fd16007d5964e91e8372a28c09aa684f" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check2": { + "enabled": true, + "fingerprint": "84972326afd68c2ff32f599ad0f129c22b91072d0eb4e1380613a91ded47e6da" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check3": { + "enabled": true, + "fingerprint": "18aad02260bda2aba9f6e51349520ac53ba3901ac25e26877d4b06c9d92832e3" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check4": { + "enabled": true, + "fingerprint": "ba2bd76a123a1c07c02d86609a9275fccedb28c2a9049ac87e5441c6c4e8aa64" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check5": { + "enabled": true, + "fingerprint": "fbb1a305ea8b9cd37c90a3612dccb065ca57d04405711dcba64a98bff46e5761" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check6": { + "enabled": true, + "fingerprint": "d6b940c0765d3ae2e773a1cba51ae8b478820bbe9bc593e005115421d1e9a415" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_check7": { + "enabled": true, + "fingerprint": "87a2ec39393813ff33449cd7a02cc4738d1e91d5b6406e212b78372def2c1778" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_document_check": { + "enabled": true, + "fingerprint": "84bdfff00c1f410606570fb522ab542616825c122b05a4e5d7b35d9937dd70e4" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_document_check1": { + "enabled": true, + "fingerprint": "763668adc07dbcc7c5275dab0bbd3b9f0b6833893acd9725f4e9913d09c3c1e9" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_document_check2": { + "enabled": true, + "fingerprint": "b393d759417c024e203b8521b8ba1e34d76e85dda5a8dfcdb1882ed2a56568f2" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_document_check3": { + "enabled": true, + "fingerprint": "2053806dff13a9802996d32867a3858703e7e9a03b11735d216582cbde48ab4b" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_document_check4": { + "enabled": true, + "fingerprint": "bf8ed87ea3b56d6eb3d7c0efa3575695b2b9e56f32bc0b3d172cc2395bcb1c9d" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_ownership_mode_check": { + "enabled": true, + "fingerprint": "4d3e5de1f3258f357fb28587d1142caf0d851d7e57b996e71e7388088177ab7f" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_pkey": { + "enabled": true, + "fingerprint": "69c464c154977fe93cbcf866d711d9dc62357603a4a2cc104b280b44984d3533" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_representation_version_check": { + "enabled": true, + "fingerprint": "0636237258851bc8ba3cb359ab12163c204ef4fc6ae8428235a218b8e2f4c49c" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_signing_key_check": { + "enabled": true, + "fingerprint": "4beb9e7f9146e3095acdb7d1dd3867766108e3ab93313deb9b6b0953e992c024" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_snapshot_id_check": { + "enabled": true, + "fingerprint": "da3012902db7fe22f476ac4cca0d315f27456cace1dbb9f2ba4b3c9eae2d0a9e" + }, + "constraint:reporting_feed_snapshots.reporting_feed_snapshots_snapshot_id_key": { + "enabled": true, + "fingerprint": "c97a22e931dedffcfbd8ab22824035126745be64aa5d616a90e33563643ca749" + }, + "function:reporting_feed_immutable()": { + "enabled": true, + "fingerprint": "b17f620de435a86c8815c3efbace4b602166bad7dc95f923434d402cddcd63f5" + }, + "index:reporting_feed_snapshots.reporting_feed_snapshots_pkey": { + "enabled": true, + "fingerprint": "14570880993d60951d32cd0af8202fd77cbeb41255e8a953339f2250d8aa76c0" + }, + "index:reporting_feed_snapshots.reporting_feed_snapshots_snapshot_id_key": { + "enabled": true, + "fingerprint": "a92d1e1e87600141c76239c1a044aadf70462a648598f28c41c9c507cca36230" + }, + "table:reporting_feed_snapshots": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_feed_snapshots.reporting_feed_immutable": { + "enabled": true, + "fingerprint": "e14c7a91478071445133f1dcf1354904686724cbe5830d08039e11c7e4568829" + } +} diff --git a/src/adcp/reporting/feed/schema.py b/src/adcp/reporting/feed/schema.py new file mode 100644 index 000000000..0a0e74823 --- /dev/null +++ b/src/adcp/reporting/feed/schema.py @@ -0,0 +1,27 @@ +"""Independent feed manifest; old ingestion and materializer guards stay intact.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts.schema import validate_receipt_schema + + +async def validate_feed_schema(connection: Any, *, notifications: bool = False) -> None: + ready = False + try: + required = json.loads( + files("adcp.reporting.feed").joinpath("required_schema.json").read_text() + ) + actual = await schema_objects(connection) + if required and all(actual.get(k) == v for k, v in required.items()): + await validate_receipt_schema(connection, notifications=notifications) + ready = True + except Exception: + ready = False + if not ready: + raise ReportingFeedError("REPORTING_FEED_SCHEMA_UNREADY") diff --git a/src/adcp/reporting/feed/snapshot.py b/src/adcp/reporting/feed/snapshot.py new file mode 100644 index 000000000..90147eb58 --- /dev/null +++ b/src/adcp/reporting/feed/snapshot.py @@ -0,0 +1,270 @@ +"""Immutable wire membership and versioned private inputs for one complete walk. + +Version 1 is permanently the legacy ownership representation. A future projector +can create version 2 for new snapshots, but must retain this decoder and serve +these stored bytes, ordering, counts and checkpoints without re-projection. +""" + +from __future__ import annotations + +import base64 +import hashlib +import hmac +import json +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Literal + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.feed.request import TOKEN_LIMIT, FeedRequest +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal + +FeedKind = Literal[ + "obligation", + "revision", + "adjustment", + "consumer_status", + "materialization", + "revision_receipt", + "adjustment_receipt", +] +_ARRAYS: dict[FeedKind, str] = { + "obligation": "periods", + "revision": "revisions", + "adjustment": "adjustments", + "consumer_status": "consumer_statuses", + "materialization": "materializations", + "revision_receipt": "receipts", + "adjustment_receipt": "adjustment_receipts", +} +GlobalKey = tuple[int, int, str, str] + + +def _digest(value: Any) -> str: + return hashlib.sha256(canonical_json_utf8_v1(value)).hexdigest() + + +@dataclass(frozen=True) +class ReportingFeedRecord: + kind: FeedKind + record_id: str + key: GlobalKey + wire_json: bytes = field(repr=False) + + def to_storage(self) -> dict[str, Any]: + return { + "kind": self.kind, + "id": self.record_id, + "key": list(self.key), + "wire": json.loads(self.wire_json), + } + + +@dataclass(frozen=True) +class ReportingFeedSnapshot: + caller: ReportingDeliveryPrincipal + snapshot_id: str + as_of: datetime + after: tuple[int, int] + through: tuple[int, int] + filters_json: bytes = field(repr=False) + common_json: bytes = field(repr=False) + records: tuple[ReportingFeedRecord, ...] = field(repr=False) + inputs_json: bytes = field(repr=False) + representation_version: int = 1 + ownership_mode: Literal["absent"] = "absent" + + @property + def total_count(self) -> int: + return len(self.records) + + @property + def inputs(self) -> dict[str, Any]: + """A detached PRIVATE copy. Never serialize this into a response/ext.""" + return dict(json.loads(self.inputs_json)) + + def to_storage(self) -> dict[str, Any]: + return { + "version": 1, + "representation_version": self.representation_version, + "ownership_mode": self.ownership_mode, + "account_id": self.caller.account_id, + "consumer_id": self.caller.consumer_id, + "snapshot_id": self.snapshot_id, + "as_of": self.as_of.isoformat(), + "after": list(self.after), + "through": list(self.through), + # Filters use ordinary JSON; keep their exact versioned bytes inside + # the restricted canonical financial-history document as a string. + "filters": self.filters_json.decode("ascii"), + "common": json.loads(self.common_json), + "total_count": self.total_count, + "records": [r.to_storage() for r in self.records], + "inputs": self.inputs, + } + + @property + def binding(self) -> str: + # Includes the complete frozen projection and every private dependency, + # not merely maxima followed by mutable reads. Principal lengths never + # increase token size because the binding is a fixed SHA-256 digest. + return _digest(self.to_storage()) + + +def decode_snapshot(value: Any) -> ReportingFeedSnapshot: + result = None + try: + if ( + type(value) is not dict + or type(value["version"]) is not int + or value["version"] != 1 + or type(value["representation_version"]) is not int + or value["representation_version"] != 1 + or value["ownership_mode"] != "absent" + or type(value["filters"]) is not str + or type(json.loads(value["filters"])) is not dict + ): + raise ValueError + records = tuple( + ReportingFeedRecord( + r["kind"], r["id"], tuple(r["key"]), canonical_json_utf8_v1(r["wire"]) + ) + for r in value["records"] + ) + if any( + r.kind not in _ARRAYS + or type(r.record_id) is not str + or len(r.key) != 4 + or type(r.key[0]) is not int + or r.key[0] not in {0, 1} + or type(r.key[1]) is not int + or r.key[1] < 1 + or r.key[2:] != (r.kind, r.record_id) + for r in records + ) or [r.key for r in records] != sorted({r.key for r in records}): + raise ValueError + if len({(r.kind, r.record_id) for r in records}) != len(records): + raise ValueError + positions = value["after"], value["through"] + if any( + type(p) is not list or len(p) != 2 or any(type(n) is not int or n < 0 for n in p) + for p in positions + ): + raise ValueError + if any(a > t for a, t in zip(*positions)): + raise ValueError + result = ReportingFeedSnapshot( + ReportingDeliveryPrincipal(value["account_id"], value["consumer_id"]), + value["snapshot_id"], + aware_utc(datetime.fromisoformat(value["as_of"])), + tuple(positions[0]), + tuple(positions[1]), + value["filters"].encode("ascii"), + canonical_json_utf8_v1(value["common"]), + records, + canonical_json_utf8_v1(value["inputs"]), + ) + if result.to_storage() != value: + raise ValueError + except (ValueError, TypeError, KeyError, IndexError, RecursionError): + result = None + if result is None: + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") + return result + + +@dataclass(frozen=True) +class StoredFeedSnapshot: + snapshot: ReportingFeedSnapshot + signing_key: bytes = field(repr=False) + + def token(self, kind: Literal["cursor", "checkpoint"], position: int) -> str: + snapshot = self.snapshot + key = snapshot.records[position - 1].key if position else None + body = canonical_json_utf8_v1( + [ + 1, + kind, + snapshot.snapshot_id, + position, + snapshot.binding, + _digest(key), + ] + ) + signature = hmac.digest(self.signing_key, body, "sha256") + return "rpf1." + base64.urlsafe_b64encode(body + signature).decode().rstrip("=") + + def check( + self, + token: str, + kind: Literal["cursor", "checkpoint"], + request: FeedRequest, + caller: ReportingDeliveryPrincipal, + ) -> int: + decoded = token_position(token) + snapshot = self.snapshot + position = decoded[3] + if ( + snapshot.caller != caller + or snapshot.filters_json != request.filters_json + or decoded[1] != kind + or decoded[2] != snapshot.snapshot_id + or type(position) is not int + or not 0 <= position <= snapshot.total_count + or (kind == "checkpoint" and position != snapshot.total_count) + or (kind == "cursor" and not 0 < position < snapshot.total_count) + or not hmac.compare_digest(token, self.token(kind, position)) + ): + raise ReportingFeedError("INVALID_CHECKPOINT") + return position + + def page(self, offset: int, limit: int) -> dict[str, Any]: + snapshot = self.snapshot + result: dict[str, Any] = json.loads(snapshot.common_json) + for array in _ARRAYS.values(): + result[array] = [] + for record in snapshot.records[offset : offset + limit]: + result[_ARRAYS[record.kind]].append(json.loads(record.wire_json)) + end = min(offset + limit, snapshot.total_count) + more = end < snapshot.total_count + result.update( + ledger_snapshot_id=snapshot.snapshot_id, + ledger_as_of=snapshot.as_of.isoformat().replace("+00:00", "Z"), + changes_checkpoint=self.token("checkpoint", snapshot.total_count), + pagination={ + "total_count": snapshot.total_count, + "has_more": more, + **({"cursor": self.token("cursor", end)} if more else {}), + }, + ) + return result + + +def token_position(token: str) -> list[Any]: + value = None + try: + if type(token) is not str or not token.startswith("rpf1.") or len(token) > TOKEN_LIMIT: + raise ValueError + text = token[5:] + raw = base64.b64decode(text + "=" * (-len(text) % 4), altchars=b"-_", validate=True) + value = json.loads(raw[:-32]) + if ( + type(value) is not list + or len(value) != 6 + or type(value[0]) is not int + or value[0] != 1 + or value[1] not in {"cursor", "checkpoint"} + or type(value[2]) is not str + or len(value[2]) != 37 + or not value[2].startswith("rpfs_") + or any(c not in "0123456789abcdef" for c in value[2][5:]) + ): + raise ValueError + except (ValueError, TypeError, KeyError, UnicodeError, RecursionError): + value = None + if value is None: + # Unbound Core and old reconciliation tokens are explicitly rejected. + raise ReportingFeedError("INVALID_CHECKPOINT") + return value diff --git a/src/adcp/reporting/feed/store.py b/src/adcp/reporting/feed/store.py new file mode 100644 index 000000000..ebb6d590f --- /dev/null +++ b/src/adcp/reporting/feed/store.py @@ -0,0 +1,30 @@ +"""Optional feed participant. All legacy required protocols remain unchanged.""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from adcp.reporting.feed.snapshot import ReportingFeedSnapshot +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal + + +@runtime_checkable +class ReportingFeedStore(Protocol): + """Trusted caller seam. The mounted handler reauthorizes every request.""" + + async def read_reporting_feed( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: ... + + async def read_reporting_feed_snapshot( + self, + snapshot_id: str, + *, + caller: ReportingDeliveryPrincipal, + ) -> ReportingFeedSnapshot | None: ... + + async def reporting_feed_ready(self) -> bool: ... diff --git a/src/adcp/reporting/ledger/reporting_feed.sql b/src/adcp/reporting/ledger/reporting_feed.sql new file mode 100644 index 000000000..fd3dc69c5 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_feed.sql @@ -0,0 +1,43 @@ +-- B2.3 isolated immutable snapshots. No old catalog object is changed. +DO $migration$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.feed.schema')); + CREATE TABLE IF NOT EXISTS reporting_feed_snapshots ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + snapshot_id TEXT COLLATE "C" NOT NULL CHECK (snapshot_id ~ '^rpfs_[0-9a-f]{32}$'), + as_of TIMESTAMPTZ NOT NULL, + representation_version INTEGER NOT NULL CHECK (representation_version = 1), + ownership_mode TEXT COLLATE "C" NOT NULL CHECK (ownership_mode = 'absent'), + document TEXT NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + signing_key BYTEA NOT NULL CHECK (octet_length(signing_key) = 32), + PRIMARY KEY (account_id, consumer_id, snapshot_id), + UNIQUE (snapshot_id), + CHECK (content_sha256 = encode(sha256(convert_to(document,'UTF8')), 'hex')), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(document::jsonb)), + CHECK ((document::jsonb->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((document::jsonb->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((document::jsonb->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((document::jsonb->>'snapshot_id') IS NOT DISTINCT FROM snapshot_id), + CHECK ((document::jsonb->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK ((document::jsonb->>'representation_version')::integer IS NOT DISTINCT FROM representation_version), + CHECK ((document::jsonb->>'ownership_mode') IS NOT DISTINCT FROM ownership_mode), + CHECK (jsonb_typeof(document::jsonb->'records') IS NOT DISTINCT FROM 'array'), + CHECK (jsonb_array_length(document::jsonb->'records') IS NOT DISTINCT FROM (document::jsonb->>'total_count')::integer), + CHECK (jsonb_typeof(document::jsonb->'inputs') IS NOT DISTINCT FROM 'object'), + CHECK (document::jsonb - ARRAY['version','representation_version','ownership_mode', + 'account_id','consumer_id','snapshot_id','as_of','after','through','filters', + 'common','total_count','records','inputs'] = '{}'::jsonb) + ); + CREATE OR REPLACE FUNCTION reporting_feed_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'reporting feed snapshot is immutable' USING ERRCODE = '23514'; + END + $function$; + DROP TRIGGER IF EXISTS reporting_feed_immutable ON reporting_feed_snapshots; + CREATE TRIGGER reporting_feed_immutable BEFORE UPDATE OR DELETE ON reporting_feed_snapshots + FOR EACH ROW EXECUTE FUNCTION reporting_feed_immutable(); +END +$migration$; diff --git a/src/adcp/reporting/receipts/handler.py b/src/adcp/reporting/receipts/handler.py index 70239e119..94de912f3 100644 --- a/src/adcp/reporting/receipts/handler.py +++ b/src/adcp/reporting/receipts/handler.py @@ -3,7 +3,7 @@ from __future__ import annotations from collections.abc import Awaitable, Callable -from typing import Any +from typing import TYPE_CHECKING, Any from adcp.decisioning.context import AuthInfo, RequestContext from adcp.decisioning.registry import BuyerAgent, BuyerAgentRegistry, HttpSigCredential @@ -14,8 +14,11 @@ from adcp.reporting.receipts.errors import ReportingReceiptError from adcp.reporting.receipts.store import ReportingReceiptBatchStore from adcp.reporting.receipts.wire import TASK, validate_receipt_request -from adcp.server.base import ADCPHandler, ToolContext -from adcp.types import Error, SyncReportingReceiptsRequest +from adcp.server.base import ADCPHandler, NotImplementedResponse, ToolContext +from adcp.types import Error, GetReportingStatusRequest, SyncReportingReceiptsRequest + +if TYPE_CHECKING: + from adcp.reporting.feed.store import ReportingFeedStore ReceiptAccountResolver = Callable[[dict[str, Any], ToolContext, str], Awaitable[str]] """Resolve AND reauthorize the exact account reference for this consumer on every call. @@ -91,7 +94,7 @@ async def _consumer(context: ToolContext, registry: BuyerAgentRegistry | None) - class ReportingReceiptHandler(ADCPHandler[ToolContext]): - """Mount only the durable receipt task; tier activation remains separately gated. + """Mount receipts and the optional frozen feed; tier activation is separately gated. ``resolve_account`` is an application ACL, called even for a completed batch. ``buyer_agents`` optionally re-resolves API/OAuth/signed commercial @@ -99,7 +102,7 @@ class ReportingReceiptHandler(ADCPHandler[ToolContext]): the consumer. Authentication middleware must populate trusted context. """ - advertised_tools = {TASK} + advertised_tools = {TASK, "get_reporting_status"} def __init__( self, @@ -107,6 +110,7 @@ def __init__( *, resolve_account: ReceiptAccountResolver, buyer_agents: BuyerAgentRegistry | None = None, + consumer_status_enabled: bool = False, ) -> None: super().__init__() if not isinstance(store, ReportingReceiptBatchStore): @@ -114,6 +118,78 @@ def __init__( self.receipt_store = store self._receipt_account_resolver = resolve_account self._receipt_registry = buyer_agents + from adcp.reporting.feed.store import ReportingFeedStore + + self.reporting_feed_store: ReportingFeedStore | None = ( + store if isinstance(store, ReportingFeedStore) else None + ) + self._feed_consumer_status_enabled = consumer_status_enabled + + def advertised_tools_for_instance(self) -> set[str]: + return {TASK, "get_reporting_status"} if self.reporting_feed_store is not None else {TASK} + + async def get_reporting_status( + self, + params: GetReportingStatusRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any] | NotImplementedResponse: + """One authenticated mount; the optional store freezes periods walks.""" + from adcp.reporting.feed.errors import ReportingFeedError + from adcp.reporting.feed.request import FeedRequest + from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler + from adcp.reporting.ledger.store import ReportingLedgerStore + + if self.reporting_feed_store is None: + return self._not_supported("get_reporting_status") + request = ( + params + if isinstance(params, dict) + else params.model_dump(mode="json", exclude_unset=True) + ) + try: + if request.get("view") == "periods": + FeedRequest.parse(request) + if context is None: + raise ReportingFeedError("UNAUTHORIZED") + try: + consumer = await _consumer(context, self._receipt_registry) + account = await self._receipt_account_resolver( + dict(request["account"]), context, consumer + ) + if isinstance(context, RequestContext) and context.account.id != account: + raise ReportingFeedError("UNAUTHORIZED") + caller = ReportingDeliveryPrincipal(account, consumer) + except (ReportingReceiptError, ReportingNotificationError, ValueError, TypeError): + raise ReportingFeedError("UNAUTHORIZED") from None + if request.get("view") == "periods": + return await self.reporting_feed_store.read_reporting_feed( + request, + caller=caller, + consumer_status_enabled=self._feed_consumer_status_enabled, + ) + pagination = request.get("pagination") + positions = ( + request.get("changes_after"), + pagination.get("cursor") if isinstance(pagination, dict) else None, + ) + if any(isinstance(p, str) and p.startswith("rpf1.") for p in positions): + # A view change cannot send a frozen position to the mutable + # legacy projector. Leave ordinary Core requests compatible. + raise ReportingFeedError("INVALID_CHECKPOINT") + if isinstance(self.receipt_store, ReportingLedgerStore): + return await ReportingStatusHandler( + self.receipt_store, + consumer_status_enabled=self._feed_consumer_status_enabled, + ).handle(request, caller=ReportingStatusCaller(account, consumer)) + return self._not_supported("get_reporting_status") + except ReportingFeedError as error: + code, message = error.code, str(error) + except Exception: + unavailable = ReportingFeedError("REPORTING_FEED_STORAGE_UNAVAILABLE") + code, message = unavailable.code, str(unavailable) + raise ADCPTaskError( + operation="get_reporting_status", errors=[Error(code=code, message=message)] + ) async def sync_reporting_receipts( self, diff --git a/src/adcp/reporting/receipts/transport.py b/src/adcp/reporting/receipts/transport.py index 6fe8b0109..a88ecd404 100644 --- a/src/adcp/reporting/receipts/transport.py +++ b/src/adcp/reporting/receipts/transport.py @@ -105,7 +105,15 @@ def _raw_json(body: str | bytes) -> Any: ) -def _a2a_receipt_invocation(body: bytes | None) -> dict[str, Any] | None: +def _parameters(params: dict[str, Any], task: str) -> dict[str, Any]: + if task == "get_reporting_status": + from adcp.reporting.feed.request import transport_parameters + + return transport_parameters(params) + return dict(_receipt_numbers(params)) + + +def _a2a_receipt_invocation(body: bytes | None, *, task: str = TASK) -> dict[str, Any] | None: try: if body is None: return None @@ -130,27 +138,27 @@ def _a2a_receipt_invocation(body: bytes | None) -> dict[str, Any] | None: continue if type(data) is dict and data.get("skill"): invocations.append(data) - if len(invocations) == 1 and invocations[0]["skill"] == TASK: + if len(invocations) == 1 and invocations[0]["skill"] == task: return dict(invocations[0]) except (ValueError, TypeError, KeyError, RecursionError): pass return None -def a2a_receipt_parameters(body: bytes | None) -> dict[str, Any] | None: +def a2a_receipt_parameters(body: bytes | None, *, task: str = TASK) -> dict[str, Any] | None: """Distinguish an exact receipt route from invalid receipt parameters. None means no uniquely identified standard invocation. An empty dictionary means that invocation's parameters are invalid and must reach the ordinary whole-shape rejection. Neither case can authorize a domain write. """ - invocation = _a2a_receipt_invocation(body) + invocation = _a2a_receipt_invocation(body, task=task) if invocation is None: return None try: params = invocation.get("parameters") if type(params) is dict: - return dict(_receipt_numbers(params)) + return _parameters(params, task) except (ValueError, TypeError, RecursionError): pass return {} @@ -176,7 +184,7 @@ def a2a_receipt_has_invalid_unicode(body: bytes | None) -> bool: return False -def mcp_receipt_parameters(body: bytes | None) -> dict[str, Any]: +def mcp_receipt_parameters(body: bytes | None, *, task: str = TASK) -> dict[str, Any]: """The already selected tools/call must match this exact receipt invocation.""" try: if body is None: @@ -186,10 +194,10 @@ def mcp_receipt_parameters(body: bytes | None) -> dict[str, Any]: type(envelope) is dict and envelope.get("jsonrpc") == "2.0" and envelope.get("method") == "tools/call" - and envelope["params"]["name"] == TASK + and envelope["params"]["name"] == task and type(envelope["params"].get("arguments")) is dict ): - return dict(_receipt_numbers(envelope["params"]["arguments"])) + return _parameters(envelope["params"]["arguments"], task) except (ValueError, TypeError, KeyError, RecursionError): pass return {} diff --git a/src/adcp/server/a2a_server.py b/src/adcp/server/a2a_server.py index 6b7602810..4404e43e8 100644 --- a/src/adcp/server/a2a_server.py +++ b/src/adcp/server/a2a_server.py @@ -544,7 +544,13 @@ async def _call_handler() -> Any: return await self._tool_callers[skill_name](params, tool_context) return await _dispatch_with_middleware( - self._middleware, skill_name, params, tool_context, _call_handler + self._middleware, + skill_name, + params, + tool_context, + _call_handler, + frozen_reporting_feed=skill_name == "get_reporting_status" + and getattr(self._handler, "reporting_feed_store", None) is not None, ) def _build_tool_context(self, skill_name: str, request: RequestContext) -> ToolContext: @@ -640,7 +646,13 @@ def _parse_request(self, context: RequestContext) -> tuple[str | None, dict[str, if raw is not None: return "sync_reporting_receipts", raw raise - if parsed[0] == "sync_reporting_receipts" and request is not None: + if request is not None and ( + parsed[0] == "sync_reporting_receipts" + or ( + parsed[0] == "get_reporting_status" + and getattr(self._handler, "reporting_feed_store", None) is not None + ) + ): from adcp.reporting.receipts.transport import ( RAW_RECEIPT_BODY_SCOPE_KEY, a2a_receipt_parameters, @@ -649,7 +661,10 @@ def _parse_request(self, context: RequestContext) -> tuple[str | None, dict[str, # The protobuf representation has already lost numeric lexemes. # Only the standard raw invocation can authorize the batch body. return parsed[0], ( - a2a_receipt_parameters(request.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY)) or {} + a2a_receipt_parameters( + request.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY), task=parsed[0] + ) + or {} ) return parsed @@ -1508,7 +1523,8 @@ def agent_card_url(request: Request) -> str: # independent of whether bearer-auth middleware is configured. app.add_middleware( _A2ARequestContextMiddleware, - receipt_ingress="sync_reporting_receipts" in executor.supported_skills, + receipt_ingress="sync_reporting_receipts" in executor.supported_skills + or getattr(handler, "reporting_feed_store", None) is not None, ) # Startup log lives on the create_a2a_server path (symmetric with diff --git a/src/adcp/server/idempotency/store.py b/src/adcp/server/idempotency/store.py index 621b4e7d1..9bb9a1276 100644 --- a/src/adcp/server/idempotency/store.py +++ b/src/adcp/server/idempotency/store.py @@ -56,6 +56,7 @@ # Set only by the SDK transport dispatcher, never from request metadata. This # also covers middleware whose wrapped callable is named call_next/execute. _RECEIPT_BATCH_DISPATCH: ContextVar[bool] = ContextVar("receipt_batch_dispatch", default=False) +_FROZEN_FEED_DISPATCH: ContextVar[bool] = ContextVar("frozen_feed_dispatch", default=False) # Registry of functions returned by IdempotencyStore.wrap. Read by # adcp.decisioning.validate_idempotency.is_wrapped() to reconcile the @@ -220,6 +221,13 @@ def wrap(self, handler: HandlerFn) -> HandlerFn: @wraps(handler) async def _wrapped(*args: Any, **kwargs: Any) -> Any: + if _FROZEN_FEED_DISPATCH.get() or ( + getattr(handler, "__name__", None) == "get_reporting_status" + and getattr(getattr(handler, "__self__", None), "reporting_feed_store", None) + is not None + ): + # A frozen page is replayable data, never cached authorization. + return await handler(*args, **kwargs) if ( _RECEIPT_BATCH_DISPATCH.get() or getattr(handler, "__name__", None) == "sync_reporting_receipts" diff --git a/src/adcp/server/mcp_tools.py b/src/adcp/server/mcp_tools.py index ebb8cd421..5859b5d22 100644 --- a/src/adcp/server/mcp_tools.py +++ b/src/adcp/server/mcp_tools.py @@ -2422,6 +2422,14 @@ def get_tools_for_handler( # stubs. Do not mutate cached upstream/generated schema inputs. definition["inputSchema"] = receipt_schema("request") definition["outputSchema"] = receipt_schema("sync") + elif ( + definition["name"] == "get_reporting_status" + and getattr(instance, "reporting_feed_store", None) is not None + ): + from adcp.reporting.feed.request import feed_schema + + definition["inputSchema"] = feed_schema("request") + definition["outputSchema"] = feed_schema("sync") return definitions if not list_validator_keys(version=resolved_version): @@ -2447,6 +2455,16 @@ def get_tools_for_handler( definition["outputSchema"] = receipt_schema("sync", version=resolved_version) versioned.append(definition) continue + if ( + name == "get_reporting_status" + and getattr(instance, "reporting_feed_store", None) is not None + ): + from adcp.reporting.feed.request import feed_schema + + definition["inputSchema"] = feed_schema("request", version=resolved_version) + definition["outputSchema"] = feed_schema("sync", version=resolved_version) + versioned.append(definition) + continue definition["inputSchema"] = input_schema output_schema = get_mcp_schema(name, "sync", version=resolved_version) if output_schema is not None: @@ -2757,6 +2775,10 @@ def create_tool_caller( }.get(method_name, method_name) method = getattr(handler, adopter_method_name) params_model = _resolve_params_pydantic_model(method) + frozen_reporting_feed = ( + method_name == "get_reporting_status" + and getattr(handler, "reporting_feed_store", None) is not None + ) # Opt-in server-side schema modes. ``None`` keeps validation off # entirely (zero overhead on the hot path) — the TS-port default for @@ -2773,6 +2795,17 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) raw_params = params # Preserve original wire params for context echo. + if frozen_reporting_feed and params.get("view") == "periods": + from adcp.reporting.feed.errors import ReportingFeedError + from adcp.reporting.feed.request import FeedRequest + + try: + FeedRequest.parse(params) + except ReportingFeedError as exc: + raise ADCPTaskError( + operation=method_name, errors=[Error(code=exc.code, message=str(exc))] + ) from None + if method_name == "sync_reporting_receipts": from adcp.reporting.receipts.errors import ReportingReceiptError from adcp.reporting.receipts.wire import ReceiptBatch @@ -3024,7 +3057,11 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) ], ) from exc - if isinstance(params, dict) and method_name != "sync_reporting_receipts": + if ( + isinstance(params, dict) + and method_name != "sync_reporting_receipts" + and not frozen_reporting_feed + ): params = _apply_unknown_field_policy( method_name, params, @@ -3054,6 +3091,8 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) call_params: Any = params if method_name == "sync_reporting_receipts": call_params = raw_params + elif frozen_reporting_feed: + call_params = params elif params_model is not None and isinstance(params, dict): try: call_params = params_model.model_validate(params) @@ -3261,6 +3300,8 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) result.pop("status") return result + if frozen_reporting_feed: + setattr(call_tool, "_adcp_frozen_reporting_feed", True) return call_tool diff --git a/src/adcp/server/serve.py b/src/adcp/server/serve.py index c28b0a557..4e2afb603 100644 --- a/src/adcp/server/serve.py +++ b/src/adcp/server/serve.py @@ -518,6 +518,8 @@ async def _dispatch_with_middleware( params: dict[str, Any], context: ToolContext, call_handler: Callable[[], Awaitable[Any]], + *, + frozen_reporting_feed: bool = False, ) -> Any: """Run ``call_handler`` wrapped in the supplied middleware chain. @@ -585,6 +587,14 @@ async def call_next() -> Any: return await _step(0) finally: _RECEIPT_BATCH_DISPATCH.reset(token) + if frozen_reporting_feed: + from adcp.server.idempotency.store import _FROZEN_FEED_DISPATCH + + token = _FROZEN_FEED_DISPATCH.set(True) + try: + return await _step(0) + finally: + _FROZEN_FEED_DISPATCH.reset(token) return await _step(0) @@ -2787,11 +2797,12 @@ def _register_tool( build_mcp_error_result, ) - if name == "sync_reporting_receipts": + frozen_reporting_feed = getattr(caller, "_adcp_frozen_reporting_feed", False) + if name == "sync_reporting_receipts" or frozen_reporting_feed: object.__setattr__(mcp, "_adcp_receipt_ingress", True) async def fn(**kwargs: Any) -> dict[str, Any]: - if name == "sync_reporting_receipts": + if name == "sync_reporting_receipts" or frozen_reporting_feed: request_context = _get_starlette_request_for_dispatch() if request_context is not None: from adcp.reporting.receipts.transport import ( @@ -2800,7 +2811,7 @@ async def fn(**kwargs: Any) -> dict[str, Any]: ) kwargs = mcp_receipt_parameters( - request_context.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY) + request_context.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY), task=name ) # Caller identity: FastMCP does not expose an authenticated principal # at the SDK level (``Context.client_id`` is a session hint, not an @@ -2845,7 +2856,12 @@ async def _call_handler() -> Any: # via ``context`` closed over by _call_handler. mw_context = context if context is not None else ToolContext() result = await _dispatch_with_middleware( - middleware, name, kwargs, mw_context, _call_handler + middleware, + name, + kwargs, + mw_context, + _call_handler, + frozen_reporting_feed=frozen_reporting_feed, ) else: result = await _call_handler() diff --git a/tests/conformance/reporting/_feed_installed_base.py b/tests/conformance/reporting/_feed_installed_base.py new file mode 100644 index 000000000..074959a14 --- /dev/null +++ b/tests/conformance/reporting/_feed_installed_base.py @@ -0,0 +1,79 @@ +"""Run outside the checkout on Python 3.10 without the optional PG drivers.""" + +import asyncio +import hashlib +import importlib +import importlib.util +import json +import sys +from importlib.resources import files +from pathlib import Path + + +async def main(settings): + assert sys.version_info[:2] == (3, 10) + assert importlib.util.find_spec("psycopg") is None + assert importlib.util.find_spec("psycopg_pool") is None + import adcp.reporting.feed as feed + from adcp.reporting.ledger import ReportingDeliveryPrincipal + from adcp.reporting.receipts import InMemoryReportingReceiptStore, ReportingReceiptHandler + from adcp.server import ADCPHandler + from adcp.server.mcp_tools import get_tools_for_handler + from adcp.types import GetReportingStatusRequest, GetReportingStatusResponse + + assert "adcp.reporting.feed.pg" not in sys.modules + for name in feed.__all__: + assert getattr(feed, name) is not None + assert "psycopg" not in sys.modules and "psycopg_pool" not in sys.modules + for relative, digest in settings["assets"].items(): + raw = files("adcp.reporting").joinpath(relative).read_bytes() + assert hashlib.sha256(raw).hexdigest() == digest + prefix = "https://buyer.example.test/" + caller = ReportingDeliveryPrincipal("installed-account", prefix + "a" * (2048 - len(prefix))) + store = feed.InMemoryReportingFeedStore() + request = GetReportingStatusRequest.model_validate( + {"account": {"account_id": caller.account_id}, "view": "periods"} + ).model_dump(mode="json", exclude_unset=True) + page = GetReportingStatusResponse.model_validate( + await store.read_reporting_feed(request, caller=caller) + ) + assert page.pagination.total_count == 0 and page.pagination.has_more is False + assert 0 < len(page.changes_checkpoint) <= 2048 + + async def resolve(reference, context, consumer): + assert consumer == caller.consumer_id + return caller.account_id + + old = ReportingReceiptHandler(InMemoryReportingReceiptStore(), resolve_account=resolve) + new = ReportingReceiptHandler(store, resolve_account=resolve) + assert await old.get_reporting_status(request) == await ADCPHandler().get_reporting_status( + request + ) + assert {t["name"] for t in get_tools_for_handler(old)} == { + "get_adcp_capabilities", + "sync_reporting_receipts", + } + assert "get_reporting_status" in {t["name"] for t in get_tools_for_handler(new)} + workspace = Path(settings["workspace"]).resolve() + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + origins = {} + for name in settings["modules"]: + path = Path(importlib.import_module(name).__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == settings["modules"][name] + assert "site-packages" in str(path) and not path.is_relative_to(workspace) + origins[name] = str(path) + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + return { + "python": "3.10", + "driver_absent": True, + "origins": origins, + "token_length": len(page.changes_checkpoint), + "assets": settings["assets"], + } + + +if __name__ == "__main__": + print(json.dumps(asyncio.run(main(json.load(sys.stdin))))) diff --git a/tests/conformance/reporting/_feed_process.py b/tests/conformance/reporting/_feed_process.py new file mode 100644 index 000000000..bc6b9c4b9 --- /dev/null +++ b/tests/conformance/reporting/_feed_process.py @@ -0,0 +1,198 @@ +"""Actual installed/source transport process; durable snapshot crash boundaries.""" + +import asyncio +import hashlib +import importlib +import importlib.util +import json +import sys +import traceback +from importlib.resources import files +from pathlib import Path +from types import SimpleNamespace + + +async def main(settings): + from psycopg_pool import AsyncConnectionPool + + from adcp.reporting.ledger import ReportingDeliveryPrincipal + from adcp.reporting.receipts import PgReportingReceiptStore, ReportingReceiptHandler + + installed = settings.get("installed") + origins = {} + if installed is not None: + workspace = Path(installed["workspace"]).resolve() + if "python" in installed: + assert list(sys.version_info[:2]) == installed["python"] + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, expected in installed["modules"].items(): + path = Path(importlib.import_module(name).__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + assert "site-packages" in str(path) and not path.is_relative_to(workspace) + origins[name] = str(path) + + async def pause(point, **evidence): + if settings.get("pause") == point: + print(json.dumps({"point": point, **evidence}), flush=True) + assert json.loads(await asyncio.to_thread(sys.stdin.readline))["continue"] + + receipt_only = settings.get("receipt_only", False) + if receipt_only: + store_type = PgReportingReceiptStore + else: + from adcp.reporting.feed import PgReportingFeedStore + + class Store(PgReportingFeedStore): + async def _capture_feed_on(self, connection, *args, **kwargs): + captured = await super()._capture_feed_on(connection, *args, **kwargs) + await pause("captured") + return captured + + async def _save_feed_snapshot_on(self, connection, stored): + await super()._save_feed_snapshot_on(connection, stored) + await pause("inserted", snapshot_id=stored.snapshot.snapshot_id) + + store_type = Store + + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=1, open=False + ) as pool: + store = store_type(pool=pool, notifications=settings["notifications"]) + if settings["action"] == "install": + await store.create_schema() + await store.create_schema() + assert await store.receipt_ingestion_ready() + if not receipt_only: + assert await store.reporting_feed_ready() + if settings.get("legacy_status_schema"): + from adcp.reporting.ledger import PgReportingReconciliationStore + from adcp.reporting.outbox import PgStatusNotificationStore + + await PgStatusNotificationStore( + PgReportingReconciliationStore(pool=pool, notifications=True) + ).create_schema() + result = { + "installed": True, + "materializer_objects": len( + json.loads( + files("adcp.reporting.materializer") + .joinpath("required_schema.json") + .read_text() + ) + ), + "receipt_objects": len( + json.loads( + files("adcp.reporting.receipts") + .joinpath("required_schema.json") + .read_text() + ) + ), + } + if not receipt_only: + result["feed_objects"] = len( + json.loads( + files("adcp.reporting.feed").joinpath("required_schema.json").read_text() + ) + ) + else: + spec = importlib.util.spec_from_file_location("frozen_transport", settings["helper"]) + transport = importlib.util.module_from_spec(spec) + spec.loader.exec_module(transport) + mount = transport.MountedReceipts(SimpleNamespace(store=store), hydrated=True) + caller = ReportingDeliveryPrincipal(**settings["caller"]) + subject = SimpleNamespace( + obligation=SimpleNamespace(account_id=caller.account_id), + binding=SimpleNamespace(consumer_id=caller.consumer_id), + ) + if not receipt_only: + mount.handler = ReportingReceiptHandler( + store, + resolve_account=mount.resolve_account, + consumer_status_enabled=settings.get("feedback", False), + ) + mount.authorize(subject) + async with mount.client() as client: + _, inventory = await mount.mcp(client, inventory=True) + expected = {"get_adcp_capabilities", "sync_reporting_receipts"} + if not receipt_only: + expected.add("get_reporting_status") + assert {tool["name"] for tool in inventory["tools"]} == expected + for path in ("/.well-known/agent.json", "/.well-known/agent-card.json"): + assert {s["id"] for s in (await client.get(path)).json()["skills"]} == expected + if settings["action"] == "receipt": + _, response = await mount.mcp(client, settings["request"]) + assert response["results"] + for v1 in (False, True): + assert (await mount.a2a(client, settings["request"], v1=v1))[1] == response + mount.grants.clear() + for call in (mount.mcp, mount.a2a): + assert ( + transport.error_code((await call(client, settings["request"]))[1]) + == "UNAUTHORIZED" + ) + result = response + else: + + def reporting(wire): + return wire.replace( + '"name": "sync_reporting_receipts"', '"name": "get_reporting_status"' + ).replace( + '"skill": "sync_reporting_receipts"', '"skill": "get_reporting_status"' + ) + + request = dict(settings["request"]) + pages = [] + while True: + if settings.get("transport", "mcp") == "mcp": + _, page = await mount.mcp(client, request, mutate_wire=reporting) + else: + _, page = await mount.a2a( + client, request, mutate_wire=reporting, v1=settings.get("v1", False) + ) + assert "pagination" in page, page + pages.append(page) + await pause("committed", result=page) + if settings["action"] != "walk" or not page["pagination"]["has_more"]: + break + assert len(pages) < 1000 + request["pagination"] = { + **request.get("pagination", {}), + "cursor": page["pagination"]["cursor"], + } + snapshot = await store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=caller + ) + mount.grants.clear() + assert ( + transport.error_code( + (await mount.a2a(client, request, mutate_wire=reporting))[1] + ) + == "UNAUTHORIZED" + ) + result = { + "pages": pages, + "binding": snapshot.binding, + "version": snapshot.representation_version, + "ownership_mode": snapshot.ownership_mode, + } + if installed is not None: + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + return {"point": "done", "result": result, "origins": origins} + + +if __name__ == "__main__": + try: + result = asyncio.run(main(json.loads(sys.stdin.readline()))) + except Exception as error: + result = { + "point": "failed", + "failure": type(error).__name__, + "frames": [ + [Path(frame.filename).name, frame.lineno] + for frame in traceback.extract_tb(error.__traceback__) + ], + } + print(json.dumps(result), flush=True) diff --git a/tests/conformance/reporting/_feed_support.py b/tests/conformance/reporting/_feed_support.py new file mode 100644 index 000000000..04b8fb26f --- /dev/null +++ b/tests/conformance/reporting/_feed_support.py @@ -0,0 +1,183 @@ +"""Shared memory/PostgreSQL feed vectors and real mounted reporting reads.""" + +from contextlib import asynccontextmanager +from copy import deepcopy +from dataclasses import replace + +import pytest + +from adcp.reporting.feed import InMemoryReportingFeedStore +from adcp.reporting.ledger.delivery_models import ReportingDeliveryScope +from adcp.reporting.receipts import ReportingReceiptHandler + +from ._durable_materializer_support import DurableHarness +from ._generation_support import isolated_reporting_pool +from ._receipt_support import adjustment_for, receipt_case, request_for +from ._receipt_transport import MountedReceipts +from ._reconciliation_support import Clock + +ARRAYS = ( + "periods", + "revisions", + "adjustments", + "consumer_statuses", + "materializations", + "receipts", + "adjustment_receipts", +) + + +@asynccontextmanager +async def feed_harness(backend, *, notifications=False): + clock = Clock() + if backend == "memory": + yield DurableHarness( + InMemoryReportingFeedStore(clock=clock, notifications=notifications), clock + ) + else: + from adcp.reporting.feed import PgReportingFeedStore + + async with isolated_reporting_pool(autocommit=True) as pool: + # Autonomous writers use DB time. A historical application clock + # on the low-level reconciliation API would reject their new rows. + store = PgReportingFeedStore(pool=pool, notifications=notifications) + await store.create_schema() + yield DurableHarness(store, clock, pool) + + +@pytest.fixture( + params=[("memory", False), ("memory", True), ("postgres", False), ("postgres", True)] +) +async def feeds(request): + backend, notifications = request.param + async with feed_harness(backend, notifications=notifications) as h: + yield h + + +async def mixed_case(h, **kwargs): + s = await receipt_case(h, **kwargs) + adjustment = await adjustment_for(h, s) + req = request_for(s, adjustment_receipts=[adjustment]) + result = await h.store.ingest_receipt_batch(req, caller=s.binding.principal) + assert [r["result"] for r in result["results"]] == ["recorded", "recorded"] + return s, req, result + + +def feed_request(s, *, limit=1, **kwargs): + return { + "adcp_version": "3.2-rc.3", + "view": "periods", + "account": {"account_id": s.obligation.account_id}, + "pagination": {"max_results": limit}, + **kwargs, + } + + +async def walk(store, request, caller, *, first=None, **kwargs): + """Reference consumption: return the checkpoint only after exact exhaustion.""" + request = deepcopy(request) + pages, rows, seen = [], {a: [] for a in ARRAYS}, set() + checkpoint = None + while True: + page = ( + first + if not pages and first is not None + else await store.read_reporting_feed(request, caller=caller, **kwargs) + ) + pages.append(page) + if checkpoint is None: + checkpoint = page["changes_checkpoint"] + assert page["changes_checkpoint"] == checkpoint + assert page["ledger_snapshot_id"] == pages[0]["ledger_snapshot_id"] + assert page["ledger_as_of"] == pages[0]["ledger_as_of"] + assert page["pagination"]["total_count"] == pages[0]["pagination"]["total_count"] + for name in ARRAYS: + rows[name].extend(page.get(name, [])) + if not page["pagination"]["has_more"]: + assert sum(map(len, rows.values())) == page["pagination"]["total_count"] + return pages, rows, checkpoint + cursor = page["pagination"]["cursor"] + assert cursor not in seen and len(pages) < 1000 + seen.add(cursor) + request["pagination"]["cursor"] = cursor + + +async def restart(h): + """Memory keeps serialized durable state; PG uses a new store instance.""" + if h.pool is None: + store = InMemoryReportingFeedStore( + clock=h.clock, notifications=h.store._notification_state is not None + ) + for key, value in vars(h.store).items(): + if key not in {"_clock", "_lock"}: + vars(store)[key] = deepcopy(value) + else: + from adcp.reporting.feed import PgReportingFeedStore + + store = PgReportingFeedStore( + pool=h.pool, clock=h.clock, notifications=h.store._notifications_enabled + ) + h.store = store + return store + + +async def second_consumer(h, s, consumer="other-buyer"): + scope = ReportingDeliveryScope( + s.obligation.generation_key, consumer, s.obligation.reporting_obligation_id + ) + binding = replace(s.binding, consumer_id=consumer) + await h.store.put_destination_binding(binding) + await h.store.bind_obligation_delivery(replace(s.delivery, scope=scope)) + await h.store.commit_materialization_attempt(replace(s.attempt, scope=scope)) + await h.store.commit_materialization(replace(s.outcome, scope=scope)) + receipt = replace(s.receipt, scope=scope) + other = replace( + s, + binding=binding, + delivery=replace(s.delivery, scope=scope), + attempt=replace(s.attempt, scope=scope), + outcome=replace(s.outcome, scope=scope), + receipt=receipt, + ) + await h.store.ingest_receipt_batch(request_for(other), caller=binding.principal) + return other + + +def without_feed(image): + return { + k: v + for k, v in image.items() + if k not in {"_reporting_feed_snapshots", "reporting_feed_snapshots"} + } + + +class MountedFeed(MountedReceipts): + def __init__(self, h, *, feedback=False, **kwargs): + super().__init__(h, **kwargs) + self.handler = ReportingReceiptHandler( + h.store, + resolve_account=self.resolve_account, + buyer_agents=self.registry, + consumer_status_enabled=feedback, + ) + self.handler.get_reporting_status = self.idempotency.wrap(self.handler.get_reporting_status) + if kwargs.get("version") is not None: + self.handler.adcp_version = kwargs["version"] + + async def mcp(self, client, request=None, *, mutate_wire=None, **kwargs): + def rewrite(wire): + wire = wire.replace( + '"name": "sync_reporting_receipts"', '"name": "get_reporting_status"' + ) + return mutate_wire(wire) if mutate_wire else wire + + return await super().mcp(client, request, mutate_wire=rewrite, **kwargs) + + async def a2a(self, client, request, *, mutate_wire=None, **kwargs): + def rewrite(wire): + wire = wire.replace( + '"skill": "sync_reporting_receipts"', '"skill": "get_reporting_status"' + ) + return mutate_wire(wire) if mutate_wire else wire + + return await super().a2a(client, request, mutate_wire=rewrite, **kwargs) diff --git a/tests/conformance/reporting/_receipt_frozen.py b/tests/conformance/reporting/_receipt_frozen.py index 5c23aed9b..98abb3fbc 100644 --- a/tests/conformance/reporting/_receipt_frozen.py +++ b/tests/conformance/reporting/_receipt_frozen.py @@ -33,7 +33,11 @@ async def main(settings): async with AsyncConnectionPool( settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=2, open=False ) as pool: - if settings["artifact"] == "b21": + if settings["artifact"] == "b22": + from adcp.reporting.receipts import PgReportingReceiptStore + + store_type = PgReportingReceiptStore + elif settings["artifact"] == "b21": from adcp.reporting.materializer import PgReportingMaterializerStore store_type = PgReportingMaterializerStore @@ -77,7 +81,7 @@ async def main(settings): ) assert len(revisions) == 1 and revisions[0].finality == "official" status, projector_turns = None, 0 - if settings["notifications"] and settings["artifact"] in {"c", "b1", "b21"}: + if settings["notifications"] and settings["artifact"] in {"c", "b1", "b21", "b22"}: from adcp.reporting.outbox import PgStatusNotificationStore status = PgStatusNotificationStore( @@ -150,7 +154,7 @@ async def project_ordinary_status(): ordinary_materializer = True await project_ordinary_status() readiness = None - if settings["artifact"] in {"a", "b", "c", "b1", "b21"}: + if settings["artifact"] in {"a", "b", "c", "b1", "b21", "b22"}: from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp.reporting.outbox._schema import validate_schema @@ -162,7 +166,7 @@ async def project_ordinary_status(): readiness = False assert readiness == (settings["artifact"] != "a") materializer = None - if settings["artifact"] == "b21": + if settings["artifact"] in {"b21", "b22"}: assert await store.materializer_ready() boundaries = await store.read_materializer_boundaries(caller=caller) assert len(boundaries) == 1 diff --git a/tests/conformance/reporting/test_reporting_feed_contract.py b/tests/conformance/reporting/test_reporting_feed_contract.py new file mode 100644 index 000000000..f79f61d9a --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_contract.py @@ -0,0 +1,334 @@ +"""Combined wire membership, exact dependency history, immutable private capture.""" + +import json +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.types import GetReportingStatusResponse + +from ._feed_support import ( + ARRAYS, + feed_request, + feeds, + mixed_case, + restart, + second_consumer, + walk, + without_feed, +) +from ._generation_support import configuration, revision_for +from ._receipt_support import extra_materialization, request_for +from .test_reporting_notification_outbox import statement + +__all__ = ["feeds"] + + +@pytest.mark.parametrize("limit", [1, 5, 6, 7]) +async def test_allowlist_exact_totals_dependency_order_and_final_only_checkpoint(feeds, limit): + h = feeds + s, receipt_request, receipt_response = await mixed_case(h) + before = without_feed(await h.image()) + pages, rows, checkpoint = await walk(h.store, feed_request(s, limit=limit), s.binding.principal) + assert len(pages) == (6 if limit == 1 else 2 if limit == 5 else 1) + assert [len(rows[a]) for a in ARRAYS] == [1, 1, 1, 0, 1, 1, 1] + assert all(p["changes_checkpoint"] == checkpoint for p in pages) + assert ( + rows["receipts"][0]["reporting_revision_id"] + == rows["revisions"][0]["reporting_revision_id"] + ) + assert ( + rows["adjustment_receipts"][0]["reporting_adjustment_id"] + == rows["adjustments"][0]["reporting_adjustment_id"] + ) + assert ( + rows["receipts"][0]["reporting_materialization_id"] + == rows["materializations"][0]["reporting_materialization_id"] + ) + assert ( + rows["materializations"][0]["reporting_obligation_id"] + == rows["periods"][0]["reporting_obligation_id"] + ) + assert all(len(p["changes_checkpoint"]) <= 2048 for p in pages) + for page in pages: + GetReportingStatusResponse.model_validate(page) + assert "ext" not in page + text = json.dumps(page) + for secret in ( + "trusted_binding_ref", + "trusted-binding-1", + "obligation_delivery", + "materialization_attempt", + "materialization_check", + "signing_key", + ): + assert secret not in text + assert without_feed(await h.image()) == before + assert ( + await h.store.ingest_receipt_batch(receipt_request, caller=s.binding.principal) + == receipt_response + ) + + +async def test_incremental_receipt_replays_old_exact_revision_materialization_adjustment_and_owner( + feeds, +): + h = feeds + s, _, _ = await mixed_case(h) + _, _, checkpoint = await walk(h.store, feed_request(s), s.binding.principal) + await extra_materialization(h, s, "new-materialization", 2) + new_receipt = replace( + s.receipt, + reporting_materialization_id="new-materialization", + reporting_receipt_id="receipt-second-0002", + observed_at=s.receipt.observed_at + timedelta(seconds=10), + ) + # Accepted terminality is per revision: consumer rejection is not a retry. + # The new outcome alone must still bring the old owner and revision. + pages, rows, _ = await walk( + h.store, feed_request(s, changes_after=checkpoint), s.binding.principal + ) + assert [len(rows[a]) for a in ARRAYS] == [1, 1, 0, 0, 1, 0, 0] + assert ( + rows["materializations"][0]["reporting_materialization_id"] + == new_receipt.reporting_materialization_id + ) + snapshot = await h.store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=s.binding.principal + ) + assert snapshot.after != (0, 0) + assert snapshot.inputs["selections"][s.obligation.reporting_obligation_id]["history"] == [ + s.revision.reporting_revision_id + ] + assert snapshot.inputs["receipt_boundaries"] + assert len(snapshot.inputs["reconciliation"]) >= 8 + + +async def test_receipt_only_delta_closes_exact_old_targets(feeds): + h = feeds + from ._receipt_support import adjustment_for, receipt_case + + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + _, _, checkpoint = await walk(h.store, feed_request(s), s.binding.principal) + request = request_for(s, adjustment_receipts=[adjustment]) + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + _, rows, _ = await walk(h.store, feed_request(s, changes_after=checkpoint), s.binding.principal) + assert [len(rows[a]) for a in ARRAYS] == [1, 1, 1, 0, 1, 1, 1] + + +@pytest.mark.parametrize("feedback", [False, True]) +async def test_foreign_consumer_writes_do_not_change_open_snapshot_or_visible_vector( + feeds, feedback +): + h = feeds + s, _, _ = await mixed_case(h) + request = feed_request(s) + first = await h.store.read_reporting_feed( + request, caller=s.binding.principal, consumer_status_enabled=feedback + ) + original = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + other = await second_consumer(h, s) + store = await restart(h) + pages, rows, checkpoint = await walk( + store, request, s.binding.principal, first=first, consumer_status_enabled=not feedback + ) + assert pages[-1]["pagination"]["total_count"] == 6 + assert ( + rows["receipts"][0]["received_at"] + != ( + await store.read_reporting_feed( + feed_request(other, limit=100), caller=other.binding.principal + ) + )["receipts"][0]["received_at"] + or h.pool is None + ) + saved = await store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + assert saved == original and saved.ownership_mode == "absent" + assert saved.inputs["consumer_status_enabled"] is feedback + assert ( + await store.read_reporting_feed_snapshot(saved.snapshot_id, caller=other.binding.principal) + is None + ) + fresh = await store.read_reporting_feed( + feed_request(s, changes_after=checkpoint), caller=s.binding.principal + ) + fresh_snapshot = await store.read_reporting_feed_snapshot( + fresh["ledger_snapshot_id"], caller=s.binding.principal + ) + assert fresh_snapshot.through == original.through + assert fresh["pagination"] == {"total_count": 0, "has_more": False} + assert other.binding.consumer_id not in json.dumps(saved.to_storage()) + + +@pytest.mark.parametrize("feedback", [False, True]) +async def test_foreign_core_status_does_not_advance_visible_maximum_or_change_counts( + feeds, feedback +): + h = feeds + s, _, _ = await mixed_case(h) + own = replace( + statement(s.obligation), + consumer_status="received", + reporting_revision_id=s.revision.reporting_revision_id, + observed_revision_content_sha256=s.revision.revision_content_sha256, + ) + await h.store.record_consumer_status_with_lifecycle(own) + pages, rows, checkpoint = await walk( + h.store, feed_request(s), s.binding.principal, consumer_status_enabled=feedback + ) + original = await h.store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=s.binding.principal + ) + foreign = replace( + own, reporting_status_id="foreign-core-status", consumer_id="status-only-consumer" + ) + await h.store.record_consumer_status_with_lifecycle(foreign) + store = await restart(h) + current = await store.read_reporting_feed( + feed_request(s, changes_after=checkpoint), + caller=s.binding.principal, + consumer_status_enabled=feedback, + ) + snapshot = await store.read_reporting_feed_snapshot( + current["ledger_snapshot_id"], caller=s.binding.principal + ) + assert snapshot.through == original.through + assert current["pagination"] == {"total_count": 0, "has_more": False} + assert rows["consumer_statuses"][0]["reporting_status_id"] == own.reporting_status_id + assert "foreign-core-status" not in json.dumps(snapshot.to_storage()) + other = await store.read_reporting_feed( + feed_request(s, limit=100), + caller=replace(s.binding.principal, consumer_id=foreign.consumer_id), + consumer_status_enabled=feedback, + ) + assert len(other["consumer_statuses"]) == 1 + assert other["consumer_statuses"][0]["reporting_status_id"] == foreign.reporting_status_id + assert other["materializations"] == other["receipts"] == other["adjustment_receipts"] == [] + + +async def test_readability_clock_configuration_and_private_inputs_survive_restart( + feeds, monkeypatch +): + h = feeds + s, _, _ = await mixed_case(h) + req = feed_request(s) + first = await h.store.read_reporting_feed(req, caller=s.binding.principal) + original = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + expected = await walk(h.store, req, s.binding.principal, first=first) + configs = await h.store.list_configurations(account_id=s.obligation.account_id) + await h.store.put_configuration( + replace(configs[0], deactivated_at=configs[0].deactivated_at + timedelta(hours=1)) + ) + await h.store.set_revision_readable( + reporting_revision_id=s.revision.reporting_revision_id, + account_id=s.obligation.account_id, + readable=False, + ) + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.delivery.scope, + s.outcome.reporting_materialization_id, + "later-check", + "corrupt", + h.clock(), + ) + ) + h.clock.now += timedelta(days=500) + store = await restart(h) + if h.pool is None: + + def forbidden(*args, **kwargs): + raise AssertionError("continuation consulted current projection") + + monkeypatch.setattr(store, "_capture_feed", forbidden) + else: + + async def forbidden(*args, **kwargs): + raise AssertionError("continuation consulted current projection") + + monkeypatch.setattr(store, "_capture_feed_on", forbidden) + assert await walk(store, req, s.binding.principal, first=first) == expected + assert ( + await store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + == original + ) + assert original.inputs["readable_materializations"][s.outcome.reporting_materialization_id] + + +async def test_identical_scope_obligations_keep_exact_ownership_across_pages(feeds): + h = feeds + s, _, _ = await mixed_case(h) + config = replace( + configuration(), + delivery_config_id="other", + feed_purpose="billing", + required_finality="official", + ) + await h.store.put_configuration(config) + obligation = replace( + s.obligation, reporting_obligation_id="identical-scope-owner", delivery_config_id="other" + ) + await h.store.commit_obligation(obligation) + revision, rows = revision_for(obligation, suffix="other-owner") + await h.store.commit_revision(revision, rows) + pages, records, _ = await walk(h.store, feed_request(s), s.binding.principal) + snapshot = await h.store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=s.binding.principal + ) + assert len(records["periods"]) == len(records["revisions"]) == 2 + assert snapshot.inputs["revision_ownership"] == sorted( + [ + { + "reporting_revision_id": s.revision.reporting_revision_id, + "reporting_obligation_id": s.obligation.reporting_obligation_id, + }, + { + "reporting_revision_id": revision.reporting_revision_id, + "reporting_obligation_id": obligation.reporting_obligation_id, + }, + ], + key=lambda b: b["reporting_revision_id"], + ) + assert records["receipts"][0]["reporting_obligation_id"] == s.obligation.reporting_obligation_id + + +async def test_core_record_bytes_and_legacy_handler_are_unchanged(feeds): + h = feeds + s, _, _ = await mixed_case(h) + old = await ReportingStatusHandler(h.store).handle( + {"view": "periods"}, + caller=ReportingStatusCaller(s.obligation.account_id, s.binding.consumer_id), + ) + new = await h.store.read_reporting_feed(feed_request(s, limit=100), caller=s.binding.principal) + for array in ("periods", "revisions", "adjustments"): + assert old[array] == new[array] + assert old["materializations"] == old["receipts"] == [] + assert all(p["reconciliation_mode"] == "delivery_only" for p in new["periods"]) + + +@pytest.mark.parametrize("empty_filter", [False, True]) +async def test_empty_final_page_still_supplies_checkpoint_for_complete_consumption( + feeds, empty_filter +): + h = feeds + s, _, _ = await mixed_case(h) + extra = {"delivery_config_ids": ["unmatched"]} if empty_filter else {} + _, _, before = await walk(h.store, feed_request(s, **extra), s.binding.principal) + pages, rows, after = await walk( + h.store, feed_request(s, changes_after=before, **extra), s.binding.principal + ) + assert len(pages) == 1 and sum(map(len, rows.values())) == 0 + assert pages[0]["pagination"] == {"total_count": 0, "has_more": False} + assert after and len(after) <= 2048 diff --git a/tests/conformance/reporting/test_reporting_feed_history.py b/tests/conformance/reporting/test_reporting_feed_history.py new file mode 100644 index 000000000..6b84309bf --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_history.py @@ -0,0 +1,464 @@ +"""Historical selections, receipts, ownership and mutable Core boundaries.""" + +import json +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.feed import ReportingFeedError +from adcp.reporting.ledger import ( + ReportingControlTotalRecord, + ReportingMaterializationCheck, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.ledger.status_projection import mismatch_key + +from ._durable_materializer_support import durable_case +from ._feed_support import ( + ARRAYS, + feed_harness, + feed_request, + feeds, + mixed_case, + restart, + walk, + without_feed, +) +from ._receipt_support import adjustment_for, extra_materialization, receipt_case, request_for +from .test_reporting_notification_outbox import statement + +__all__ = ["feeds"] + + +@pytest.mark.parametrize("feedback", [False, True]) +async def test_legacy_status_without_obligation_id_closes_over_its_exact_revision(feeds, feedback): + from adcp.reporting.ledger.status import _consumer_status_to_wire + + h = feeds + s, _, _ = await mixed_case(h) + _, _, checkpoint = await walk(h.store, feed_request(s), s.binding.principal) + legacy = replace( + statement(s.obligation, s.binding.consumer_id), + consumer_status="received", + reporting_obligation_id=None, + reporting_revision_id=s.revision.reporting_revision_id, + observed_revision_content_sha256=s.revision.revision_content_sha256, + ) + # The approved public Core API accepts a revision without repeating its + # owner's ID. Its exact revision record supplies the dependency, never scope. + await h.store.record_consumer_status_with_lifecycle(legacy) + before = without_feed(await h.image()) + pages, rows, _ = await walk( + h.store, + feed_request(s, changes_after=checkpoint), + s.binding.principal, + consumer_status_enabled=feedback, + ) + assert len(pages) == 3 + assert rows["consumer_statuses"] == [_consumer_status_to_wire(legacy)] + assert rows["periods"][0]["reporting_obligation_id"] == s.obligation.reporting_obligation_id + assert rows["revisions"][0]["reporting_revision_id"] == s.revision.reporting_revision_id + snapshot = await h.store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=s.binding.principal + ) + member = next( + entry + for entry in snapshot.inputs["dependency_membership"] + if entry["record"] == ["consumer_status", legacy.reporting_status_id] + ) + assert member["dependencies"] == [ + ["obligation", s.obligation.reporting_obligation_id], + ["revision", s.revision.reporting_revision_id], + ] + assert without_feed(await h.image()) == before + + +async def test_snapshot_restatement_and_official_keep_exact_receipt_adjustment_targets( + feeds, +): + h = feeds + case = await durable_case(h.store, count=3, reconciliation_mode="consumer_receipt") + assert (await case.service().run_once()).state == "verified" + request = { + "account": {"account_id": case.config.account_id}, + "view": "periods", + "pagination": {"max_results": 1}, + } + _, initial, checkpoint = await walk(h.store, request, case.scope.principal) + assert len(initial["materializations"]) == 1 + restatement = await case.publish( + "restated-snapshot", finality="snapshot", supersedes=case.revision.reporting_revision_id + ) + assert (await case.service().run_once()).state == "verified" + outcomes = await case.outcomes() + retained_outcome = next( + r for r in outcomes if r.reporting_revision_id == restatement.reporting_revision_id + ) + evidence = retained_outcome.verification + receipt = ReportingRevisionReceiptRecord( + case.scope, + "accepted-snapshot-artifact", + restatement.reporting_revision_id, + retained_outcome.reporting_materialization_id, + "accepted", + evidence.verification_profile, + evidence.row_count, + evidence.control_totals, + retained_outcome.completed_at, + observed_canonical_content_digest=evidence.canonical_content_digest, + ) + admitted = await h.store.ingest_receipt_batch( + { + "account": request["account"], + "idempotency_key": "snapshot-receipt", + "receipts": [receipt_to_wire(receipt)], + }, + caller=case.scope.principal, + ) + assert admitted["results"][0]["result"] == "recorded" + official = await case.publish() + assert official.supersedes_reporting_revision_id is None + official_case = replace(case, revision=official) + adjustment = await adjustment_for( + h, + official_case, + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", case.obligation.currency), + ), + ) + admitted = await h.store.ingest_receipt_batch( + { + "account": request["account"], + "idempotency_key": "official-and-adjustment", + "adjustment_receipts": [adjustment], + }, + caller=case.scope.principal, + ) + assert all(r["result"] == "recorded" for r in admitted["results"]) + before = without_feed(await h.image()) + pages, rows, _ = await walk( + h.store, {**request, "changes_after": checkpoint}, case.scope.principal + ) + snapshot = await h.store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=case.scope.principal + ) + assert len(rows["periods"]) == 1 + assert {r["reporting_revision_id"] for r in rows["revisions"]} == { + case.revision.reporting_revision_id, + official.reporting_revision_id, + restatement.reporting_revision_id, + } + assert ( + snapshot.inputs["selections"][case.obligation.reporting_obligation_id][ + "reporting_revision_id" + ] + == official.reporting_revision_id + ) + assert rows["receipts"][0]["reporting_revision_id"] == restatement.reporting_revision_id + assert rows["adjustments"][0]["adjusts_reporting_revision_id"] == official.reporting_revision_id + assert ( + rows["adjustment_receipts"][0]["adjusts_reporting_revision_id"] + == official.reporting_revision_id + ) + assert {r["reporting_revision_id"] for r in rows["materializations"]} == { + restatement.reporting_revision_id + } + assert ( + len(snapshot.inputs["materializer_boundaries"]) + == len(snapshot.inputs["receipt_boundaries"]) + == 2 + ) + assert len(snapshot.inputs["revision_ownership"]) == 3 + assert {b["reporting_obligation_id"] for b in snapshot.inputs["revision_ownership"]} == { + case.obligation.reporting_obligation_id + } + assert all("ext" not in page for page in pages) + assert sum(len(rows[array]) for array in ARRAYS) == snapshot.total_count + assert without_feed(await h.image()) == before + # The newest official has no artifact: older accepted evidence stays intact + # and cannot be mistaken for evidence of the current selection in B2.4. + assert not any( + r.reporting_revision_id == official.reporting_revision_id for r in await case.outcomes() + ) + + +async def test_rejected_adjustment_digest_and_replacement_chain_are_preserved_without_retry(feeds): + h = feeds + s = await receipt_case(h) + item = await adjustment_for(h, s) + rejected = { + **item, + "status": "rejected", + "rejection_codes": ["LOAD_FAILED"], + "observed_adjustment_sha256": "f" * 64, + } + request = request_for(s, adjustment_receipts=[rejected]) + admitted = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert all(r["result"] == "recorded" for r in admitted["results"]) + before = without_feed(await h.image()) + first = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + original = await walk(h.store, feed_request(s), s.binding.principal, first=first) + assert original[1]["adjustment_receipts"][0]["observed_adjustment_sha256"] == "f" * 64 + assert without_feed(await h.image()) == before + accepted = { + **item, + "reporting_receipt_id": "accepted-adjustment-leaf", + "supersedes_reporting_receipt_id": item["reporting_receipt_id"], + } + response = await h.store.ingest_receipt_batch( + { + "account": request["account"], + "idempotency_key": "replacement-only", + "adjustment_receipts": [accepted], + }, + caller=s.binding.principal, + ) + assert response["results"][0]["result"] == "recorded" + before = without_feed(await h.image()) + assert ( + await walk(await restart(h), feed_request(s), s.binding.principal, first=first) == original + ) + _, delta, _ = await walk( + h.store, feed_request(s, changes_after=original[2]), s.binding.principal + ) + assert [r["status"] for r in delta["adjustment_receipts"]] == ["rejected", "accepted"] + assert len(delta["adjustments"]) == len(delta["revisions"]) == len(delta["periods"]) == 1 + assert without_feed(await h.image()) == before + + +@pytest.mark.parametrize("feedback", [False, True]) +async def test_page_one_freezes_issue_waiver_status_replacement_configuration_and_clock( + feeds, feedback +): + h = feeds + s, _, _ = await mixed_case(h) + bad = replace( + statement(s.obligation), + consumer_status="unreadable", + reporting_revision_id=s.revision.reporting_revision_id, + failure_code="access_denied", + ) + await h.store.record_consumer_status_with_lifecycle(bad) + issue = await h.store.get_issue(account_id=s.obligation.account_id, issue_key=mismatch_key(bad)) + assert issue is not None + req = feed_request(s) + first = await h.store.read_reporting_feed( + req, caller=s.binding.principal, consumer_status_enabled=feedback + ) + original = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + expected = await walk( + h.store, req, s.binding.principal, first=first, consumer_status_enabled=feedback + ) + await h.store.set_issue_state( + account_id=s.obligation.account_id, issue_key=issue.issue_key, state="waived", at=h.clock() + ) + good = replace( + bad, + reporting_status_id="consumer-status-replacement", + consumer_status="received", + supersedes_reporting_status_id=bad.reporting_status_id, + failure_code=None, + observed_revision_content_sha256=s.revision.revision_content_sha256, + ) + await h.store.record_consumer_status_with_lifecycle(good) + configs = await h.store.list_configurations(account_id=s.obligation.account_id) + await h.store.put_configuration( + replace( + configs[0], + deactivated_at=None, + status_retention_days=configs[0].status_retention_days + 10, + ) + ) + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.delivery.scope, + s.outcome.reporting_materialization_id, + "later-unavailable-check", + "unavailable", + h.clock(), + ) + ) + h.clock.now += timedelta(days=500) + store = await restart(h) + before = without_feed(await h.image()) + assert ( + await walk( + store, req, s.binding.principal, first=first, consumer_status_enabled=not feedback + ) + == expected + ) + assert ( + await store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + == original + ) + assert [r["reporting_status_id"] for r in expected[1]["consumer_statuses"]] == [ + bad.reporting_status_id + ] + assert original.inputs["core"]["lifecycles"] + assert original.inputs["readable_materializations"][s.outcome.reporting_materialization_id] + assert without_feed(await h.image()) == before + + +@pytest.mark.parametrize("later", ["success", "failure", "expired", "corrupt"]) +async def test_accepted_artifact_and_canonical_evidence_remain_frozen_after_later_outcomes( + feeds, later, monkeypatch +): + h = feeds + s, _, response = await mixed_case(h) + first = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + original = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + if later == "success": + await extra_materialization(h, s, "later-success", 2) + elif later == "failure": + await h.store.commit_materialization_attempt( + replace(s.attempt, reporting_materialization_id="later-failure", attempt=2) + ) + await h.store.commit_materialization( + replace( + s.outcome, + reporting_materialization_id="later-failure", + status="failed", + resource=None, + verification=None, + failure_code="WRITE_FAILED", + ) + ) + elif later == "expired": + # A deterministic future clock boundary; immutable resource expiry is + # preserved. Actual unmodified DB-time selection has a separate vector. + h.clock.now += timedelta(days=500) + if h.pool is not None: + from adcp.reporting.feed import pg as feed_pg + + async def future(connection): + return ( + await ( + await connection.execute("SELECT clock_timestamp() + interval '500 days'") + ).fetchone() + )[0] + + monkeypatch.setattr(feed_pg, "_now", future) + else: + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.delivery.scope, + s.outcome.reporting_materialization_id, + "later-health", + later, + h.clock(), + ) + ) + before = without_feed(await h.image()) + _, rows, _ = await walk(await restart(h), feed_request(s), s.binding.principal, first=first) + assert rows["receipts"][0] == response["results"][0]["receipt"] + assert len(rows["materializations"]) == 1 + assert ( + rows["materializations"][0]["reporting_materialization_id"] + == s.outcome.reporting_materialization_id + ) + assert original.inputs["readable_materializations"][s.outcome.reporting_materialization_id] + fresh = await h.store.read_reporting_feed( + feed_request(s, limit=100), caller=s.binding.principal + ) + assert fresh["receipts"] == rows["receipts"] + if later in {"expired", "corrupt"}: + after = await h.store.read_reporting_feed_snapshot( + fresh["ledger_snapshot_id"], caller=s.binding.principal + ) + assert not after.inputs["readable_materializations"][s.outcome.reporting_materialization_id] + assert without_feed(await h.image()) == before + + +async def test_late_committed_backdated_check_cannot_rewrite_receipt_admission(feeds): + h = feeds + s, request, response = await mixed_case(h) + first = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + original = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + # This is a valid later commit with an earlier observation time. Immutable + # acceptance must use the evidence committed when the receipt was admitted. + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.delivery.scope, + s.outcome.reporting_materialization_id, + "late-backdated-check", + "unavailable", + s.receipt.observed_at, + ) + ) + before = without_feed(await h.image()) + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + fresh = await h.store.read_reporting_feed( + feed_request(s, limit=100), caller=s.binding.principal + ) + assert fresh["receipts"] == [response["results"][0]["receipt"]] + frozen = await h.store.read_reporting_feed_snapshot( + fresh["ledger_snapshot_id"], caller=s.binding.principal + ) + assert not frozen.inputs["readable_materializations"][s.outcome.reporting_materialization_id] + assert original.inputs["readable_materializations"][s.outcome.reporting_materialization_id] + assert original.inputs["receipt_boundaries"] == frozen.inputs["receipt_boundaries"] + assert ( + await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + == original + ) + assert without_feed(await h.image()) == before + + +async def test_conflicting_owner_in_retained_receipt_boundary_fails_new_capture_only(feeds): + h = feeds + s, _, _ = await mixed_case(h) + first = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + if h.pool is None: + boundary = h.store._receipt_boundaries[0] + bad_revision = replace( + boundary.core.revisions[0], reporting_obligation_id="different-owner" + ) + h.store._receipt_boundaries[0] = replace( + boundary, core=replace(boundary.core, revisions=(bad_revision,)) + ) + else: + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role=replica") + await c.execute( + "UPDATE reporting_receipt_ingestion_boundaries SET input=jsonb_set(input," + "'{core,revisions,0,reporting_obligation_id}',%s::jsonb)," + " content_sha256=reporting_receipt_ingestion_sha256(jsonb_set(input," + "'{core,revisions,0,reporting_obligation_id}',%s::jsonb)) WHERE sequence=1", + (json.dumps("different-owner"), json.dumps("different-owner")), + ) + store = await restart(h) + before = await h.image() + assert (await walk(store, feed_request(s), s.binding.principal, first=first))[1]["receipts"] + with pytest.raises(ReportingFeedError) as error: + await store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + assert error.value.code == "REPORTING_FEED_HISTORY_CORRUPT" + assert await h.image() == before + + +async def test_memory_feed_owner_tag_cannot_relabel_another_callers_history(): + async with feed_harness("memory") as h: + s, _, _ = await mixed_case(h) + seq, owner, record = h.store._delivery_records[0] + h.store._delivery_records[0] = (seq, replace(owner, consumer_id="wrong-owner"), record) + before = await h.image() + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + assert error.value.code == "REPORTING_FEED_HISTORY_CORRUPT" + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_feed_installed_pg.py b/tests/conformance/reporting/test_reporting_feed_installed_pg.py new file mode 100644 index 000000000..a2067eb4b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_installed_pg.py @@ -0,0 +1,225 @@ +"""Actual Python 3.10 distributions: mounted page one, SIGKILL, changed state.""" + +import asyncio +import hashlib +import json +import os +import shutil +import sys +from dataclasses import replace +from datetime import timedelta +from pathlib import Path + +import pytest + +from adcp.reporting.feed import PgReportingFeedStore +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger.status_projection import mismatch_key + +from ._durable_materializer_support import DurableHarness +from ._feed_support import feed_request, mixed_case, second_consumer, walk, without_feed +from ._generation_support import assert_c_collated_rolling_database, isolated_reporting_pool +from ._reconciliation_support import Clock +from .test_reporting_feed_packaging import ( + ROOT, + b1_wheels, + built_distribution, + feed_modules, + feed_wheels, + run_step, +) +from .test_reporting_feed_process import feed_process +from .test_reporting_notification_outbox import statement + +__all__ = ["b1_wheels", "built_distribution", "feed_wheels"] + + +@pytest.fixture(scope="module", params=["vcs", "sdist"]) +def installed_feed(request): + assert_c_collated_rolling_database() + root, wheels, assets = request.getfixturevalue("feed_wheels") + interpreter = os.environ.get("ADCP_PYTHON310") or sys.executable + environment = root / f"feed-pg-{request.param}" + run_step( + [interpreter, "-m", "venv", str(environment)], + label=f"feed-{request.param}-pg-environment", + cwd=root, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, f"{wheels[request.param]}[pg]", "asgi-lifespan==2.1.0"], + label=f"feed-{request.param}-pg-install", + cwd=root, + timeout=180, + ) + version = json.loads( + run_step( + [str(python), "-I", "-c", "import json,sys;print(json.dumps(sys.version_info[:2]))"], + label="feed-installed-version", + cwd=root, + ) + ) + if os.environ.get("ADCP_PYTHON310"): + assert version == [3, 10] + script, helper = root / "feed_process.py", root / "feed_transport.py" + shutil.copy2(Path(__file__).with_name("_feed_process.py"), script) + shutil.copy2(Path(__file__).with_name("_receipt_transport.py"), helper) + installed = { + "workspace": str(ROOT), + "python": version, + "modules": feed_modules(), + "assets": assets, + "distribution": request.param, + "wheel_sha256": hashlib.sha256(wheels[request.param].read_bytes()).hexdigest(), + } + return root, python, script, helper, installed + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_installed_python310_cold_continuation_freezes_mutable_history_and_receipt_replay( + installed_feed, notifications +): + root, python, script, helper, installed = installed_feed + async with isolated_reporting_pool(autocommit=True) as pool: + ready = json.loads( + await asyncio.to_thread( + run_step, + [str(python), "-I", str(script)], + label="feed-installed-sql", + cwd=root, + value={ + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "notifications": notifications, + "action": "install", + "installed": installed, + }, + timeout=90, + ) + ) + assert ready["result"] == { + "installed": True, + "materializer_objects": 187, + "receipt_objects": 102, + "feed_objects": 33, + } + h = DurableHarness( + PgReportingFeedStore(pool=pool, notifications=notifications), Clock(), pool + ) + s, receipt_request, receipt_response = await mixed_case( + h, consumer_id="https://buyer.example.test/installed" + ) + bad = replace( + statement(s.obligation, s.binding.consumer_id), + reporting_status_id="installed-consumer-before", + consumer_status="unreadable", + reporting_revision_id=s.revision.reporting_revision_id, + failure_code="access_denied", + ) + await h.store.record_consumer_status_with_lifecycle(bad) + config = (await h.store.list_configurations(account_id=s.obligation.account_id))[0] + await h.store.put_configuration(replace(config, deactivated_at=None)) + options = {"python": python, "script": script, "helper": helper, "installed": installed} + async with feed_process(h, s, feed_request(s), pause="committed", **options) as child: + first = (await child.event("committed"))["result"] + await child.kill() + original = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + expected = await walk(h.store, feed_request(s), s.binding.principal, first=first) + issue = await h.store.get_issue( + account_id=s.obligation.account_id, issue_key=mismatch_key(bad) + ) + await h.store.set_issue_state( + account_id=s.obligation.account_id, + issue_key=issue.issue_key, + state="waived", + at=h.clock(), + ) + await h.store.record_consumer_status_with_lifecycle( + replace( + bad, + reporting_status_id="installed-consumer-after", + consumer_status="received", + supersedes_reporting_status_id=bad.reporting_status_id, + failure_code=None, + observed_revision_content_sha256=s.revision.revision_content_sha256, + ) + ) + await h.store.put_configuration( + replace( + config, + deactivated_at=h.clock(), + status_retention_days=config.status_retention_days + 10, + ) + ) + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.delivery.scope, + s.outcome.reporting_materialization_id, + "installed-corrupt", + "corrupt", + h.clock(), + ) + ) + # Seed the foreign artifact before its source becomes unreadable. + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=True, + ) + await second_consumer(h, s, "https://buyer.example.test/foreign-installed") + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + h.clock.now += timedelta(days=500) + await h.store.create_schema() + before = without_feed(await h.image()) + continuation = feed_request( + s, pagination={"cursor": first["pagination"]["cursor"], "max_results": 1} + ) + for v1 in (False, True): + async with feed_process( + h, s, continuation, action="walk", transport="a2a", v1=v1, feedback=True, **options + ) as child: + done = await child.event("done") + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + assert done["result"]["pages"] == expected[0][1:] + assert done["result"]["binding"] == original.binding + assert done["result"]["version"] == 1 and done["result"]["ownership_mode"] == "absent" + assert done["origins"] == ready["origins"] + assert ( + await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + == original + ) + assert ( + await h.store.ingest_receipt_batch(receipt_request, caller=s.binding.principal) + == receipt_response + ) + assert without_feed(await h.image()) == before + print( + json.dumps( + { + "feed_installed": installed, + "notifications": notifications, + "origins": done["origins"], + "pages": len(expected[0]), + "token_length": len(first["changes_checkpoint"]), + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_feed_migration.py b/tests/conformance/reporting/test_reporting_feed_migration.py new file mode 100644 index 000000000..a3a333e87 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_migration.py @@ -0,0 +1,194 @@ +"""Isolated feed migration, immutable snapshots, unchanged upstream catalogs.""" + +import asyncio +import json +from importlib.resources import files + +import pytest + +from adcp.reporting.feed import PgReportingFeedStore, ReportingFeedError +from adcp.reporting.materializer import ReportingMaterializerLease +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts import PgReportingReceiptStore + +from ._durable_materializer_support import DurableHarness, durable_case +from ._feed_support import feed_harness, feed_request, mixed_case, walk, without_feed +from ._generation_support import isolated_reporting_pool +from ._receipt_support import receipt_case, request_for +from ._reconciliation_support import Clock + +SQL = files("adcp.reporting.ledger").joinpath("reporting_feed.sql").read_text() +MANIFEST = json.loads(files("adcp.reporting.feed").joinpath("required_schema.json").read_text()) + + +async def fairness(pool): + async with pool.connection() as c: + return await ( + await c.execute( + "SELECT to_jsonb(t) FROM adcp_reporting_configuration_lease_turns t" + " ORDER BY to_jsonb(t)::text" + ) + ).fetchall() + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_feed_migration_preserves_parent_catalog_receipts_pending_and_fairness( + notifications, autocommit +): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + parent = PgReportingReceiptStore(pool=pool, notifications=notifications) + await parent.create_schema() + h = DurableHarness(parent, Clock(), pool) + s = await receipt_case(h) + req = request_for(s) + response = await parent.ingest_receipt_batch(req, caller=s.binding.principal) + pending = await durable_case(parent, account="pending-account") + for _ in range(8): + lease = await pending.claim() + if isinstance(lease, ReportingMaterializerLease): + break + assert isinstance(lease, ReportingMaterializerLease) and lease.scope == pending.scope + assert ( + lease.attempt.reporting_materialization_id == lease.request.reporting_materialization_id + ) + turn = await parent.lease_period_close( + worker_id="feed-upgrade", now=h.clock(), lease_seconds=30 + ) + assert turn is not None + await parent.release_period_close(turn, worker_id="feed-upgrade") + turns = await fairness(pool) + before = await h.image() + async with pool.connection() as c: + original = await schema_objects(c) + assert len(original) == 742 + new = PgReportingFeedStore(pool=pool, notifications=notifications) + with pytest.raises(ReportingFeedError) as error: + await new.reporting_feed_ready() + assert error.value.code == "REPORTING_FEED_SCHEMA_UNREADY" + await asyncio.gather(*(new.create_schema() for _ in range(3))) + assert await new.reporting_feed_ready() + async with pool.connection() as c: + actual = await schema_objects(c) + assert {k: actual[k] for k in original} == original + assert {k: v for k, v in actual.items() if "reporting_feed_" in k} == MANIFEST + assert len(MANIFEST) == 33 + for package, count in (("materializer", 187), ("receipts", 102)): + required = json.loads( + files("adcp.reporting." + package).joinpath("required_schema.json").read_text() + ) + assert len(required) == count and all(actual[k] == v for k, v in required.items()) + assert without_feed(await h.image()) == before + assert await fairness(pool) == turns + first = await new.read_reporting_feed(feed_request(s), caller=s.binding.principal) + saved = await new.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=s.binding.principal + ) + await new.create_schema() + assert ( + await new.read_reporting_feed_snapshot(saved.snapshot_id, caller=s.binding.principal) + == saved + ) + assert await parent.ingest_receipt_batch(req, caller=s.binding.principal) == response + assert await fairness(pool) == turns + assert await parent.materializer_ready() and await parent.receipt_ingestion_ready() + + +async def test_interrupted_feed_migration_is_invisible_and_retry_retains_history(): + async with isolated_reporting_pool(autocommit=True) as pool: + parent = PgReportingReceiptStore(pool=pool, clock=Clock()) + await parent.create_schema() + h = DurableHarness(parent, parent._clock, pool) + s = await receipt_case(h) + response = await parent.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + before = await h.image() + entered, release = asyncio.Event(), asyncio.Event() + + async def migrate(): + async with pool.connection() as c, c.transaction(): + await c.execute(SQL) + entered.set() + await release.wait() + + task = asyncio.create_task(migrate()) + try: + await asyncio.wait_for(entered.wait(), 10) + async with pool.connection() as c: + assert ( + await ( + await c.execute("SELECT to_regclass('reporting_feed_snapshots')") + ).fetchone() + )[0] is None + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert await h.image() == before + new = PgReportingFeedStore(pool=pool) + with pytest.raises(ReportingFeedError): + await new.reporting_feed_ready() + await new.create_schema() + assert ( + await parent.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + == response + ) + assert without_feed(await h.image()) == before + + +@pytest.mark.parametrize( + "damage", + [ + "ALTER TABLE reporting_feed_snapshots DISABLE TRIGGER reporting_feed_immutable", + "ALTER TABLE reporting_feed_snapshots ALTER COLUMN document DROP NOT NULL", + "DROP TABLE reporting_feed_snapshots", + "CREATE OR REPLACE FUNCTION reporting_feed_immutable() RETURNS TRIGGER" + " LANGUAGE plpgsql AS $body$ BEGIN RETURN NEW; END $body$", + "ALTER TABLE reporting_receipt_ingestion_results" + " DISABLE TRIGGER reporting_receipt_ingestion_result", + "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", + ], +) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_old_partial_or_mismatched_schema_refuses_new_reads_and_continuations( + damage, notifications +): + async with feed_harness("postgres", notifications=notifications) as h: + s, _, _ = await mixed_case(h) + first = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + async with h.pool.connection() as c: + await c.execute(damage) + before = await h.image() + for req in ( + feed_request(s), + feed_request(s, pagination={"cursor": first["pagination"]["cursor"]}), + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(req, caller=s.binding.principal) + assert error.value.code == "REPORTING_FEED_SCHEMA_UNREADY" + assert await h.image() == before + + +@pytest.mark.parametrize( + "assignment", + [ + "ownership_mode='present'", + "representation_version=2", + "signing_key=decode(repeat('00',32),'hex')", + "document=document", + ], +) +async def test_snapshot_membership_representation_ownership_and_signing_identity_are_immutable( + assignment, +): + async with feed_harness("postgres") as h: + s, _, _ = await mixed_case(h) + first = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + before = await h.image() + import psycopg + + async with h.pool.connection() as c: + with pytest.raises(psycopg.errors.CheckViolation): + await c.execute("UPDATE reporting_feed_snapshots SET " + assignment) + assert await h.image() == before + pages, _, _ = await walk(h.store, feed_request(s), s.binding.principal, first=first) + assert pages[-1]["pagination"]["total_count"] == 6 diff --git a/tests/conformance/reporting/test_reporting_feed_packaging.py b/tests/conformance/reporting/test_reporting_feed_packaging.py new file mode 100644 index 000000000..332c04525 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_packaging.py @@ -0,0 +1,142 @@ +"""VCS and sdist installed Python 3.10, separate feed assets and strict adopter.""" + +import hashlib +import json +import os +import shutil +import sys +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from .test_reporting_materializer_packaging import ROOT, b1_wheels, built_distribution, run_step + +__all__ = ["b1_wheels", "built_distribution"] + + +def feed_modules(): + paths = list((ROOT / "src/adcp/reporting/feed").glob("*.py")) + paths.extend( + ROOT / "src/adcp" / name + for name in ( + "reporting/receipts/handler.py", + "reporting/receipts/transport.py", + "server/mcp_tools.py", + "server/a2a_server.py", + "server/serve.py", + "server/idempotency/store.py", + ) + ) + return { + "adcp." + + str(p.relative_to(ROOT / "src/adcp")) + .removesuffix(".py") + .replace("/", ".") + .removesuffix(".__init__"): hashlib.sha256(p.read_bytes()) + .hexdigest() + for p in paths + } + + +@pytest.fixture(scope="module") +def feed_wheels(b1_wheels, built_distribution): + root, wheels, _ = b1_wheels + _, _, source = built_distribution + assets = { + name: (ROOT / "src/adcp/reporting" / name).read_bytes() + for name in ( + "ledger/reporting_feed.sql", + "feed/required_schema.json", + "ledger/reporting_materializer.sql", + "materializer/required_schema.json", + "ledger/reporting_receipt_ingestion.sql", + "receipts/required_schema.json", + ) + } + with tarfile.open(source) as sdist: + prefix = sdist.getnames()[0].split("/")[0] + for name, raw in assets.items(): + assert sdist.extractfile(f"{prefix}/src/adcp/reporting/{name}").read() == raw + for wheel in wheels.values(): + with zipfile.ZipFile(wheel) as archive: + for name, raw in assets.items(): + assert archive.read("adcp/reporting/" + name) == raw + return root, wheels, {name: hashlib.sha256(raw).hexdigest() for name, raw in assets.items()} + + +@pytest.mark.parametrize("kind", ["vcs", "sdist"]) +def test_python310_feed_without_pg_exports_sql_and_strict_adopter(request, kind): + interpreter = os.environ.get("ADCP_PYTHON310") or ( + sys.executable if sys.version_info[:2] == (3, 10) else None + ) + if interpreter is None: + pytest.skip("Python 3.10 matrix owns this cell; ADCP_PYTHON310 enables it locally") + root, wheels, assets = request.getfixturevalue("feed_wheels") + environment = root / f"feed-base-{kind}" + run_step( + [interpreter, "-m", "venv", str(environment)], + label=f"feed-{kind}-base-environment", + cwd=root, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, str(wheels[kind]), "mypy==1.20.2"], + label=f"feed-{kind}-base-install", + cwd=root, + timeout=180, + ) + script = root / f"feed_base_{kind}.py" + shutil.copy2(Path(__file__).with_name("_feed_installed_base.py"), script) + settings = {"workspace": str(ROOT), "assets": assets, "modules": feed_modules()} + result = json.loads( + run_step( + [str(python), "-I", str(script)], + label=f"feed-{kind}-base-smoke", + cwd=root, + value=settings, + timeout=120, + ) + ) + assert result["driver_absent"] and result["python"] == "3.10" + adopter = root / f"feed_adopter_{kind}.py" + example = root / f"feed_example_{kind}.py" + shutil.copy2(ROOT / "tests/type_checks/reporting_frozen_feed.py", adopter) + shutil.copy2(ROOT / "examples/reporting_receipt_ingress.py", example) + config = root / f"feed_mypy_{kind}.ini" + config.write_text( + "[mypy]\npython_version=3.10\nstrict=True\nplugins=adcp.types.mypy_plugin\nfollow_imports=silent\n" + ) + run_step( + [ + str(python), + "-I", + "-m", + "mypy", + "--config-file", + str(config), + "--strict", + "--no-incremental", + str(adopter), + str(example), + ], + label=f"feed-{kind}-base-adopter", + cwd=root, + timeout=120, + ) + print( + json.dumps( + { + "feed_base_install": kind, + "wheel_sha256": hashlib.sha256(wheels[kind].read_bytes()).hexdigest(), + **result, + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_feed_positions.py b/tests/conformance/reporting/test_reporting_feed_positions.py new file mode 100644 index 000000000..6763d08b9 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_positions.py @@ -0,0 +1,216 @@ +"""Opaque positions bind the whole frozen vector and semantic request.""" + +import base64 +import json +from copy import deepcopy +from dataclasses import replace + +import pytest + +from adcp.reporting.feed import ReportingFeedError +from adcp.reporting.feed.request import FeedRequest +from adcp.reporting.feed.snapshot import token_position +from adcp.reporting.ledger.store import encode_cursor +from adcp.types import GetReportingStatusRequest, GetReportingStatusResponse + +from ._feed_support import feed_request, feeds, mixed_case, restart, walk, without_feed + +__all__ = ["feeds"] + + +async def test_every_semantic_filter_and_both_principal_coordinates_are_bound(feeds): + h = feeds + s, _, _ = await mixed_case(h) + request = feed_request(s) + page = await h.store.read_reporting_feed(request, caller=s.binding.principal) + tokens = (page["pagination"]["cursor"], page["changes_checkpoint"]) + variations = [ + {"delivery_config_ids": ["other"]}, + {"media_buy_ids": ["other"]}, + {"feed_purposes": ["analytics"]}, + {"health": ["healthy"]}, + {"finality": ["snapshot"]}, + {"period": {"start": "2026-09-01T00:00:00Z", "end": "2026-09-02T00:00:00Z"}}, + {"ext": {"vendor": {"scope": "changed"}}}, + ] + before = await h.image() + for token, parameter in zip(tokens, ("cursor", "changes_after")): + for changes in variations: + changed = feed_request(s, **changes) + if parameter == "cursor": + changed["pagination"]["cursor"] = token + else: + changed[parameter] = token + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(changed, caller=s.binding.principal) + assert error.value.code == "INVALID_CHECKPOINT" + for caller in ( + replace(s.binding.principal, account_id="other-account"), + replace(s.binding.principal, consumer_id="other-consumer"), + ): + changed = feed_request(s) + if parameter == "cursor": + changed["pagination"]["cursor"] = token + else: + changed[parameter] = token + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(changed, caller=caller) + assert error.value.code == "INVALID_CHECKPOINT" + assert await h.image() == before + + +async def test_tampered_positions_cannot_rebind_offset_boundary_count_or_last_key(feeds): + h = feeds + s, _, _ = await mixed_case(h) + page = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + original = page["pagination"]["cursor"] + packed = base64.urlsafe_b64decode(original[5:] + "=" * (-len(original[5:]) % 4)) + fields = json.loads(packed[:-32]) + for index, value in [ + (0, True), + (0, 2), + (1, "checkpoint"), + (2, "rpfs_" + "0" * 32), + (3, -1), + (3, True), + (3, 2), + (4, "f" * 64), + (5, "0" * 64), + ]: + changed = deepcopy(fields) + changed[index] = value + forged = "rpf1." + base64.urlsafe_b64encode( + json.dumps(changed, separators=(",", ":")).encode() + packed[-32:] + ).decode().rstrip("=") + req = feed_request(s, pagination={"cursor": forged}) + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(req, caller=s.binding.principal) + assert error.value.code == "INVALID_CHECKPOINT" + # Either type in the wrong field fails, even if fully authenticated. + for req in ( + feed_request(s, changes_after=original), + feed_request(s, pagination={"cursor": page["changes_checkpoint"]}), + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(req, caller=s.binding.principal) + assert error.value.code == "INVALID_CHECKPOINT" + + +async def test_unbound_legacy_and_malformed_tokens_are_explicitly_rejected(feeds): + h = feeds + s, _, _ = await mixed_case(h) + before = await h.image() + for token in ( + encode_cursor({"seq": 999999}), + encode_cursor({"feed": "reporting-reconciliation-v1", "seq": 0}), + "rpf1.invalid", + "rpf1." + "a" * 2043, + "", + "x" * 2049, + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed( + feed_request(s, changes_after=token), caller=s.binding.principal + ) + assert error.value.code in {"INVALID_REQUEST", "INVALID_CHECKPOINT"} + assert await h.image() == before + + +async def test_page_size_echo_context_and_normalized_filter_order_do_not_rebind(feeds): + h = feeds + s, _, _ = await mixed_case(h) + original = feed_request( + s, + delivery_config_ids=["daily", "unused"], + context={"trace": "first", "sample": 0.125}, + ext={"vendor": {"threshold": 0.5}}, + ) + first = await h.store.read_reporting_feed(original, caller=s.binding.principal) + continued = feed_request( + s, + delivery_config_ids=["unused", "daily", "daily"], + context={"trace": "second", "sample": 0.75}, + ext={"vendor": {"threshold": 0.5}}, + pagination={"cursor": first["pagination"]["cursor"], "max_results": 100}, + ) + # Duplicates are normalized only when the protocol accepts them. Generated + # schemas may require uniqueItems, so compare the canonical reordered set. + continued["delivery_config_ids"] = ["unused", "daily"] + second = await (await restart(h)).read_reporting_feed( + continued, caller=s.binding.principal, consumer_status_enabled=True + ) + assert second["changes_checkpoint"] == first["changes_checkpoint"] + assert second["ledger_snapshot_id"] == first["ledger_snapshot_id"] + assert second["context"] == {"trace": "second", "sample": 0.75} + assert second["pagination"] == {"total_count": 6, "has_more": False} + + +async def test_actual_generated_client_tokens_fit_maximum_supported_url_principal(feeds): + h = feeds + prefix = "https://buyer.example.test/" + consumer = prefix + "a" * (2048 - len(prefix)) + s, _, _ = await mixed_case(h, consumer_id=consumer) + request = GetReportingStatusRequest.model_validate(feed_request(s)).model_dump( + mode="json", exclude_unset=True + ) + page = GetReportingStatusResponse.model_validate( + await h.store.read_reporting_feed(request, caller=s.binding.principal) + ) + assert 0 < len(page.changes_checkpoint) <= 2048 + assert 0 < len(page.pagination.cursor) <= 2048 + assert consumer not in page.pagination.cursor + again = GetReportingStatusRequest.model_validate( + feed_request(s, pagination={"cursor": page.pagination.cursor}) + ).model_dump(mode="json", exclude_unset=True) + tail = GetReportingStatusResponse.model_validate( + await (await restart(h)).read_reporting_feed(again, caller=s.binding.principal) + ) + assert tail.changes_checkpoint == page.changes_checkpoint + assert tail.pagination.has_more is False + checkpoint_request = GetReportingStatusRequest.model_validate( + feed_request(s, changes_after=tail.changes_checkpoint) + ).model_dump(mode="json", exclude_unset=True) + empty = await h.store.read_reporting_feed(checkpoint_request, caller=s.binding.principal) + assert empty["pagination"]["total_count"] == 0 + + +async def test_original_changes_after_may_accompany_continuation_but_a_different_boundary_cannot( + feeds, +): + h = feeds + s, _, _ = await mixed_case(h) + # An empty filtered feed can advance through the same public vector, but + # changing filters never grants an incremental position in another scope. + from ._receipt_support import extra_materialization + + _, _, previous = await walk(h.store, feed_request(s), s.binding.principal) + await extra_materialization(h, s, "position-next-materialization", 2) + req = feed_request(s, changes_after=previous) + first = await h.store.read_reporting_feed(req, caller=s.binding.principal) + pages, _, checkpoint = await walk(h.store, req, s.binding.principal, first=first) + assert len(pages) == 3 + changed = feed_request( + s, changes_after=checkpoint, pagination={"cursor": first["pagination"]["cursor"]} + ) + before = without_feed(await h.image()) + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(changed, caller=s.binding.principal) + assert error.value.code == "INVALID_CHECKPOINT" + assert without_feed(await h.image()) == before + + +@pytest.mark.parametrize("value", [None, False, 0, 101, 1.0, "1"]) +def test_page_limit_requires_a_bounded_json_integer(value): + with pytest.raises(ReportingFeedError): + FeedRequest.parse( + { + "view": "periods", + "account": {"account_id": "acct_a"}, + "pagination": {"max_results": value}, + } + ) + + +def test_decoding_an_opaque_position_does_not_authorize_it(): + with pytest.raises(ReportingFeedError): + token_position(encode_cursor({"seq": 1})) diff --git a/tests/conformance/reporting/test_reporting_feed_process.py b/tests/conformance/reporting/test_reporting_feed_process.py new file mode 100644 index 000000000..1979cfcd7 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_process.py @@ -0,0 +1,86 @@ +"""Real process death before/after snapshot commit and mounted cold continuation.""" + +import asyncio +import sys +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest + +from ._feed_support import feed_harness, feed_request, mixed_case, walk, without_feed +from .test_reporting_materializer_process import Child + + +@asynccontextmanager +async def feed_process(h, s, request, *, python=None, script=None, helper=None, **options): + process = await asyncio.create_subprocess_exec( + str(python or sys.executable), + *(["-I"] if options.get("installed") else []), + str(script or Path(__file__).with_name("_feed_process.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "notifications": h.store._notifications_enabled, + "caller": { + "account_id": s.obligation.account_id, + "consumer_id": s.binding.consumer_id, + }, + "request": request, + "action": "page", + "helper": str(helper or Path(__file__).with_name("_receipt_transport.py")), + **options, + } + ) + yield child + finally: + await child.kill() + diagnostic = await process.stderr.read() + assert process.returncode in {0, -9}, (process.returncode, len(diagnostic)) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("point", ["captured", "inserted", "committed"]) +async def test_process_crash_preserves_receipts_and_cold_mcp_a2a_pages(point, notifications): + async with feed_harness("postgres", notifications=notifications) as h: + s, request, response = await mixed_case(h) + before = without_feed(await h.image()) + async with feed_process(h, s, feed_request(s), pause=point) as child: + boundary = await child.event(point) + # Observe MVCC from another connection while the child holds the + # account lock; no partial snapshot or receipt/history mutation leaks. + async with h.pool.connection() as c: + count = ( + await ( + await c.execute("SELECT count(*) FROM reporting_feed_snapshots") + ).fetchone() + )[0] + assert count == int(point == "committed") + await child.kill() + assert without_feed(await h.image()) == before + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + if point == "committed": + first = boundary["result"] + else: + async with feed_process(h, s, feed_request(s)) as child: + first = (await child.event("done"))["result"]["pages"][0] + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + expected = await walk(h.store, feed_request(s), s.binding.principal, first=first) + continuation = feed_request( + s, pagination={"max_results": 1, "cursor": first["pagination"]["cursor"]} + ) + for v1 in (False, True): + async with feed_process( + h, s, continuation, action="walk", transport="a2a", v1=v1, feedback=True + ) as child: + result = (await child.event("done"))["result"] + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + assert result["pages"] == expected[0][1:] + assert result["version"] == 1 and result["ownership_mode"] == "absent" + assert without_feed(await h.image()) == before diff --git a/tests/conformance/reporting/test_reporting_feed_rolling.py b/tests/conformance/reporting/test_reporting_feed_rolling.py new file mode 100644 index 000000000..3bf508f75 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_rolling.py @@ -0,0 +1,321 @@ +"""Nine actual historical binaries on exact B2.2 then the additive feed schema.""" + +import asyncio +import hashlib +import json +import shutil +import sys +from dataclasses import replace +from pathlib import Path + +import pytest + +from adcp.reporting.feed import PgReportingFeedStore +from adcp.reporting.ledger import ( + ReportingControlTotalRecord, + ReportingDeliveryScope, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts import PgReportingReceiptStore + +from ._durable_materializer_support import DurableHarness, durable_case +from ._feed_support import walk, without_feed +from ._generation_support import isolated_reporting_pool +from ._receipt_support import adjustment_for +from .test_reporting_feed_packaging import ROOT, run_step +from .test_reporting_materializer_process import worker +from .test_reporting_materializer_rolling import ARTIFACTS, build_frozen, frozen_call +from .test_reporting_receipt_rolling import B21, immutable_parent_rows, receipt_probe + +B22 = "74b338d81653f4f65bc1520e638da10225f199f3" +B22_TREE = "b694de7cb216b1ea8456f44ea9ccc9fb66257c79" + + +@pytest.fixture(scope="module") +def approved_feed_b21(tmp_path_factory, request): + return receipt_probe(build_frozen("b21", tmp_path_factory, request, sha=B21)) + + +@pytest.fixture(scope="module") +def approved_feed_b22(tmp_path_factory, request): + root, python, ordinary, original = receipt_probe( + build_frozen("b22", tmp_path_factory, request, sha=B22) + ) + modules = dict(original["modules"]) + for relative in ( + "reporting/receipts/pg.py", + "reporting/receipts/handler.py", + "reporting/receipts/wire.py", + "reporting/receipts/transport.py", + "server/mcp_tools.py", + "server/a2a_server.py", + "server/serve.py", + ): + content = run_step( + ["git", "show", f"{B22}:src/adcp/{relative}"], label="b22-exact-module", cwd=ROOT + ) + modules["adcp." + relative.removesuffix(".py").replace("/", ".")] = hashlib.sha256( + content.encode() + ).hexdigest() + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step([*installer, "asgi-lifespan==2.1.0"], label="b22-mounted-probe-dependency", cwd=root) + shutil.copy2(Path(__file__).with_name("_feed_process.py"), root / "feed_process.py") + shutil.copy2(Path(__file__).with_name("_receipt_transport.py"), root / "receipt_transport.py") + return root, python, ordinary, {**original, "modules": modules, "tree": B22_TREE} + + +@pytest.fixture(scope="module", params=(*ARTIFACTS, "b21", "b22")) +def installed_feed_history(request, tmp_path_factory, approved_feed_b21, approved_feed_b22): + if request.param == "b21": + return approved_feed_b21 + if request.param == "b22": + return approved_feed_b22 + return receipt_probe(build_frozen(request.param, tmp_path_factory, request)) + + +async def b22_mounted(artifact, pool, action, **kwargs): + root, python, _, settings = artifact + value = { + "installed": settings, + "receipt_only": True, + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "action": action, + "helper": str(root / "receipt_transport.py"), + **kwargs, + } + result = json.loads( + await asyncio.to_thread( + run_step, + [str(python), "-I", str(root / "feed_process.py")], + label=f"frozen-b22-mounted-{action}", + cwd=root, + value=value, + timeout=90, + ) + ) + assert result["point"] == "done", result + return result + + +async def other_artifact_consumer(parent, case, outcome): + consumer = "https://buyer.example.test/post-migration" + scope = ReportingDeliveryScope( + case.config.generation_key, consumer, case.obligation.reporting_obligation_id + ) + snapshot = await parent.read_reconciliation_snapshot(caller=case.scope.principal) + delivery = next(r for r in snapshot.records if r.kind == "obligation_delivery") + attempt = next( + r + for r in snapshot.records + if r.kind == "materialization_attempt" + and r.reporting_materialization_id == outcome.reporting_materialization_id + ) + await parent.put_destination_binding(replace(case.binding, consumer_id=consumer)) + await parent.bind_obligation_delivery(replace(delivery, scope=scope)) + await parent.commit_materialization_attempt(replace(attempt, scope=scope)) + await parent.commit_materialization(replace(outcome, scope=scope)) + return scope.principal + + +async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mounted_replay( + installed_feed_history, approved_feed_b21, approved_feed_b22, tmp_path +): + artifact = installed_feed_history[3]["artifact"] + modes = [False, True] if artifact in {"b", "c", "b1", "b21", "b22"} else [False] + for notifications in modes: + async with isolated_reporting_pool(autocommit=True) as pool: + installed = await b22_mounted( + approved_feed_b22, + pool, + "install", + notifications=notifications, + legacy_status_schema=True, + ) + assert installed["result"] == { + "installed": True, + "materializer_objects": 187, + "receipt_objects": 102, + } + parent = PgReportingReceiptStore(pool=pool, notifications=notifications) + case = await durable_case( + parent, + count=3, + finality="official", + required="official", + reconciliation_mode="consumer_receipt", + consumer=( + "frozen-buyer" + if artifact in {"beta15", "records", "integration", "a", "b"} + else "https://buyer.example.test/frozen-feed" + ), + legacy_definition=artifact == "beta15", + ) + root, python, _, settings = approved_feed_b21 + process_script = root / "feed_parent_materializer.py" + shutil.copy2(Path(__file__).with_name("_materializer_process.py"), process_script) + destination = tmp_path / f"destination-{notifications}" + destination.mkdir(mode=0o700) + h = DurableHarness(parent, None, pool) + async with worker( + h, + case, + destination, + python=python, + script=process_script, + installed={**settings, "python": list(sys.version_info[:2])}, + notifications=notifications, + ) as child: + produced = await child.event("done") + assert produced["state"] == "verified" + assert await asyncio.wait_for(child.process.wait(), 5) == 0 + outcome = (await case.outcomes())[0] + second = await other_artifact_consumer(parent, case, outcome) + evidence = outcome.verification + receipt = ReportingRevisionReceiptRecord( + case.scope, + "frozen-feed-revision-0001", + case.revision.reporting_revision_id, + outcome.reporting_materialization_id, + "accepted", + evidence.verification_profile, + evidence.row_count, + evidence.control_totals, + outcome.completed_at, + observed_canonical_content_digest=evidence.canonical_content_digest, + ) + adjustment = await adjustment_for( + h, + case, + managed_control_total_deltas=( + ReportingControlTotalRecord( + "spend", "-1.50", "decimal", case.obligation.currency + ), + ), + ) + request = { + "adcp_version": "3.2-rc.3", + "account": {"account_id": case.config.account_id}, + "idempotency_key": "frozen-feed-mixed-batch", + "receipts": [receipt_to_wire(receipt)], + "adjustment_receipts": [adjustment], + } + caller = {"account_id": case.config.account_id, "consumer_id": case.binding.consumer_id} + admitted = await b22_mounted( + approved_feed_b22, + pool, + "receipt", + notifications=notifications, + caller=caller, + request=request, + ) + assert [r["result"] for r in admitted["result"]["results"]] == ["recorded", "recorded"] + captures = await parent.read_receipt_boundaries(caller=case.scope.principal) + kwargs = { + "account": case.config.account_id, + "consumer": case.binding.consumer_id, + "obligation": case.obligation.reporting_obligation_id, + "notifications": notifications, + "receipt_count": 2, + } + before = await frozen_call( + installed_feed_history, pool, "exercise", phase="before", **kwargs + ) + saved = await immutable_parent_rows(pool) + queue = await h.queue() + async with pool.connection() as c: + old_objects = await schema_objects(c) + feed = PgReportingFeedStore(pool=pool, notifications=notifications) + await feed.create_schema() + async with pool.connection() as c: + new_objects = await schema_objects(c) + assert {k: new_objects[k] for k in old_objects} == old_objects + added = new_objects.keys() - old_objects.keys() + assert len(added) == 33 and all("reporting_feed_" in k for k in added) + query = { + "account": request["account"], + "view": "periods", + "pagination": {"max_results": 1}, + } + first = await feed.read_reporting_feed(query, caller=case.scope.principal) + expected = await walk(feed, query, case.scope.principal, first=first) + frozen = await feed.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=case.scope.principal + ) + after = await frozen_call( + installed_feed_history, pool, "exercise", phase="after", **kwargs + ) + replayed = await b22_mounted( + approved_feed_b22, + pool, + "receipt", + notifications=notifications, + caller=caller, + request=request, + ) + assert replayed["result"] == admitted["result"] + # Actual B2.2 binary admits the same IDs/key for another canonical + # consumer after migration, while the original walk is still open. + second_result = await b22_mounted( + approved_feed_b22, + pool, + "receipt", + notifications=notifications, + caller={"account_id": second.account_id, "consumer_id": second.consumer_id}, + request=request, + ) + assert [r["result"] for r in second_result["result"]["results"]] == [ + "recorded", + "recorded", + ] + assert second_result["result"] != admitted["result"] + image = without_feed(await h.image()) + fresh_reader = PgReportingFeedStore(pool=pool, notifications=notifications) + assert await walk(fresh_reader, query, case.scope.principal, first=first) == expected + assert ( + await fresh_reader.read_reporting_feed_snapshot( + frozen.snapshot_id, caller=case.scope.principal + ) + == frozen + ) + assert without_feed(await h.image()) == image + assert await parent.read_receipt_boundaries(caller=case.scope.principal) == captures + assert await immutable_parent_rows(pool) == saved + assert await h.queue() == queue + assert await PgReportingMaterializerStore( + pool=pool, notifications=notifications + ).materializer_ready() + assert before["ordinary_core"] and after["ordinary_core"] + assert ( + before["ordinary_materializer"] + == after["ordinary_materializer"] + == (artifact != "beta15") + ) + assert before["notification_readiness"] == after["notification_readiness"] + print( + json.dumps( + { + "feed_rolling": artifact, + "notifications": notifications, + "parent_head": B22, + "parent_tree": B22_TREE, + "b21_producer_origins": produced["origins"], + "before": before, + "after": after, + "b22_mounted_origins": replayed["origins"], + "historical_wheel_sha256": installed_feed_history[3]["wheel_sha256"], + "b22_wheel_sha256": approved_feed_b22[3]["wheel_sha256"], + "feed_objects": len(added), + "page_count": len(expected[0]), + "quarantine_preserved": True, + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_feed_transactions.py b/tests/conformance/reporting/test_reporting_feed_transactions.py new file mode 100644 index 000000000..c1f78394f --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_transactions.py @@ -0,0 +1,329 @@ +"""Snapshot faults and races cannot change the approved writer transactions.""" + +import asyncio +import hashlib +import json +from datetime import datetime, timezone +from uuid import UUID + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed import PgReportingFeedStore, ReportingFeedError +from adcp.reporting.feed.snapshot import StoredFeedSnapshot + +from ._durable_materializer_support import DurableHarness, durable_case +from ._feed_support import feed_request, feeds, mixed_case, restart, walk, without_feed +from ._generation_support import isolated_reporting_pool +from ._receipt_support import receipt_case, request_for +from ._reconciliation_support import Clock + +__all__ = ["feeds"] + + +async def test_snapshot_identity_collision_cannot_replace_an_open_walk(feeds, monkeypatch): + from adcp.reporting.feed import projection + + h = feeds + s, _, _ = await mixed_case(h) + req = feed_request(s) + first = await h.store.read_reporting_feed(req, caller=s.binding.principal) + expected = await walk(h.store, req, s.binding.principal, first=first) + monkeypatch.setattr(projection, "uuid4", lambda: UUID(first["ledger_snapshot_id"][5:])) + before = await h.image() + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(req, caller=s.binding.principal) + assert error.value.code == "REPORTING_FEED_STORAGE_UNAVAILABLE" + assert await h.image() == before + assert await walk(h.store, req, s.binding.principal, first=first) == expected + + +@pytest.mark.parametrize("damage", ["missing", "record-kind", "signature"]) +async def test_lost_or_corrupt_snapshot_fails_without_reconstruction(feeds, damage, monkeypatch): + h = feeds + s, _, _ = await mixed_case(h) + first = await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + snapshot_id = first["ledger_snapshot_id"] + if h.pool is None: + document, _, signing_key = h.store._reporting_feed_snapshots[snapshot_id] + else: + async with h.pool.connection() as c: + document, signing_key = await ( + await c.execute( + "SELECT document, signing_key FROM reporting_feed_snapshots" + " WHERE snapshot_id=%s", + (snapshot_id,), + ) + ).fetchone() + if damage == "record-kind": + value = json.loads(document) + value["records"][0]["kind"] = "destination_binding" + document = canonical_json_utf8_v1(value) + elif isinstance(document, str): + document = document.encode() + if damage == "signature": + signing_key = b"x" * 32 + digest = hashlib.sha256(document).hexdigest() + if h.pool is None: + if damage == "missing": + del h.store._reporting_feed_snapshots[snapshot_id] + else: + h.store._reporting_feed_snapshots[snapshot_id] = (document, digest, signing_key) + + def forbidden_capture(*args, **kwargs): + raise AssertionError("a damaged continuation cannot open a new snapshot") + + monkeypatch.setattr(h.store, "_capture_feed", forbidden_capture) + else: + # Simulate operator loss/corruption without changing the schema guards. + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role=replica") + if damage == "missing": + await c.execute( + "DELETE FROM reporting_feed_snapshots WHERE snapshot_id=%s", (snapshot_id,) + ) + else: + await c.execute( + "UPDATE reporting_feed_snapshots SET document=%s,content_sha256=%s," + "signing_key=%s WHERE snapshot_id=%s", + (document.decode(), digest, signing_key, snapshot_id), + ) + + async def forbidden_capture(*args, **kwargs): + raise AssertionError("a damaged continuation cannot open a new snapshot") + + monkeypatch.setattr(h.store, "_capture_feed_on", forbidden_capture) + before = await h.image() + for req in ( + feed_request(s, pagination={"cursor": first["pagination"]["cursor"]}), + feed_request(s, changes_after=first["changes_checkpoint"]), + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(req, caller=s.binding.principal) + assert error.value.code == ( + "REPORTING_FEED_HISTORY_CORRUPT" if damage == "record-kind" else "INVALID_CHECKPOINT" + ) + assert await h.image() == before + + +@pytest.mark.parametrize("fault", ["capture", "insert", "assembly"]) +async def test_every_snapshot_fault_rolls_back_all_collections_heads_and_upstream_history( + feeds, monkeypatch, fault +): + h = feeds + s, request, response = await mixed_case(h) + before = await h.image() + cls = type(h.store) + if fault == "assembly": + + def failing(*args, **kwargs): + raise RuntimeError("injected feed assembly failure") + + monkeypatch.setattr(StoredFeedSnapshot, "page", failing) + elif h.pool is None: + name = "_capture_feed" if fault == "capture" else "_save_feed_snapshot" + original = getattr(cls, name) + + def failing(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected feed persistence failure") + + monkeypatch.setattr(cls, name, failing) + else: + name = "_capture_feed_on" if fault == "capture" else "_save_feed_snapshot_on" + original = getattr(cls, name) + + async def failing(*args, **kwargs): + await original(*args, **kwargs) + raise RuntimeError("injected feed persistence failure") + + monkeypatch.setattr(cls, name, failing) + with pytest.raises((RuntimeError, ReportingFeedError)): + await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + assert await h.image() == before + monkeypatch.undo() + await restart(h) + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + assert (await walk(h.store, feed_request(s), s.binding.principal))[0][-1]["pagination"][ + "total_count" + ] == 6 + assert without_feed(await h.image()) == without_feed(before) + + +@pytest.mark.parametrize("missing", ["obligation", "revision", "adjustment", "attempt"]) +async def test_missing_references_fail_new_snapshot_but_never_rebuild_persisted_continuation( + feeds, missing +): + h = feeds + s, _, _ = await mixed_case(h) + request = feed_request(s) + first = await h.store.read_reporting_feed(request, caller=s.binding.principal) + expected = await walk(h.store, request, s.binding.principal, first=first) + if h.pool is None: + if missing == "obligation": + del h.store._obligations[s.obligation.reporting_obligation_id] + elif missing == "revision": + del h.store._revisions[s.revision.reporting_revision_id] + elif missing == "adjustment": + del h.store._adjustments["adjustment-1"] + else: + h.store._delivery_records = [ + (seq, who, record) + for seq, who, record in h.store._delivery_records + if record.kind != "materialization_attempt" + ] + else: + # An operator can bypass triggers. The persisted snapshot is independent + # of lost current rows; opening a new snapshot must diagnose the loss. + statements = { + "obligation": ( + "DELETE FROM reporting_obligations WHERE reporting_obligation_id=%s", + s.obligation.reporting_obligation_id, + ), + "revision": ( + "DELETE FROM reporting_revisions WHERE reporting_revision_id=%s", + s.revision.reporting_revision_id, + ), + "adjustment": ( + "DELETE FROM reporting_adjustments WHERE reporting_adjustment_id=%s", + "adjustment-1", + ), + "attempt": ( + "DELETE FROM reporting_reconciliation_records" + " WHERE record_kind='materialization_attempt' AND reporting_materialization_id=%s", + s.outcome.reporting_materialization_id, + ), + } + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role=replica") + query, value = statements[missing] + await c.execute(query, (value,)) + store = await restart(h) + before = await h.image() + assert await walk(store, request, s.binding.principal, first=first) == expected + with pytest.raises(ReportingFeedError) as error: + await store.read_reporting_feed(request, caller=s.binding.principal) + assert error.value.code == "REPORTING_FEED_HISTORY_CORRUPT" + assert await h.image() == before + + +async def test_verified_finish_capture_and_epoch_zero_queue_are_frozen_without_read_side_effects( + feeds, +): + h = feeds + case = await durable_case(h.store, count=3) + assert (await case.service().run_once()).state == "verified" + boundaries = await h.store.read_materializer_boundaries(caller=case.scope.principal) + assert len(boundaries) == 1 + request = { + "account": {"account_id": case.config.account_id}, + "view": "periods", + "pagination": {"max_results": 1}, + } + before = without_feed(await h.image()) + queue = await h.queue() + page = await h.store.read_reporting_feed(request, caller=case.scope.principal) + snapshot = await h.store.read_reporting_feed_snapshot( + page["ledger_snapshot_id"], caller=case.scope.principal + ) + assert snapshot.inputs["materializer_boundaries"] == [b.to_storage() for b in boundaries] + assert snapshot.inputs["ownership_mode"] == snapshot.ownership_mode == "absent" + assert without_feed(await h.image()) == before + assert await h.queue() == queue + assert set(queue[1]) <= {"quarantined"} + await restart(h) + assert await h.store.read_materializer_boundaries(caller=case.scope.principal) == boundaries + assert await h.queue() == queue + + +async def test_postgres_captures_on_one_connection_under_account_lock_and_uses_database_time( + monkeypatch, +): + from psycopg_pool import AsyncConnectionPool + + async with isolated_reporting_pool(autocommit=True) as outer: + async with AsyncConnectionPool( + outer.conninfo, kwargs=outer.kwargs, min_size=1, max_size=1, open=False + ) as pool: + store = PgReportingFeedStore(pool=pool, clock=Clock()) + await store.create_schema() + h = DurableHarness(store, store._clock, pool) + s, _, _ = await mixed_case(h) + h.clock.now = datetime(2000, 1, 1, tzinfo=timezone.utc) + original = store._capture_feed_on + connections = [] + + async def capture(connection, *args, **kwargs): + connections.append(connection) + return await original(connection, *args, **kwargs) + + save = store._save_feed_snapshot_on + + async def persist(connection, *args, **kwargs): + connections.append(connection) + return await save(connection, *args, **kwargs) + + monkeypatch.setattr(store, "_capture_feed_on", capture) + monkeypatch.setattr(store, "_save_feed_snapshot_on", persist) + async with pool.connection() as c: + before = (await (await c.execute("SELECT clock_timestamp()")).fetchone())[0] + page = await asyncio.wait_for( + store.read_reporting_feed(feed_request(s), caller=s.binding.principal), 10 + ) + async with pool.connection() as c: + after = (await (await c.execute("SELECT clock_timestamp()")).fetchone())[0] + assert ( + before + <= datetime.fromisoformat(page["ledger_as_of"].replace("Z", "+00:00")) + <= after + ) + assert len(connections) == 2 and connections[0] is connections[1] + + +async def test_same_account_receipt_writer_waits_for_complete_snapshot_then_is_deferred( + monkeypatch, +): + from ._feed_support import feed_harness + + async with feed_harness("postgres", notifications=True) as h: + s = await receipt_case(h) + entered, release = asyncio.Event(), asyncio.Event() + original = h.store._capture_feed_on + + async def pause(connection, *args, **kwargs): + result = await original(connection, *args, **kwargs) + entered.set() + await release.wait() + return result + + monkeypatch.setattr(h.store, "_capture_feed_on", pause) + reader = asyncio.create_task( + h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + ) + writer = None + try: + await asyncio.wait_for(entered.wait(), 10) + writer = asyncio.create_task( + h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + ) + with pytest.raises(asyncio.TimeoutError): + await asyncio.wait_for(asyncio.shield(writer), 0.1) + release.set() + first = await asyncio.wait_for(reader, 10) + receipt = await asyncio.wait_for(writer, 10) + finally: + release.set() + for task in (reader, writer): + if task is not None and not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + monkeypatch.undo() + _, rows, checkpoint = await walk(h.store, feed_request(s), s.binding.principal, first=first) + assert rows["receipts"] == [] + _, later, _ = await walk( + h.store, feed_request(s, changes_after=checkpoint), s.binding.principal + ) + assert later["receipts"] == [receipt["results"][0]["receipt"]] + assert ( + len(later["periods"]) == len(later["revisions"]) == len(later["materializations"]) == 1 + ) diff --git a/tests/conformance/reporting/test_reporting_feed_transports.py b/tests/conformance/reporting/test_reporting_feed_transports.py new file mode 100644 index 000000000..47894aa06 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_feed_transports.py @@ -0,0 +1,507 @@ +"""Public MCP/A2A pages: exact authorization, raw schemas, generated clients.""" + +import json +import re +from contextlib import AsyncExitStack +from copy import deepcopy +from functools import partial + +import pytest +from jsonschema import Draft7Validator, FormatChecker + +from adcp.decisioning import Account, AuthInfo, RequestContext +from adcp.exceptions import ADCPTaskError +from adcp.reporting.feed.request import feed_schema +from adcp.reporting.receipts import ReportingReceiptHandler +from adcp.server import ADCPHandler, ToolContext +from adcp.server.mcp_tools import get_tools_for_handler +from adcp.types import GetReportingStatusRequest, GetReportingStatusResponse + +from ._feed_support import ( + MountedFeed, + feed_request, + feeds, + mixed_case, + second_consumer, + without_feed, +) +from ._receipt_support import receipt_case, receipt_harness, request_for +from ._receipt_transport import MountedReceipts, error_code + +__all__ = ["feeds"] + + +async def test_mounted_fractional_context_and_filters_keep_exact_integer_page_bounds(feeds): + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + req = feed_request(s, context={"sample": 0.125}, ext={"vendor": {"threshold": 0.25}}) + async with mounted.client() as client: + _, first = await mounted.mcp(client, req) + assert first["context"] == req["context"], first + assert first["pagination"]["has_more"] is True + continued = { + **req, + "context": {"sample": 0.75}, + "pagination": {"cursor": first["pagination"]["cursor"], "max_results": 100}, + } + for v1 in (False, True): + _, last = await mounted.a2a(client, continued, v1=v1) + assert last["context"] == continued["context"], last + assert last["changes_checkpoint"] == first["changes_checkpoint"] + assert last["pagination"]["has_more"] is False + before = await h.image() + for call in (mounted.mcp, mounted.a2a, partial(mounted.a2a, v1=True)): + _, invalid = await call( + client, + req, + mutate_wire=lambda raw: re.sub( + r'"max_results": 1(?:\.0)?(?=\s*[,}])', + '"max_results": 1.000000000000000000001', + raw, + ), + ) + assert error_code(invalid) in {"INVALID_REQUEST", "VALIDATION_ERROR"}, invalid + assert await h.image() == before + + +@pytest.mark.parametrize("view", ["summary", "revision"]) +async def test_cross_view_positions_fail_before_current_projection_on_every_mount( + feeds, view, monkeypatch +): + from adcp.reporting.ledger.status import ReportingStatusHandler + + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + async with mounted.client() as client: + _, first = await mounted.mcp(client, feed_request(s)) + + async def forbidden_projection(*args, **kwargs): + raise AssertionError("a bound feed token must never consult today's Core projection") + + monkeypatch.setattr(ReportingStatusHandler, "handle", forbidden_projection) + for position in ( + {"pagination": {"cursor": first["pagination"]["cursor"]}}, + {"changes_after": first["changes_checkpoint"]}, + ): + req = feed_request(s, view=view, **position) + if view == "revision": + req["reporting_revision_id"] = s.revision.reporting_revision_id + with pytest.raises(ADCPTaskError) as error: + await mounted.handler.get_reporting_status( + req, ToolContext(caller_identity=s.binding.consumer_id) + ) + assert error.value.errors[0].code == "INVALID_CHECKPOINT" + for transport in (mounted.mcp, mounted.a2a): + _, rejected = await transport(client, req) + assert error_code(rejected) in { + "INVALID_CHECKPOINT", + "INVALID_REQUEST", + "VALIDATION_ERROR", + } + mounted.grants.clear() + with pytest.raises(ADCPTaskError) as error: + await mounted.handler.get_reporting_status( + req, ToolContext(caller_identity=s.binding.consumer_id) + ) + assert error.value.errors[0].code == "UNAUTHORIZED" + _, denied = await mounted.mcp( + client, feed_request(s, pagination={"cursor": first["pagination"]["cursor"]}) + ) + assert error_code(denied) == "UNAUTHORIZED" + + +async def test_feed_mount_preserves_closed_tier_and_new_store_notification_admission(feeds): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox import ( + InMemoryReportingOutbox, + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class NoSubscriptions: + async def list_active(self, **kwargs): + raise AssertionError("an unadmitted store must not expand notification recipients") + + async def get_active(self, **kwargs): + raise AssertionError("an unadmitted store must not authorize delivery") + + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + assert await mounted.handler.get_adcp_capabilities({}) == ( + await ADCPHandler().get_adcp_capabilities({}) + ) + before = without_feed(await h.image()) + async with mounted.client() as client: + for transport in (mounted.mcp, mounted.a2a): + _, raw = await transport( + client, + {}, + mutate_wire=lambda wire: wire.replace( + '"get_reporting_status"', '"get_adcp_capabilities"' + ), + ) + # The inherited default stub is unsupported; strict output schema + # validation may translate that into its existing validation error. + assert error_code(raw) in {"NOT_SUPPORTED", "VALIDATION_ERROR"}, raw + assert all( + field not in json.dumps(raw) + for field in ("managed_delivery", "reconciled_billing", "delivery_ready") + ) + # The mounted Core polling path is available while admission stays shut. + _, page = await mounted.mcp(client, feed_request(s, limit=100)) + assert page["periods"] and page["revisions"] + if h.pool is None and h.store._notification_state is None: + with pytest.raises(ValueError, match="notifications=True"): + InMemoryReportingOutbox(h.store) + assert without_feed(await h.image()) == before + return + outbox = ( + PgReportingOutbox(pool=h.pool) if h.pool is not None else InMemoryReportingOutbox(h.store) + ) + worker = ReportingNotificationWorker( + outbox=outbox, + subscriptions=NoSubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ) + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await worker.advertised_notifications( + h.store, account_id=s.obligation.account_id, ready_scope=None + ) + assert without_feed(await h.image()) == before + + +@pytest.mark.parametrize( + "hydrated,registry_kind", + [(False, None), (True, None), (True, "api_key"), (True, "oauth"), (True, "http_sig")], +) +async def test_mounted_mcp_to_a2a_continuation_reauthorizes_exact_canonical_consumer( + feeds, hydrated, registry_kind +): + h = feeds + s, _, _ = await mixed_case(h, consumer_id="https://buyer.example.test/first") + other = await second_consumer(h, s, "https://buyer.example.test/second") + mounted = MountedFeed(h, hydrated=hydrated, registry_kind=registry_kind) + mounted.authorize(s) + mounted.authorize(other, token="token-two") + req = GetReportingStatusRequest.model_validate(feed_request(s)).model_dump( + mode="json", exclude_unset=True + ) + before = without_feed(await h.image()) + async with mounted.client() as client: + status, first = await mounted.mcp(client, req) + assert status == 200 and first["pagination"]["total_count"] == 6, first + GetReportingStatusResponse.model_validate(first) + cursor = first["pagination"]["cursor"] + continued = feed_request(s, pagination={"cursor": cursor, "max_results": 100}) + _, other_result = await mounted.a2a(client, continued, token="token-two") + assert error_code(other_result) == "INVALID_CHECKPOINT", other_result + for v1 in (False, True): + status, tail = await mounted.a2a(client, continued, v1=v1) + assert status == 200 and tail["pagination"]["has_more"] is False, tail + assert tail["changes_checkpoint"] == first["changes_checkpoint"] + GetReportingStatusResponse.model_validate(tail) + assert tail["receipts"][0]["reporting_receipt_id"] == s.receipt.reporting_receipt_id + mounted.grants.remove((s.obligation.account_id, s.binding.consumer_id)) + for transport in (mounted.mcp, mounted.a2a): + _, denied = await transport(client, continued) + assert error_code(denied) == "UNAUTHORIZED", denied + assert len(mounted.auth_calls) >= 6 + assert without_feed(await h.image()) == before + + +@pytest.mark.parametrize("feedback", [False, True]) +async def test_mounted_feed_private_counts_do_not_depend_on_feedback_flag(feeds, feedback): + h = feeds + s, _, _ = await mixed_case(h) + other = await second_consumer(h, s) + mounted = MountedFeed(h, feedback=feedback) + mounted.authorize(s) + mounted.authorize(other, token="token-two") + async with mounted.client() as client: + _, first = await mounted.mcp(client, feed_request(s, limit=100)) + _, second = await mounted.a2a(client, feed_request(other, limit=100), token="token-two") + assert first["pagination"]["total_count"] == 6, first + assert second["pagination"]["total_count"] == 5, second + assert len(first["receipts"]) == len(second["receipts"]) == 1 + assert first["adjustment_receipts"] and second["adjustment_receipts"] == [] + assert first["changes_checkpoint"] != second["changes_checkpoint"] + + +async def test_mounted_malformed_scoped_positions_and_generic_cache_cannot_bypass_authorization( + feeds, +): + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + async with mounted.client() as client: + _, page = await mounted.mcp(client, feed_request(s)) + mutations = [ + {"changes_after": "old-unbound-token"}, + {"changes_after": "x" * 2049}, + {"pagination": {"cursor": "x" * 2049}}, + {"pagination": {"max_results": 101}}, + {"consumer_id": "spoofed"}, + {"idempotency_key": "feed-must-not-cache"}, + { + "delivery_config_ids": ["foreign"], + "pagination": {"cursor": page["pagination"]["cursor"]}, + }, + ] + for changes in mutations: + for transport in (mounted.mcp, mounted.a2a): + _, result = await transport(client, feed_request(s, **changes)) + assert error_code(result) in {"INVALID_REQUEST", "INVALID_CHECKPOINT"}, result + # A registry revocation is independently re-resolved, even if the ACL + # and opaque AccountStore cache key have remained unchanged. + mounted.grants.clear() + _, denied = await mounted.mcp( + client, + feed_request( + s, + pagination={"cursor": page["pagination"]["cursor"]}, + idempotency_key="feed-must-not-cache", + ), + ) + assert error_code(denied) in {"UNAUTHORIZED", "INVALID_REQUEST"} + + +@pytest.mark.parametrize("version", [None, "3.2.0-rc.3"]) +async def test_actual_inventory_and_fallback_schemas_bound_positions_without_mutating_legacy( + feeds, version +): + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h, version=version) + mounted.authorize(s) + before_schema = feed_schema("request") + async with mounted.client() as client: + _, inventory = await mounted.mcp(client, inventory=True) + tool = next(t for t in inventory["tools"] if t["name"] == "get_reporting_status") + validator = Draft7Validator(tool["inputSchema"], format_checker=FormatChecker()) + assert validator.is_valid(feed_request(s)) + assert not validator.is_valid(feed_request(s, changes_after="x" * 2049)) + assert not validator.is_valid(feed_request(s, pagination={"cursor": "x" * 2049})) + _, page = await mounted.mcp(client, feed_request(s)) + assert Draft7Validator(tool["outputSchema"], format_checker=FormatChecker()).is_valid(page) + assert feed_schema("request") == before_schema + # Public definitions are copies, not aliases into memoized upstream state. + definitions = get_tools_for_handler(mounted.handler) + for definition in definitions: + if definition["name"] == "get_reporting_status": + definition["inputSchema"]["properties"]["changes_after"]["maxLength"] = 1 + assert feed_schema("request")["properties"]["changes_after"]["maxLength"] == 2048 + + +async def test_direct_hydrated_identity_conflicts_and_revoked_registry_fail_closed(feeds): + h = feeds + s, _, _ = await mixed_case(h, consumer_id="https://buyer.example.test/identity") + mounted = MountedFeed(h, hydrated=True, registry_kind="oauth") + mounted.authorize(s) + req = feed_request(s) + contexts = [ + None, + ToolContext(caller_identity="anonymous", tenant_id=s.binding.consumer_id), + ToolContext( + caller_identity=s.binding.consumer_id, + metadata={"auth_info": AuthInfo(kind="oauth", principal="https://other.example.test/")}, + ), + RequestContext( + account=Account(id=s.obligation.account_id), + caller_identity=s.binding.consumer_id, + tenant_id=s.binding.consumer_id, + ), + RequestContext( + account=Account(id="other-account"), + caller_identity="opaque-cache-key", + auth_principal=s.binding.consumer_id, + ), + ] + for context in contexts: + with pytest.raises(ADCPTaskError) as error: + await mounted.handler.get_reporting_status(req, context) + assert error.value.errors[0].code == "UNAUTHORIZED" + async with mounted.client() as client: + _, first = await mounted.mcp(client, req) + mounted.registry.agents.clear() + _, denied = await mounted.a2a( + client, feed_request(s, pagination={"cursor": first["pagination"]["cursor"]}) + ) + assert error_code(denied) == "UNAUTHORIZED" + + +async def test_model_generation_fallback_still_mounts_bound_feed_schema_without_changing_core( + feeds, monkeypatch +): + from adcp.server import mcp_tools + + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + stub = {"type": "object", "properties": {"view": {"type": "string"}}} + definitions = deepcopy(mcp_tools.ADCP_TOOL_DEFINITIONS) + for definition in definitions: + if definition["name"] == "get_reporting_status": + definition["inputSchema"] = deepcopy(stub) + definition["outputSchema"] = {"type": "object"} + monkeypatch.setattr(mcp_tools, "ADCP_TOOL_DEFINITIONS", definitions) + monkeypatch.setattr(mcp_tools, "_ensure_pydantic_schemas_applied", lambda names: None) + + class CoreOnly(ADCPHandler): + advertised_tools = {"get_reporting_status"} + + async def get_reporting_status(self, params, context=None): + return {} + + async with mounted.client() as client: + _, inventory = await mounted.mcp(client, inventory=True) + feed_tool = next(t for t in inventory["tools"] if t["name"] == "get_reporting_status") + validator = Draft7Validator(feed_tool["inputSchema"]) + assert validator.is_valid(feed_request(s)) + assert not validator.is_valid(feed_request(s, pagination={"cursor": "x" * 2049})) + _, response = await mounted.mcp(client, feed_request(s)) + assert response["pagination"]["total_count"] == 6 + core_tool = next( + t for t in get_tools_for_handler(CoreOnly()) if t["name"] == "get_reporting_status" + ) + assert core_tool["inputSchema"] == stub + + +@pytest.mark.parametrize("feed_first", [True, False], ids=["feed-first", "receipts-first"]) +async def test_paired_handler_instances_keep_mcp_a2a_inventory_calls_and_authorization_isolated( + feeds, feed_first +): + """Both construction/mount orders exercise the same class-level registry.""" + h = feeds + s, original_request, original_response = await mixed_case(h) + other_consumer = await second_consumer(h, s) + other_account = await receipt_case(h, account_id="paired-account") + await h.store.ingest_receipt_batch( + request_for(other_account), caller=other_account.binding.principal + ) + backend = "postgres" if h.pool is not None else "memory" + notifications = ( + h.store._notifications_enabled + if h.pool is not None + else h.store._notification_state is not None + ) + async with receipt_harness(backend, notifications=notifications) as old: + # Same account/consumer/materialization/receipt/key on distinct stores. + # The receipt-only handler has no adjustment, so cached cross-handler + # admission or replay would give the wrong exact batch response. + legacy = await receipt_case(old) + constructors = { + "feed": lambda: MountedFeed(h, hydrated=True), + "receipts": lambda: MountedReceipts(old, hydrated=True), + } + order = ("feed", "receipts") if feed_first else ("receipts", "feed") + mounts = {name: constructors[name]() for name in order} + feed, receipts = mounts["feed"], mounts["receipts"] + assert type(feed.handler) is type(receipts.handler) is ReportingReceiptHandler + feed.authorize(s) + feed.authorize(other_consumer, token="token-two") + feed.authorize(other_account, token="token-account") + receipts.authorize(legacy) + unsupported = await ADCPHandler().get_reporting_status(feed_request(s)) + assert await receipts.handler.get_reporting_status(feed_request(s)) == unsupported + async with AsyncExitStack() as stack: + clients = { + name: await stack.enter_async_context(mounts[name].client()) for name in order + } + fc, rc = clients["feed"], clients["receipts"] + expected = { + "feed": { + "get_adcp_capabilities", + "sync_reporting_receipts", + "get_reporting_status", + }, + "receipts": {"get_adcp_capabilities", "sync_reporting_receipts"}, + } + for name in order + tuple(reversed(order)): + mount, client = mounts[name], clients[name] + _, inventory = await mount.mcp(client, inventory=True) + assert {t["name"] for t in inventory["tools"]} == expected[name] + for path in ("/.well-known/agent.json", "/.well-known/agent-card.json"): + response = await client.get(path) + assert response.status_code == 200, response.text + assert {skill["id"] for skill in response.json()["skills"]} == expected[name] + _, first = await feed.mcp(fc, feed_request(s)) + assert first["pagination"]["total_count"] == 6 + continuation = feed_request(s, pagination={"cursor": first["pagination"]["cursor"]}) + # Same idempotency key and external IDs remain store/handler scoped. + _, legacy_response = await receipts.mcp(rc, request_for(legacy)) + assert len(legacy_response["results"]) == 1 + for v1 in (False, True): + _, replay = await receipts.a2a(rc, request_for(legacy), v1=v1) + assert replay == legacy_response + _, feed_replay = await MountedReceipts.a2a(feed, fc, original_request, v1=v1) + assert feed_replay == original_response + _, unsupported_wire = await receipts.a2a( + rc, + continuation, + v1=v1, + mutate_wire=lambda wire: wire.replace( + '"skill": "sync_reporting_receipts"', '"skill": "get_reporting_status"' + ), + ) + assert "Unknown skill: get_reporting_status" in json.dumps(unsupported_wire) + # MCP's unmapped task has the legacy protocol error, not a feed page. + headers = { + "accept": "application/json, text/event-stream", + "authorization": "Bearer token-one", + } + if receipts.sessions["token-one"] is not None: + headers["mcp-session-id"] = receipts.sessions["token-one"] + response = await rc.post( + "/mcp/", + headers=headers, + json={ + "jsonrpc": "2.0", + "id": 901, + "method": "tools/call", + "params": {"name": "get_reporting_status", "arguments": continuation}, + }, + ) + assert response.status_code == 200 + assert "Unknown tool: get_reporting_status" in response.text + assert first["ledger_snapshot_id"] not in response.text + # Revocation on one handler cannot be masked by authorization of + # the same account/consumer on its neighbor. + receipts.grants.clear() + _, denied = await receipts.a2a(rc, request_for(legacy)) + assert error_code(denied) == "UNAUTHORIZED" + _, feed_tail = await feed.a2a(fc, continuation) + assert feed_tail["ledger_snapshot_id"] == first["ledger_snapshot_id"] + for token, subject, total in ( + ("token-two", other_consumer, 5), + ("token-account", other_account, 4), + ): + for transport in (feed.mcp, feed.a2a): + _, crossed = await transport( + fc, + feed_request(subject, pagination=continuation["pagination"]), + token=token, + ) + assert error_code(crossed) == "INVALID_CHECKPOINT" + _, own = await transport(fc, feed_request(subject, limit=100), token=token) + assert own["pagination"]["total_count"] == total + assert own["ledger_snapshot_id"] != first["ledger_snapshot_id"] + assert own["adjustment_receipts"] == [] + receipts.grants.add((legacy.obligation.account_id, legacy.binding.consumer_id)) + feed.grants.remove((s.obligation.account_id, s.binding.consumer_id)) + for transport in (feed.mcp, feed.a2a): + _, denied = await transport(fc, continuation) + assert error_code(denied) == "UNAUTHORIZED" + _, replay = await receipts.mcp(rc, request_for(legacy)) + assert replay == legacy_response + assert await receipts.handler.get_reporting_status(feed_request(s)) == unsupported + _, inventory = await receipts.mcp(rc, inventory=True) + assert {t["name"] for t in inventory["tools"]} == expected["receipts"] diff --git a/tests/type_checks/reporting_frozen_feed.py b/tests/type_checks/reporting_frozen_feed.py new file mode 100644 index 000000000..9fab8e1a7 --- /dev/null +++ b/tests/type_checks/reporting_frozen_feed.py @@ -0,0 +1,55 @@ +"""One optional feed participant composes with the approved receipt/materializer.""" + +from typing import Any + +from typing_extensions import assert_type + +from adcp.reporting.feed import ( + InMemoryReportingFeedStore, + PgReportingFeedStore, + ReportingFeedSnapshot, + ReportingFeedStore, +) +from adcp.reporting.ledger import ReportingDeliveryPrincipal, ReportingLedgerStore +from adcp.reporting.materializer import ReportingMaterializerStore +from adcp.reporting.receipts import ( + ReceiptAccountResolver, + ReportingReceiptBatchStore, + ReportingReceiptHandler, +) +from adcp.server import ToolContext +from adcp.server.base import NotImplementedResponse + + +async def adopter( + postgres: PgReportingFeedStore, + memory: InMemoryReportingFeedStore, + caller: ReportingDeliveryPrincipal, + request: dict[str, Any], + resolve_account: ReceiptAccountResolver, + context: ToolContext, +) -> ReportingMaterializerStore: + await postgres.create_schema() + feed: ReportingFeedStore = postgres + feed = memory + receipts: ReportingReceiptBatchStore = postgres + receipts = memory + ledger: ReportingLedgerStore = postgres + ledger = memory + materializer: ReportingMaterializerStore = postgres + materializer = memory + assert_type(await feed.reporting_feed_ready(), bool) + assert_type(await feed.read_reporting_feed(request, caller=caller), dict[str, Any]) + assert_type( + await feed.read_reporting_feed_snapshot("snapshot", caller=caller), + ReportingFeedSnapshot | None, + ) + assert_type(await receipts.ingest_receipt_batch(request, caller=caller), dict[str, Any]) + await ledger.list_configurations(account_id=caller.account_id) + handler = ReportingReceiptHandler(postgres, resolve_account=resolve_account) + assert_type(handler.reporting_feed_store, ReportingFeedStore | None) + assert_type( + await handler.get_reporting_status(request, context), + dict[str, Any] | NotImplementedResponse, + ) + return materializer From 44f9e72eccc48e5c204e0faf8adc793e8563ac59 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 20:39:23 +0000 Subject: [PATCH 2/5] fix(reporting): bind transport-equivalent numeric feed filters --- docs/reporting-frozen-feed.md | 7 +- src/adcp/reporting/feed/request.py | 21 +++- tests/conformance/reporting/_feed_support.py | 63 ++++++++++- .../test_reporting_feed_transports.py | 100 ++++++++++++++++++ 4 files changed, 188 insertions(+), 3 deletions(-) diff --git a/docs/reporting-frozen-feed.md b/docs/reporting-frozen-feed.md index 489e7efde..0f3e69f5a 100644 --- a/docs/reporting-frozen-feed.md +++ b/docs/reporting-frozen-feed.md @@ -61,7 +61,12 @@ the final deduplicated wire set **after** closure; it stays identical on all pag Use the same semantic filters throughout a walk. `pagination.max_results` may change (1–100), and `context` is echoed independently on each request. All semantic fields, including vendor `ext`, bind the snapshot. Ordinary finite JSON -numbers are supported in context and vendor filters. Account aliases are bound +numbers are supported in context and vendor filters. Integral numeric forms +such as `1` and `1.0` bind identically across MCP and A2A's protobuf encoding, +including nested vendor values. Fractions, booleans, strings and changed numbers +remain distinct. An A2A double that rounds a large integer to a different value +cannot continue the original filter. Page bounds and receipt financial numeric +validation retain their separate strict rules. Account aliases are bound through the authenticated canonical account, not their spelling in the request. Each page supplies one identical `changes_checkpoint`. Persist or advance it diff --git a/src/adcp/reporting/feed/request.py b/src/adcp/reporting/feed/request.py index ce8ad6736..3aaaa197e 100644 --- a/src/adcp/reporting/feed/request.py +++ b/src/adcp/reporting/feed/request.py @@ -20,6 +20,23 @@ TOKEN_LIMIT = 2048 +def _semantic_numbers(value: Any) -> Any: + """Bind equal JSON numbers identically across MCP and protobuf Struct. + + Struct represents every number as a double, including integral values. + Keep exact integers (and booleans) intact: rounding an integer through a + float here could silently authorize a genuinely different vendor filter. + The caller validates finite ordinary JSON before applying this transform. + """ + if type(value) is float and value.is_integer(): + return int(value) + if type(value) is dict: + return {key: _semantic_numbers(item) for key, item in value.items()} + if type(value) is list: + return [_semantic_numbers(item) for item in value] + return value + + def transport_parameters(params: dict[str, Any]) -> dict[str, Any]: """Read ordinary JSON without rounding the raw pagination integer first. @@ -162,10 +179,12 @@ def parse(cls, request: dict[str, Any]) -> FeedRequest: # fractional numbers. The restricted financial evidence encoder is # not their wire contract. Store deterministic JSON *bytes* as part # of the snapshot binding, without normalizing the caller's context. + # Integral number spellings are transport-equivalent; fractional + # values and nonnumeric JSON types retain their distinct meanings. if json.loads(json.dumps(request, allow_nan=False)) != request: raise ValueError encoded = json.dumps( - filters, allow_nan=False, sort_keys=True, separators=(",", ":") + _semantic_numbers(filters), allow_nan=False, sort_keys=True, separators=(",", ":") ).encode("ascii") return cls(encoded, cursor, checkpoint, limit) except (ValueError, TypeError, KeyError, OverflowError, RecursionError): diff --git a/tests/conformance/reporting/_feed_support.py b/tests/conformance/reporting/_feed_support.py index 04b8fb26f..44407329a 100644 --- a/tests/conformance/reporting/_feed_support.py +++ b/tests/conformance/reporting/_feed_support.py @@ -1,6 +1,7 @@ """Shared memory/PostgreSQL feed vectors and real mounted reporting reads.""" -from contextlib import asynccontextmanager +import json +from contextlib import AsyncExitStack, asynccontextmanager from copy import deepcopy from dataclasses import replace @@ -164,6 +165,66 @@ def __init__(self, h, *, feedback=False, **kwargs): if kwargs.get("version") is not None: self.handler.adcp_version = kwargs["version"] + @asynccontextmanager + async def sdk_clients(self, a2a_version, *, token="token-one"): + """Public typed clients and real encoders; only HTTP I/O uses ASGI.""" + import httpx2 + + from adcp import ADCPClient, AgentConfig + + observed = [] + + async def capture(request): + if request.method != "POST": + return + body = json.loads(request.content) + method = body.get("method") + if method == "tools/call": + params = body["params"]["arguments"] + protocol = "mcp" + elif method in {"message/send", "SendMessage"}: + params = body["params"]["message"]["parts"][0]["data"]["parameters"] + protocol = "a2a" + else: + return + observed.append((protocol, method, params)) + + async with self.client(public_url="http://localhost") as transport: + # Keep both public SDK transport stacks, negotiation and encoders. + # The factory/HTTP pool injection only replaces the network socket. + def mcp_http(**kwargs): + return httpx2.AsyncClient( + transport=httpx2.ASGITransport(app=transport._transport.app), + event_hooks={"request": [capture]}, + **kwargs, + ) + + clients = {} + async with AsyncExitStack() as stack: + for protocol in ("mcp", "a2a"): + config = AgentConfig( + id=f"feed-{protocol}", + agent_uri=( + "http://localhost/mcp/" if protocol == "mcp" else "http://localhost" + ), + protocol=protocol, + auth_token=token, + auth_header="Authorization", + auth_type="bearer", + ) + clients[protocol] = await stack.enter_async_context( + ADCPClient( + config, + adcp_version="3.2-rc.3", + force_a2a_version=a2a_version if protocol == "a2a" else None, + httpx_client_factory=mcp_http if protocol == "mcp" else None, + ) + ) + transport.headers["Authorization"] = f"Bearer {token}" + transport.event_hooks["request"].append(capture) + clients["a2a"].adapter._httpx_client = transport + yield clients, observed + async def mcp(self, client, request=None, *, mutate_wire=None, **kwargs): def rewrite(wire): wire = wire.replace( diff --git a/tests/conformance/reporting/test_reporting_feed_transports.py b/tests/conformance/reporting/test_reporting_feed_transports.py index 47894aa06..944421c72 100644 --- a/tests/conformance/reporting/test_reporting_feed_transports.py +++ b/tests/conformance/reporting/test_reporting_feed_transports.py @@ -31,6 +31,106 @@ __all__ = ["feeds"] +@pytest.fixture(autouse=True) +def _a2a_compat_send_and_aggregate(): + # Override the repository's unit-mock adapter shim. These mounted tests + # consume the actual public SDK's async-generator response stream. + pass + + +@pytest.mark.parametrize("a2a_version", ["0.3", "1.0"]) +@pytest.mark.parametrize("direction", ["mcp-to-a2a", "a2a-to-mcp"]) +async def test_public_sdk_numeric_filter_continuation_in_both_directions( + feeds, a2a_version, direction +): + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + req = feed_request( + s, + context={"sample": 0.125}, + ext={"vendor": {"integral": 1, "fraction": 0.25, "nested": [-2, -0.0, True, 1.5]}}, + ) + start, finish = direction.split("-to-") + async with mounted.sdk_clients(a2a_version) as (clients, observed): + first_result = await clients[start].get_reporting_status( + GetReportingStatusRequest.model_validate(req) + ) + assert first_result.success, first_result + first = first_result.data.model_dump(mode="json", exclude_unset=True) + assert first["context"] == req["context"] + assert first["pagination"]["has_more"] is True + continued = { + **req, + "context": {"sample": 0.75}, + "pagination": {"max_results": 100, "cursor": first["pagination"]["cursor"]}, + } + before = await h.image() + last_result = await clients[finish].get_reporting_status( + GetReportingStatusRequest.model_validate(continued) + ) + assert last_result.success, last_result + last = last_result.data.model_dump(mode="json", exclude_unset=True) + assert last["context"] == continued["context"] + for key in ("changes_checkpoint", "ledger_snapshot_id", "ledger_as_of"): + assert last[key] == first[key] + assert last["pagination"] == {"total_count": 6, "has_more": False} + for value in (True, "1", 1.25): + for position in ( + {"pagination": continued["pagination"]}, + {"changes_after": last["changes_checkpoint"]}, + ): + changed = deepcopy(req) + changed.update(position) + changed["ext"]["vendor"]["integral"] = value + rejected = await clients[finish].adapter.get_reporting_status(changed) + assert rejected.success is False, rejected + assert rejected.adcp_error["code"] == "INVALID_CHECKPOINT", rejected + assert await h.image() == before + empty = await clients[finish].get_reporting_status( + GetReportingStatusRequest.model_validate( + {**req, "changes_after": last["changes_checkpoint"]} + ) + ) + assert empty.success, empty + assert empty.data.pagination.total_count == 0 + assert empty.data.pagination.has_more is False + calls = {protocol: (method, params) for protocol, method, params in observed} + assert set(calls) == {"mcp", "a2a"} + assert calls["a2a"][0] == ("SendMessage" if a2a_version == "1.0" else "message/send") + assert type(calls["mcp"][1]["ext"]["vendor"]["integral"]) is int + assert type(calls["a2a"][1]["ext"]["vendor"]["integral"]) is float + assert calls["a2a"][1]["ext"]["vendor"]["fraction"] == 0.25 + + +@pytest.mark.parametrize("a2a_version", ["0.3", "1.0"]) +async def test_public_sdk_rounding_a_large_integer_is_a_changed_filter(feeds, a2a_version): + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + exact = 9007199254740993 + req = feed_request(s, ext={"vendor": {"integral": exact}}) + async with mounted.sdk_clients(a2a_version) as (clients, observed): + first = await clients["mcp"].get_reporting_status( + GetReportingStatusRequest.model_validate(req) + ) + assert first.success, first + before = await h.image() + for position in ( + {"pagination": {"cursor": first.data.pagination.cursor}}, + {"changes_after": first.data.changes_checkpoint}, + ): + changed = await clients["a2a"].adapter.get_reporting_status({**req, **position}) + assert changed.success is False, changed + assert changed.adcp_error["code"] == "INVALID_CHECKPOINT", changed + assert await h.image() == before + calls = {protocol: params for protocol, _, params in observed} + assert calls["mcp"]["ext"]["vendor"]["integral"] == exact + assert calls["a2a"]["ext"]["vendor"]["integral"] == 9007199254740992.0 + + async def test_mounted_fractional_context_and_filters_keep_exact_integer_page_bounds(feeds): h = feeds s, _, _ = await mixed_case(h) From 7b4e873a0863eeef128f776e4dcff84cdc8f71a3 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 22:38:47 +0000 Subject: [PATCH 3/5] test(reporting): guard optional PostgreSQL snapshot import --- .../conformance/reporting/test_reporting_feed_transactions.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/conformance/reporting/test_reporting_feed_transactions.py b/tests/conformance/reporting/test_reporting_feed_transactions.py index c1f78394f..ad4b5ca02 100644 --- a/tests/conformance/reporting/test_reporting_feed_transactions.py +++ b/tests/conformance/reporting/test_reporting_feed_transactions.py @@ -239,9 +239,9 @@ async def test_verified_finish_capture_and_epoch_zero_queue_are_frozen_without_r async def test_postgres_captures_on_one_connection_under_account_lock_and_uses_database_time( monkeypatch, ): - from psycopg_pool import AsyncConnectionPool - async with isolated_reporting_pool(autocommit=True) as outer: + from psycopg_pool import AsyncConnectionPool + async with AsyncConnectionPool( outer.conninfo, kwargs=outer.kwargs, min_size=1, max_size=1, open=False ) as pool: From 50e35f0ae3540f19b40e8fc460f5870dfe018bf9 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 18 Sep 2026 01:29:08 +0000 Subject: [PATCH 4/5] fix(reporting): preserve status errors and release feed writer locks --- docs/reporting-frozen-feed.md | 33 +- src/adcp/reporting/feed/errors.py | 4 + src/adcp/reporting/feed/memory.py | 6 + src/adcp/reporting/feed/pg.py | 117 ++++-- src/adcp/reporting/feed/projection.py | 5 +- src/adcp/reporting/feed/store.py | 9 +- src/adcp/reporting/receipts/handler.py | 59 ++- .../reporting/test_reporting_feed_process.py | 4 +- .../test_reporting_feed_transactions.py | 341 +++++++++++++++++- .../test_reporting_feed_transports.py | 120 ++++++ tests/type_checks/reporting_frozen_feed.py | 11 + 11 files changed, 649 insertions(+), 60 deletions(-) diff --git a/docs/reporting-frozen-feed.md b/docs/reporting-frozen-feed.md index 0f3e69f5a..0ebe1e2d0 100644 --- a/docs/reporting-frozen-feed.md +++ b/docs/reporting-frozen-feed.md @@ -33,13 +33,32 @@ deliveries, pending attempts, checks and private captured inputs stay private. Consumer privacy applies with `consumer_status_enabled` both off and on. Core adjustments appear once. -The first page acquires the account lock, captures Core and caller histories, -both visible maxima and database `as_of`, and persists the **actual wire -records** and private historical inputs on that same connection. It never -settles an issue, acknowledges work, changes receipt replay or mutates a -notification queue. Partial capture, insertion or page assembly rolls back -the entire snapshot transaction. Memory follows the same unconditional rollback -boundary, including newly initialized collections and sequence heads. +The first page captures Core and caller histories and database `as_of` on one +connection under the account lock. PostgreSQL releases that transaction and +pooled connection before projecting the detached inputs, closing dependencies, +serializing the **actual wire records** and constructing the first page in a +worker thread. Receipt and materializer writers can commit during projection, +even with a one-connection pool; their later writes belong to the next walk. +Visible maxima, ownership, readability, counts and wire membership all come +from the captured inputs. + +The handler rechecks the exact canonical account and consumer after projection, +before publication. A changed account alias or revoked identity cannot publish +that prepared snapshot. This request-local callback runs without a borrowed +PostgreSQL connection, so the application ACL can share the same pool. A short +second transaction inserts the completed immutable snapshot without taking the +account writer lock. The handler also reauthorizes before returning a page, +including continuations. An already persisted snapshot is never reconstructed. + +Capture, projection or page-construction failure publishes no snapshot; insert +or commit failure rolls back its publication. Cancellation during projection +cannot later publish a discarded result. These failures never undo another +writer's committed work. Feed reads never settle an issue, acknowledge work, +change receipt replay or mutate a notification queue. Memory retains its +unconditional rollback boundary, including newly initialized collections and +sequence heads. Call PostgreSQL feed reads outside `store.transaction()`: +`REPORTING_FEED_TRANSACTION_UNAVAILABLE` rejects an outer transaction whose +writer lock or uncommitted history could otherwise survive the capture phase. Ordering is `(domain_rank, sequence, record_kind, record_id)`, with Core rank 0 and caller reconciliation rank 1. Each domain retains its original committed diff --git a/src/adcp/reporting/feed/errors.py b/src/adcp/reporting/feed/errors.py index bd165c1b2..e00d1d694 100644 --- a/src/adcp/reporting/feed/errors.py +++ b/src/adcp/reporting/feed/errors.py @@ -11,6 +11,7 @@ "REPORTING_FEED_SCHEMA_UNREADY", "REPORTING_FEED_HISTORY_CORRUPT", "REPORTING_FEED_STORAGE_UNAVAILABLE", + "REPORTING_FEED_TRANSACTION_UNAVAILABLE", ] _MESSAGES: dict[FeedErrorCode, str] = { @@ -22,6 +23,9 @@ "REPORTING_FEED_STORAGE_UNAVAILABLE": ( "reporting feed storage is unavailable; retry the request" ), + "REPORTING_FEED_TRANSACTION_UNAVAILABLE": ( + "read reporting feeds outside an existing ledger transaction" + ), } diff --git a/src/adcp/reporting/feed/memory.py b/src/adcp/reporting/feed/memory.py index 99e649ea8..7ddfd641b 100644 --- a/src/adcp/reporting/feed/memory.py +++ b/src/adcp/reporting/feed/memory.py @@ -5,6 +5,7 @@ import hashlib import json import secrets +from collections.abc import Awaitable, Callable from typing import Any from adcp.reporting.canonical_json import canonical_json_utf8_v1 @@ -98,6 +99,7 @@ async def read_reporting_feed( *, caller: ReportingDeliveryPrincipal, consumer_status_enabled: bool = False, + reauthorize: Callable[[], Awaitable[None]] | None = None, ) -> dict[str, Any]: parsed = FeedRequest.parse(request) try: @@ -123,7 +125,11 @@ async def read_reporting_feed( self._capture_feed(caller, parsed, after, consumer_status_enabled), secrets.token_bytes(32), ) + if reauthorize is not None: + await reauthorize() self._save_feed_snapshot(stored) + elif reauthorize is not None: + await reauthorize() return inject_context(request, stored.page(offset, parsed.limit)) except LedgerConflictError: raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") from None diff --git a/src/adcp/reporting/feed/pg.py b/src/adcp/reporting/feed/pg.py index 51aa94aff..778a0b9e4 100644 --- a/src/adcp/reporting/feed/pg.py +++ b/src/adcp/reporting/feed/pg.py @@ -2,9 +2,12 @@ from __future__ import annotations +import asyncio import hashlib import json import secrets +from collections.abc import Awaitable, Callable +from dataclasses import dataclass from importlib.resources import files from typing import Any @@ -20,7 +23,10 @@ decode_snapshot, token_position, ) +from adcp.reporting.ledger.delivery_changes import ReportingReconciliationChange from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.pg import _BOUND_CONNECTION +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot from adcp.reporting.ledger.status_snapshot import read_snapshot_on from adcp.reporting.materializer.capture import decode_materializer_boundary from adcp.reporting.materializer.pg import _now @@ -29,6 +35,55 @@ from adcp.server.helpers import inject_context +@dataclass(frozen=True, repr=False) +class _CapturedFeed: + core: ReportingStatusSnapshot + changes: tuple[ReportingReconciliationChange, ...] + materializer: tuple[dict[str, Any], ...] + receipts: tuple[dict[str, Any], ...] + + +@dataclass(frozen=True, repr=False) +class _PreparedFeed: + snapshot: ReportingFeedSnapshot + signing_key: bytes + document: str + content_sha256: str + page: dict[str, Any] + + +def _prepare_feed( + captured: _CapturedFeed, + caller: ReportingDeliveryPrincipal, + request: FeedRequest, + after: tuple[int, int], + consumer_status_enabled: bool, +) -> _PreparedFeed: + """Pure work over detached capture; never owns a connection or writer lock.""" + snapshot = capture_feed( + captured.core, + captured.changes, + caller=caller, + request=request, + after=after, + consumer_status_enabled=consumer_status_enabled, + materializer_boundaries=tuple( + decode_materializer_boundary(r) for r in captured.materializer + ), + receipt_boundaries=tuple(decode_receipt_boundary(r) for r in captured.receipts), + ) + stored = StoredFeedSnapshot(snapshot, secrets.token_bytes(32)) + document = canonical_json_utf8_v1(snapshot.to_storage()) + decode_snapshot(json.loads(document)) + return _PreparedFeed( + snapshot, + stored.signing_key, + document.decode(), + hashlib.sha256(document).hexdigest(), + stored.page(0, request.limit), + ) + + class PgReportingFeedStore(PgReportingReceiptStore): @storage_errors async def create_schema(self) -> None: @@ -67,10 +122,8 @@ async def _feed_snapshot_on( raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") return StoredFeedSnapshot(snapshot, bytes(row[2])) - async def _save_feed_snapshot_on(self, connection: Any, stored: StoredFeedSnapshot) -> None: + async def _save_feed_snapshot_on(self, connection: Any, stored: _PreparedFeed) -> None: snapshot = stored.snapshot - document = canonical_json_utf8_v1(snapshot.to_storage()) - decode_snapshot(json.loads(document)) await connection.execute( "INSERT INTO reporting_feed_snapshots" " (account_id,consumer_id,snapshot_id,as_of,representation_version,ownership_mode," @@ -82,8 +135,8 @@ async def _save_feed_snapshot_on(self, connection: Any, stored: StoredFeedSnapsh snapshot.as_of, snapshot.representation_version, snapshot.ownership_mode, - document.decode(), - hashlib.sha256(document).hexdigest(), + stored.document, + stored.content_sha256, stored.signing_key, ), ) @@ -92,10 +145,7 @@ async def _capture_feed_on( self, connection: Any, caller: ReportingDeliveryPrincipal, - request: FeedRequest, - after: tuple[int, int], - consumer_status_enabled: bool, - ) -> ReportingFeedSnapshot: + ) -> _CapturedFeed: # Database time and both histories share the writer's account lock. No # public reader, second connection, lifecycle settlement, or queue write. as_of = await _now(connection) @@ -120,15 +170,11 @@ async def _capture_feed_on( ).fetchall() if any(not r[1] for r in (*materializer, *receipts)): raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") - return capture_feed( + return _CapturedFeed( core, changes, - caller=caller, - request=request, - after=after, - consumer_status_enabled=consumer_status_enabled, - materializer_boundaries=tuple(decode_materializer_boundary(r[0]) for r in materializer), - receipt_boundaries=tuple(decode_receipt_boundary(r[0]) for r in receipts), + tuple(r[0] for r in materializer), + tuple(r[0] for r in receipts), ) @storage_errors @@ -138,10 +184,15 @@ async def read_reporting_feed( *, caller: ReportingDeliveryPrincipal, consumer_status_enabled: bool = False, + reauthorize: Callable[[], Awaitable[None]] | None = None, ) -> dict[str, Any]: parsed = FeedRequest.parse(request) + bound = _BOUND_CONNECTION.get() + if bound is not None and bound[:2] == (self._pool, asyncio.current_task()): + # A savepoint cannot release the outer transaction's account lock + # or establish that its uncommitted history is a public boundary. + raise ReportingFeedError("REPORTING_FEED_TRANSACTION_UNAVAILABLE") async with self._connection() as connection, connection.transaction(): - await self._lock_account(connection, caller.account_id) await validate_feed_schema(connection, notifications=self._notifications_enabled) stored = None offset = 0 @@ -166,19 +217,37 @@ async def read_reporting_feed( if stored is not None and stored.snapshot.after != after: raise ReportingFeedError("INVALID_CHECKPOINT") if stored is None: - snapshot = await self._capture_feed_on( - connection, caller, parsed, after, consumer_status_enabled - ) - stored = StoredFeedSnapshot(snapshot, secrets.token_bytes(32)) - await self._save_feed_snapshot_on(connection, stored) - return inject_context(request, stored.page(offset, parsed.limit)) + await self._lock_account(connection, caller.account_id) + captured = await self._capture_feed_on(connection, caller) + # Both the transaction and pooled connection have been released. Slow + # graph closure, canonicalization and token/page construction must not + # serialize receipt/materializer writers, including a size-one pool. + if stored is not None: + page = await asyncio.to_thread(stored.page, offset, parsed.limit) + if reauthorize is not None: + await reauthorize() + return inject_context(request, page) + prepared = await asyncio.to_thread( + _prepare_feed, captured, caller, parsed, after, consumer_status_enabled + ) + page = inject_context(request, prepared.page) + if reauthorize is not None: + # The application ACL can use the same size-one pool: no connection + # is borrowed while the callback rechecks this captured principal. + await reauthorize() + async with self._connection() as connection, connection.transaction(): + await validate_feed_schema(connection, notifications=self._notifications_enabled) + # This independent immutable row needs no account writer lock. + # Insert/commit failure publishes no page and cannot undo writers + # that committed after the historical capture boundary. + await self._save_feed_snapshot_on(connection, prepared) + return page @storage_errors async def read_reporting_feed_snapshot( self, snapshot_id: str, *, caller: ReportingDeliveryPrincipal ) -> ReportingFeedSnapshot | None: async with self._connection() as connection, connection.transaction(): - await self._lock_account(connection, caller.account_id) await validate_feed_schema(connection, notifications=self._notifications_enabled) stored = await self._feed_snapshot_on(connection, snapshot_id, caller) return stored.snapshot if stored else None diff --git a/src/adcp/reporting/feed/projection.py b/src/adcp/reporting/feed/projection.py index 86fee2429..ed56e9cb3 100644 --- a/src/adcp/reporting/feed/projection.py +++ b/src/adcp/reporting/feed/projection.py @@ -84,7 +84,10 @@ def capture_feed( materializer_boundaries: tuple[ReportingMaterializerBoundary, ...] = (), receipt_boundaries: tuple[ReportingReceiptBoundary, ...] = (), ) -> ReportingFeedSnapshot: - """Caller owns the account lock and connection until persistence completes. + """Project detached histories captured under one account-lock boundary. + + No live store, connection, mutable configuration, or current clock is read. + PostgreSQL releases its capture transaction before running this projection. Domain rank, original domain sequence, kind, and wire ID form the total order. The two sequence spaces are never compared or collapsed with max(). diff --git a/src/adcp/reporting/feed/store.py b/src/adcp/reporting/feed/store.py index ebb6d590f..4ce0e1810 100644 --- a/src/adcp/reporting/feed/store.py +++ b/src/adcp/reporting/feed/store.py @@ -2,6 +2,7 @@ from __future__ import annotations +from collections.abc import Awaitable, Callable from typing import Any, Protocol, runtime_checkable from adcp.reporting.feed.snapshot import ReportingFeedSnapshot @@ -10,7 +11,12 @@ @runtime_checkable class ReportingFeedStore(Protocol): - """Trusted caller seam. The mounted handler reauthorizes every request.""" + """Trusted caller seam. The mounted handler reauthorizes every request. + + The request-local callback rechecks the same canonical principal before + publishing a prepared snapshot or returning a continuation. It is never + retained in a store, snapshot, or shared handler state. + """ async def read_reporting_feed( self, @@ -18,6 +24,7 @@ async def read_reporting_feed( *, caller: ReportingDeliveryPrincipal, consumer_status_enabled: bool = False, + reauthorize: Callable[[], Awaitable[None]] | None = None, ) -> dict[str, Any]: ... async def read_reporting_feed_snapshot( diff --git a/src/adcp/reporting/receipts/handler.py b/src/adcp/reporting/receipts/handler.py index 94de912f3..c564b4e88 100644 --- a/src/adcp/reporting/receipts/handler.py +++ b/src/adcp/reporting/receipts/handler.py @@ -137,7 +137,7 @@ async def get_reporting_status( from adcp.reporting.feed.errors import ReportingFeedError from adcp.reporting.feed.request import FeedRequest from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler - from adcp.reporting.ledger.store import ReportingLedgerStore + from adcp.reporting.ledger.store import LedgerConflictError, ReportingLedgerStore if self.reporting_feed_store is None: return self._not_supported("get_reporting_status") @@ -149,24 +149,38 @@ async def get_reporting_status( try: if request.get("view") == "periods": FeedRequest.parse(request) - if context is None: - raise ReportingFeedError("UNAUTHORIZED") - try: - consumer = await _consumer(context, self._receipt_registry) - account = await self._receipt_account_resolver( - dict(request["account"]), context, consumer - ) - if isinstance(context, RequestContext) and context.account.id != account: + + async def authorize() -> ReportingDeliveryPrincipal: + if context is None: raise ReportingFeedError("UNAUTHORIZED") - caller = ReportingDeliveryPrincipal(account, consumer) - except (ReportingReceiptError, ReportingNotificationError, ValueError, TypeError): - raise ReportingFeedError("UNAUTHORIZED") from None + try: + consumer = await _consumer(context, self._receipt_registry) + account = await self._receipt_account_resolver( + dict(request["account"]), context, consumer + ) + if isinstance(context, RequestContext) and context.account.id != account: + raise ReportingFeedError("UNAUTHORIZED") + return ReportingDeliveryPrincipal(account, consumer) + except (ReportingReceiptError, ReportingNotificationError, ValueError, TypeError): + raise ReportingFeedError("UNAUTHORIZED") from None + + caller = await authorize() if request.get("view") == "periods": - return await self.reporting_feed_store.read_reporting_feed( + + async def reauthorize() -> None: + # A still-authorized alias must not change which account or + # canonical consumer owns the already captured boundary. + if await authorize() != caller: + raise ReportingFeedError("UNAUTHORIZED") + + response = await self.reporting_feed_store.read_reporting_feed( request, caller=caller, consumer_status_enabled=self._feed_consumer_status_enabled, + reauthorize=reauthorize, ) + await reauthorize() + return response pagination = request.get("pagination") positions = ( request.get("changes_after"), @@ -177,11 +191,20 @@ async def get_reporting_status( # legacy projector. Leave ordinary Core requests compatible. raise ReportingFeedError("INVALID_CHECKPOINT") if isinstance(self.receipt_store, ReportingLedgerStore): - return await ReportingStatusHandler( - self.receipt_store, - consumer_status_enabled=self._feed_consumer_status_enabled, - ).handle(request, caller=ReportingStatusCaller(account, consumer)) - return self._not_supported("get_reporting_status") + try: + return await ReportingStatusHandler( + self.receipt_store, + consumer_status_enabled=self._feed_consumer_status_enabled, + ).handle( + request, + caller=ReportingStatusCaller(caller.account_id, caller.consumer_id), + ) + except LedgerConflictError as error: + # Only the legacy Core projector exposes its established + # domain errors. ACL/provider/feed failures stay redacted. + code, message = error.code, str(error) + else: + return self._not_supported("get_reporting_status") except ReportingFeedError as error: code, message = error.code, str(error) except Exception: diff --git a/tests/conformance/reporting/test_reporting_feed_process.py b/tests/conformance/reporting/test_reporting_feed_process.py index 1979cfcd7..a9f496643 100644 --- a/tests/conformance/reporting/test_reporting_feed_process.py +++ b/tests/conformance/reporting/test_reporting_feed_process.py @@ -53,8 +53,8 @@ async def test_process_crash_preserves_receipts_and_cold_mcp_a2a_pages(point, no before = without_feed(await h.image()) async with feed_process(h, s, feed_request(s), pause=point) as child: boundary = await child.event(point) - # Observe MVCC from another connection while the child holds the - # account lock; no partial snapshot or receipt/history mutation leaks. + # Observe MVCC at each phase: captured inputs and uncommitted INSERT + # are invisible; no partial snapshot or receipt/history change leaks. async with h.pool.connection() as c: count = ( await ( diff --git a/tests/conformance/reporting/test_reporting_feed_transactions.py b/tests/conformance/reporting/test_reporting_feed_transactions.py index ad4b5ca02..906dbc866 100644 --- a/tests/conformance/reporting/test_reporting_feed_transactions.py +++ b/tests/conformance/reporting/test_reporting_feed_transactions.py @@ -3,7 +3,10 @@ import asyncio import hashlib import json +import threading +from contextlib import asynccontextmanager from datetime import datetime, timezone +from types import SimpleNamespace from uuid import UUID import pytest @@ -13,14 +16,317 @@ from adcp.reporting.feed.snapshot import StoredFeedSnapshot from ._durable_materializer_support import DurableHarness, durable_case -from ._feed_support import feed_request, feeds, mixed_case, restart, walk, without_feed +from ._feed_support import ( + MountedFeed, + feed_harness, + feed_request, + feeds, + mixed_case, + restart, + walk, + without_feed, +) from ._generation_support import isolated_reporting_pool from ._receipt_support import receipt_case, request_for +from ._receipt_transport import error_code from ._reconciliation_support import Clock __all__ = ["feeds"] +@asynccontextmanager +async def paused_feed_projection(monkeypatch, *, failure=False): + """Pause the real pure projector, without blocking the test event loop.""" + from adcp.reporting.feed import pg + + loop = asyncio.get_running_loop() + loop_thread = threading.get_ident() + state = SimpleNamespace( + entered=asyncio.Event(), + release=threading.Event(), + finished=asyncio.Event(), + threads=[], + ) + original = pg.capture_feed + + def pause(*args, **kwargs): + state.threads.append(threading.get_ident()) + loop.call_soon_threadsafe(state.entered.set) + try: + if threading.get_ident() == loop_thread: + raise RuntimeError("feed projection must not block the writer event loop") + assert state.release.wait(30), "projection pause was not released" + if failure: + raise RuntimeError("private-projection-provider-failure") + return original(*args, **kwargs) + finally: + loop.call_soon_threadsafe(state.finished.set) + + with monkeypatch.context() as patch: + patch.setattr(pg, "capture_feed", pause) + try: + yield state + finally: + state.release.set() + if state.entered.is_set(): + await asyncio.wait_for(state.finished.wait(), 10) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("pool_size", [1, 2]) +@pytest.mark.parametrize("writer_kind", ["receipt", "materializer"]) +async def test_postgres_writers_commit_while_projection_is_paused_and_stay_out_of_frozen_walk( + monkeypatch, notifications, pool_size, writer_kind +): + async with isolated_reporting_pool(autocommit=True) as outer: + from psycopg_pool import AsyncConnectionPool + + async with AsyncConnectionPool( + outer.conninfo, + kwargs=outer.kwargs, + min_size=1, + max_size=pool_size, + open=False, + ) as pool: + store = PgReportingFeedStore(pool=pool, notifications=notifications) + await store.create_schema() + h = DurableHarness(store, Clock(), pool) + s = await receipt_case(h) if writer_kind == "receipt" else await durable_case(store) + caller = s.binding.principal + req = feed_request(s) + authorized = [] + + async def reauthorize(): + # A real ACL can share this size-one pool without waiting for + # the feed's capture or publication connection to be released. + async with pool.connection() as connection: + row = await ( + await connection.execute("SELECT count(*) FROM reporting_feed_snapshots") + ).fetchone() + assert row == (0,) + authorized.append(caller) + + async with paused_feed_projection(monkeypatch) as pause: + reader = asyncio.create_task( + store.read_reporting_feed(req, caller=caller, reauthorize=reauthorize) + ) + try: + await asyncio.wait_for(pause.entered.wait(), 10) + assert len(pause.threads) == 1 + assert pause.threads[0] != threading.get_ident() + assert not reader.done() + if writer_kind == "receipt": + result = await asyncio.wait_for( + store.ingest_receipt_batch(request_for(s), caller=caller), 10 + ) + assert result["results"][0]["result"] == "recorded" + else: + result = await asyncio.wait_for(s.service().run_once(), 10) + assert result.state == "verified" + # A mutable input changes after capture, before projection. + await asyncio.wait_for( + store.set_revision_readable( + account_id=caller.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ), + 10, + ) + written = await asyncio.wait_for(h.image(), 10) + assert written["reporting_feed_snapshots"] == [] + assert authorized == [] + assert not reader.done() + pause.release.set() + first = await asyncio.wait_for(reader, 10) + assert authorized == [caller] + finally: + pause.release.set() + if not reader.done(): + reader.cancel() + await asyncio.gather(reader, return_exceptions=True) + snapshot = await store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + assert snapshot.inputs["core"]["revisions"][0]["readable"] is True + assert snapshot.inputs["receipt_boundaries"] == [] + assert snapshot.inputs["materializer_boundaries"] == [] + _, records, checkpoint = await walk(store, req, caller, first=first) + assert records["receipts"] == [] + assert len(records["materializations"]) == int(writer_kind == "receipt") + assert without_feed(await h.image()) == without_feed(written) + fresh = await restart(h) + _, changes, _ = await walk(fresh, feed_request(s, changes_after=checkpoint), caller) + assert len(changes["periods"]) == len(changes["revisions"]) == 1 + if writer_kind == "receipt": + assert changes["receipts"] == [result["results"][0]["receipt"]] + assert await fresh.ingest_receipt_batch(request_for(s), caller=caller) == result + else: + assert len(changes["materializations"]) == 1 + assert changes["receipts"] == [] + assert set((await h.queue())[1]) <= {"quarantined"} + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("revocation", ["account-grant", "registry", "account-remap"]) +async def test_mounted_revocation_during_projection_denies_publication_and_preserves_continuation( + monkeypatch, notifications, revocation +): + async with feed_harness("postgres", notifications=notifications) as h: + s, _, _ = await mixed_case(h) + other = await receipt_case(h, account_id="feed-remapped-account") + mounted = MountedFeed(h, registry_kind="oauth") + mounted.authorize(s) + mounted.authorize(other, token="token-other") + req = feed_request(s) + async with mounted.client() as client: + _, first = await mounted.mcp(client, req) + old = feed_request(s, pagination={"cursor": first["pagination"]["cursor"]}) + before = await h.image() + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + async with paused_feed_projection(monkeypatch) as pause: + call = ( + mounted.mcp(client, req) + if transport == "mcp" + else mounted.a2a(client, req, v1=transport == "a2a-1.0") + ) + reader = asyncio.create_task(call) + try: + await asyncio.wait_for(pause.entered.wait(), 10) + assert pause.threads[0] != threading.get_ident() + assert not reader.done() + if revocation == "account-grant": + mounted.grants.remove((s.obligation.account_id, s.binding.consumer_id)) + elif revocation == "registry": + mounted.registry.agents.clear() + else: + mounted.accounts[s.obligation.account_id] = other.obligation.account_id + pause.release.set() + _, denied = await asyncio.wait_for(reader, 10) + assert error_code(denied) == "UNAUTHORIZED", denied + assert "ledger_snapshot_id" not in denied + finally: + pause.release.set() + if not reader.done(): + reader.cancel() + await asyncio.gather(reader, return_exceptions=True) + assert await h.image() == before + _, denied = await mounted.a2a(client, old) + # The remapped account is authorized, but cannot restore the + # original account's snapshot. Revoked grants fail earlier. + expected = "INVALID_CHECKPOINT" if revocation == "account-remap" else "UNAUTHORIZED" + assert error_code(denied) == expected, denied + assert "ledger_snapshot_id" not in denied + mounted.authorize(s) + _, resumed = await mounted.a2a(client, old) + assert resumed["ledger_snapshot_id"] == first["ledger_snapshot_id"] + assert resumed["changes_checkpoint"] == first["changes_checkpoint"] + assert await h.image() == before + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("fault", ["projection", "cancelled", "insert"]) +async def test_postgres_projection_failure_or_cancellation_cannot_undo_concurrent_receipt( + monkeypatch, notifications, fault +): + async with feed_harness("postgres", notifications=notifications) as h: + s = await receipt_case(h) + caller = s.binding.principal + if fault == "insert": + save = h.store._save_feed_snapshot_on + + async def fail_after_insert(*args, **kwargs): + await save(*args, **kwargs) + raise RuntimeError("private-snapshot-insert-failure") + + monkeypatch.setattr(h.store, "_save_feed_snapshot_on", fail_after_insert) + async with paused_feed_projection(monkeypatch, failure=fault == "projection") as pause: + reader = asyncio.create_task( + h.store.read_reporting_feed(feed_request(s), caller=caller) + ) + try: + await asyncio.wait_for(pause.entered.wait(), 10) + assert pause.threads[0] != threading.get_ident() + response = await asyncio.wait_for( + h.store.ingest_receipt_batch(request_for(s), caller=caller), 10 + ) + written = await h.image() + if fault == "cancelled": + reader.cancel() + with pytest.raises(asyncio.CancelledError): + await reader + pause.release.set() + else: + pause.release.set() + with pytest.raises(ReportingFeedError) as error: + await asyncio.wait_for(reader, 10) + assert error.value.code == "REPORTING_FEED_STORAGE_UNAVAILABLE" + assert error.value.__context__ is None + assert "private-" not in repr(error.value) + finally: + pause.release.set() + if not reader.done(): + reader.cancel() + await asyncio.gather(reader, return_exceptions=True) + assert await h.image() == written + await restart(h) + assert await h.store.ingest_receipt_batch(request_for(s), caller=caller) == response + assert await h.image() == written + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_postgres_feed_refuses_caller_transaction_before_reading_uncommitted_history( + notifications, +): + async with feed_harness("postgres", notifications=notifications) as h: + s = await receipt_case(h) + before = await h.image() + with pytest.raises(RuntimeError, match="rollback caller write"): + async with h.store.transaction(): + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(feed_request(s), caller=s.binding.principal) + assert error.value.code == "REPORTING_FEED_TRANSACTION_UNAVAILABLE" + raise RuntimeError("rollback caller write") + assert await h.image() == before + + +@pytest.mark.parametrize("failure", ["denied", "cancelled", "provider"]) +async def test_failed_reauthorization_leaves_no_snapshot_or_changed_history(feeds, failure): + h = feeds + s, request, response = await mixed_case(h) + before = await h.image() + calls = [] + + async def reauthorize(): + calls.append(s.binding.principal) + if failure == "denied": + raise ReportingFeedError("UNAUTHORIZED") + if failure == "cancelled": + raise asyncio.CancelledError + raise RuntimeError("private-authorization-provider-failure") + + expected = asyncio.CancelledError if failure == "cancelled" else ReportingFeedError + with pytest.raises(expected) as error: + await h.store.read_reporting_feed( + feed_request(s), caller=s.binding.principal, reauthorize=reauthorize + ) + assert calls == [s.binding.principal] + if failure != "cancelled": + assert error.value.code == ( + "UNAUTHORIZED" if failure == "denied" else "REPORTING_FEED_STORAGE_UNAVAILABLE" + ) + assert error.value.__context__ is None + assert "private-" not in repr(error.value) + assert await h.image() == before + await restart(h) + assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + assert await h.image() == before + + async def test_snapshot_identity_collision_cannot_replace_an_open_walk(feeds, monkeypatch): from adcp.reporting.feed import projection @@ -106,7 +412,7 @@ async def forbidden_capture(*args, **kwargs): assert await h.image() == before -@pytest.mark.parametrize("fault", ["capture", "insert", "assembly"]) +@pytest.mark.parametrize("fault", ["capture", "insert", "assembly", "context"]) async def test_every_snapshot_fault_rolls_back_all_collections_heads_and_upstream_history( feeds, monkeypatch, fault ): @@ -114,12 +420,17 @@ async def test_every_snapshot_fault_rolls_back_all_collections_heads_and_upstrea s, request, response = await mixed_case(h) before = await h.image() cls = type(h.store) - if fault == "assembly": + if fault in {"assembly", "context"}: def failing(*args, **kwargs): raise RuntimeError("injected feed assembly failure") - monkeypatch.setattr(StoredFeedSnapshot, "page", failing) + if fault == "assembly": + monkeypatch.setattr(StoredFeedSnapshot, "page", failing) + else: + from adcp.reporting.feed import memory, pg + + monkeypatch.setattr(memory if h.pool is None else pg, "inject_context", failing) elif h.pool is None: name = "_capture_feed" if fault == "capture" else "_save_feed_snapshot" original = getattr(cls, name) @@ -252,15 +563,32 @@ async def test_postgres_captures_on_one_connection_under_account_lock_and_uses_d h.clock.now = datetime(2000, 1, 1, tzinfo=timezone.utc) original = store._capture_feed_on connections = [] + transactions = [] async def capture(connection, *args, **kwargs): connections.append(connection) + transactions.append( + (await (await connection.execute("SELECT txid_current()")).fetchone())[0] + ) + async with outer.connection() as observer, observer.transaction(): + acquired = ( + await ( + await observer.execute( + "SELECT pg_try_advisory_xact_lock(hashtext(%s))", + (f"adcp.reporting:{s.obligation.account_id}",), + ) + ).fetchone() + )[0] + assert acquired is False return await original(connection, *args, **kwargs) save = store._save_feed_snapshot_on async def persist(connection, *args, **kwargs): connections.append(connection) + transactions.append( + (await (await connection.execute("SELECT txid_current()")).fetchone())[0] + ) return await save(connection, *args, **kwargs) monkeypatch.setattr(store, "_capture_feed_on", capture) @@ -278,13 +606,12 @@ async def persist(connection, *args, **kwargs): <= after ) assert len(connections) == 2 and connections[0] is connections[1] + assert transactions[0] != transactions[1] -async def test_same_account_receipt_writer_waits_for_complete_snapshot_then_is_deferred( +async def test_same_account_receipt_writer_waits_only_for_database_capture_then_is_deferred( monkeypatch, ): - from ._feed_support import feed_harness - async with feed_harness("postgres", notifications=True) as h: s = await receipt_case(h) entered, release = asyncio.Event(), asyncio.Event() diff --git a/tests/conformance/reporting/test_reporting_feed_transports.py b/tests/conformance/reporting/test_reporting_feed_transports.py index 944421c72..4abee6914 100644 --- a/tests/conformance/reporting/test_reporting_feed_transports.py +++ b/tests/conformance/reporting/test_reporting_feed_transports.py @@ -31,6 +31,126 @@ __all__ = ["feeds"] +@pytest.mark.parametrize( + "code,message", + [ + ("INVALID_VIEW", "unsupported reporting status view"), + ("MISSING_REVISION_ID", "a revision view requires reporting_revision_id"), + ("LOOKUP_UNAVAILABLE", "no such revision is available to this caller"), + ( + "CURSOR_SNAPSHOT_MISMATCH", + "this cursor belongs to a different snapshot, caller or filter set; restart the walk", + ), + ("STATUS_PROJECTION_UNAVAILABLE", "status lifecycle is pending"), + ], +) +async def test_mounted_legacy_status_domain_errors_retain_code_and_message( + feeds, monkeypatch, code, message +): + from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler + from adcp.reporting.ledger.store import LedgerConflictError + + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + before = await h.image() + callers = [] + + async def domain_error(self, request, *, caller): + assert request["view"] == "summary" + callers.append(caller) + raise LedgerConflictError(code, message) + + monkeypatch.setattr(ReportingStatusHandler, "handle", domain_error) + request = feed_request(s, view="summary") + del request["pagination"] + async with mounted.client() as client: + for call in (mounted.mcp, mounted.a2a, partial(mounted.a2a, v1=True)): + _, response = await call(client, request) + assert error_code(response) == code, response + assert message in json.dumps(response), response + with pytest.raises(ADCPTaskError) as error: + await mounted.handler.get_reporting_status( + request, ToolContext(caller_identity=s.binding.consumer_id) + ) + assert (error.value.errors[0].code, error.value.errors[0].message) == (code, message) + assert error.value.__context__ is None + assert callers == [ReportingStatusCaller(s.obligation.account_id, s.binding.consumer_id)] * 4 + assert await h.image() == before + + +async def test_real_legacy_status_errors_and_unexpected_provider_errors_stay_distinct( + feeds, monkeypatch +): + from adcp.reporting.feed.errors import ReportingFeedError + from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler + from adcp.reporting.ledger.store import LedgerConflictError, encode_cursor + + h = feeds + s, _, _ = await mixed_case(h) + mounted = MountedFeed(h) + mounted.authorize(s) + before = await h.image() + caller = ReportingStatusCaller(s.obligation.account_id, s.binding.consumer_id) + context = ToolContext(caller_identity=s.binding.consumer_id) + requests = [ + feed_request(s, view="invalid-view"), + feed_request(s, view="revision"), + feed_request(s, view="revision", reporting_revision_id="unavailable-revision"), + feed_request( + s, view="summary", pagination={"cursor": encode_cursor({"snapshot": "other"})} + ), + ] + for request in requests[:3]: + del request["pagination"] + validator = Draft7Validator(feed_schema("request"), format_checker=FormatChecker()) + async with mounted.client() as client: + for request in requests: + with pytest.raises(LedgerConflictError) as legacy: + await ReportingStatusHandler(h.store).handle(request, caller=caller) + with pytest.raises(ADCPTaskError) as mounted_error: + await mounted.handler.get_reporting_status(request, context) + assert mounted_error.value.errors[0].code == legacy.value.code + assert mounted_error.value.errors[0].message == str(legacy.value) + for call in (mounted.mcp, mounted.a2a, partial(mounted.a2a, v1=True)): + _, response = await call(client, request) + if validator.is_valid(request): + assert error_code(response) == legacy.value.code, response + assert str(legacy.value) in json.dumps(response) + else: + # Invalid schema shapes are rejected before the handler. + assert error_code(response) in {"INVALID_REQUEST", "VALIDATION_ERROR"}, response + + assert await h.image() == before + secret = "provider-secret-never-on-the-wire" + + async def provider_failure(*args, **kwargs): + raise RuntimeError(secret) + + monkeypatch.setattr(ReportingStatusHandler, "handle", provider_failure) + for call in (mounted.mcp, mounted.a2a, partial(mounted.a2a, v1=True)): + request = feed_request(s, view="summary") + del request["pagination"] + _, response = await call(client, request) + assert error_code(response) == "REPORTING_FEED_STORAGE_UNAVAILABLE" + assert secret not in json.dumps(response) + + async def acl_failure(*args, **kwargs): + raise LedgerConflictError("LOOKUP_UNAVAILABLE", secret) + + monkeypatch.setattr(mounted.handler, "_receipt_account_resolver", acl_failure) + with pytest.raises(ADCPTaskError) as error: + await mounted.handler.get_reporting_status(feed_request(s), context) + assert error.value.errors[0].code == "REPORTING_FEED_STORAGE_UNAVAILABLE" + assert error.value.errors[0].message == str( + ReportingFeedError("REPORTING_FEED_STORAGE_UNAVAILABLE") + ) + assert error.value.__context__ is None + assert secret not in repr(error.value) + assert await h.image() == before + + @pytest.fixture(autouse=True) def _a2a_compat_send_and_aggregate(): # Override the repository's unit-mock adapter shim. These mounted tests diff --git a/tests/type_checks/reporting_frozen_feed.py b/tests/type_checks/reporting_frozen_feed.py index 9fab8e1a7..3326538ba 100644 --- a/tests/type_checks/reporting_frozen_feed.py +++ b/tests/type_checks/reporting_frozen_feed.py @@ -7,6 +7,7 @@ from adcp.reporting.feed import ( InMemoryReportingFeedStore, PgReportingFeedStore, + ReportingFeedError, ReportingFeedSnapshot, ReportingFeedStore, ) @@ -40,6 +41,16 @@ async def adopter( materializer = memory assert_type(await feed.reporting_feed_ready(), bool) assert_type(await feed.read_reporting_feed(request, caller=caller), dict[str, Any]) + + async def reauthorize() -> None: + account = await resolve_account(request["account"], context, caller.consumer_id) + if account != caller.account_id: + raise ReportingFeedError("UNAUTHORIZED") + + assert_type( + await feed.read_reporting_feed(request, caller=caller, reauthorize=reauthorize), + dict[str, Any], + ) assert_type( await feed.read_reporting_feed_snapshot("snapshot", caller=caller), ReportingFeedSnapshot | None, From a9bc445446cb4579f21a73ded9cb71509d83d45f Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 20:45:16 +0000 Subject: [PATCH 5/5] fix(reporting): reconcile frozen feed gates and fixtures with main --- .github/workflows/ci.yml | 42 ++++++++++------ docs/reporting-frozen-feed.md | 23 +++++++-- .../reporting/_feed_installed_base.py | 5 +- tests/conformance/reporting/_feed_process.py | 23 ++++++--- tests/conformance/reporting/_feed_support.py | 4 +- .../conformance/reporting/_receipt_frozen.py | 6 ++- .../reporting/test_reporting_feed_contract.py | 12 +++-- .../reporting/test_reporting_feed_history.py | 25 +++++----- .../test_reporting_feed_installed_pg.py | 13 ++--- .../test_reporting_feed_migration.py | 49 ++++++++++++++----- .../reporting/test_reporting_feed_process.py | 9 ++-- .../reporting/test_reporting_feed_rolling.py | 15 +++--- .../test_reporting_feed_transactions.py | 26 ++++++---- .../test_reporting_feed_transports.py | 11 +++-- 14 files changed, 174 insertions(+), 89 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 015843bad..058f7507b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -488,13 +488,14 @@ jobs: # Crash processes and shared memory/SQL vectors have their own database # and bounded suite; setup and cleanup have five minutes of headroom. timeout-minutes: 25 + permissions: + contents: read services: postgres: image: postgres:16 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_feed_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: - 5432:5432 options: >- @@ -517,14 +518,21 @@ jobs: env: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_feed_test run: | + # Filter before pytest receives explicit paths; --ignore cannot do this. + shopt -s nullglob + feed_tests=() + for test_file in tests/conformance/reporting/test_reporting_feed_*.py; do + case "$test_file" in + tests/conformance/reporting/test_reporting_feed_rolling.py|tests/conformance/reporting/test_reporting_feed_packaging.py|tests/conformance/reporting/test_reporting_feed_installed_pg.py) ;; + *) feed_tests+=("$test_file") ;; + esac + done + if (( ${#feed_tests[@]} == 0 )); then + echo "No native feed test files selected" >&2 + exit 1 + fi python scripts/reporting_test_harness.py pytest \ - tests/conformance/reporting/test_reporting_feed_contract.py \ - tests/conformance/reporting/test_reporting_feed_positions.py \ - tests/conformance/reporting/test_reporting_feed_history.py \ - tests/conformance/reporting/test_reporting_feed_transports.py \ - tests/conformance/reporting/test_reporting_feed_transactions.py \ - tests/conformance/reporting/test_reporting_feed_migration.py \ - tests/conformance/reporting/test_reporting_feed_process.py \ + "${feed_tests[@]}" \ -v -s -ra | tee pg-reporting-feed-evidence.log - name: Preserve exact public, private, restart and fault evidence if: always() @@ -540,13 +548,14 @@ jobs: # Actual B2.1 produces the artifact and actual B2.2 mounts receipts on # both sides of migration for every historical reader/writer cell. timeout-minutes: 50 + permissions: + contents: read services: postgres: image: postgres:16 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_feed_rolling_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: - 5432:5432 options: >- @@ -563,12 +572,12 @@ jobs: 3e76aa54623529a3dda01cd690b8a5c287c75641 \ 3c405a21f978ed9d3208611bb4a7a8434a056933 \ 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ - 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 \ - 198d50e61c74fb82aedbf2c77e06a0e200b91db6 \ - ea150fabd5ad90e3abf93f89729d2919f1c61798 \ - 1c91311ec28d25506d5db43f59d0c34936ecb8f7 \ - 8e18ca12b9a0c3750f80aa822058c02982ab3e52 \ - 74b338d81653f4f65bc1520e638da10225f199f3 + 17ee407ae3978c8a2bb54437287afbf9dafb8130 \ + 0f34c666ac1961e9832fce43ef0ef6937b3c1dde \ + 967b6e286301d7e5d089aea6fdbb90bea8ee5a16 \ + 5487f2bdef23c5102118b305be9e868228f6ce61 \ + 3fd62121c96a074e3ea458c30c5224d6a586f169 \ + 09fd87f79a746665d828dea66b3a1dd9d1fc189e - uses: actions/setup-python@v6 with: python-version: "3.12" @@ -597,13 +606,14 @@ jobs: name: Installed frozen feed (Python 3.10 VCS and sdist) runs-on: ubuntu-latest timeout-minutes: 25 + permissions: + contents: read services: postgres: image: postgres:16 env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_feed_installed_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: - 5432:5432 options: >- diff --git a/docs/reporting-frozen-feed.md b/docs/reporting-frozen-feed.md index 0ebe1e2d0..70bebc17c 100644 --- a/docs/reporting-frozen-feed.md +++ b/docs/reporting-frozen-feed.md @@ -164,9 +164,10 @@ automation/facade and pinned cross-language program remain #1172 prerequisites. Leave the production application-clock override unset for DB-timed workers. 3. Check `await store.reporting_feed_ready()` and `await store.receipt_ingestion_ready()` in the typed composition. The isolated - feed manifest contains 33 objects. It adds to the preserved 453 ledger objects, - 187 materializer objects and 102 receipt objects: 775 enumerated objects when - those features are installed. Private fairness objects remain outside these + feed manifest contains 33 objects. It adds to the preserved 464 ledger objects, + 10 rc.6 waiver-binding objects, 187 materializer objects and 102 receipt objects: + 796 enumerated objects when those features are installed. With the complete + status and selector schemas, the exact combined catalog contains 1,045 objects. Private fairness objects remain outside these counts. No prior manifest acquires feed objects. Missing/partial/mismatched objects refuse new snapshots and continuation; no silent legacy fallback. 4. Mount the same authenticated handler on MCP/A2A. New snapshots use the new @@ -189,3 +190,19 @@ quarantined epoch-zero readiness records remain quarantined after restart, pending resume, replay, migration and rollback. Disabled notifications enqueue nothing; an enabled path retains its original atomic enqueue/rollback contract. Feed readiness does not release these higher-tier activation gates. + + +### Integrated rolling baselines + +The nine-artifact controls retain the beta15, records and integration binaries. +Their A/B/C/B1/B2.1/B2.2 pins now name the integrated commits `17ee407a`, +`0f34c666`, `967b6e28`, `5487f2bd`, `3fd62121` and `09fd87f7`, respectively. +Rolling compatibility with pre-`17ee407a` A, pre-`0f34c666` B, pre-`967b6e28` C, +pre-`5487f2bd` B1, pre-`3fd62121` B2.1 and pre-`09fd87f7` B2.2 binaries is +untested and unclaimed by these controls. Carry these limits into release notes. +The earlier A snapshot has locale-dependent schema fingerprints; the integrated +pins carry the catalog, transaction, permission and receipt-dispatch corrections. +Controls use the database's actual locale, without a C-only service setting. +A's whole-trigger readiness remains false after C; the supported restart path +uses the later subset readiness contract and still requires draining old workers. +These rolling checks do not activate epoch-zero/quarantined higher tiers. diff --git a/tests/conformance/reporting/_feed_installed_base.py b/tests/conformance/reporting/_feed_installed_base.py index 074959a14..0d58155ed 100644 --- a/tests/conformance/reporting/_feed_installed_base.py +++ b/tests/conformance/reporting/_feed_installed_base.py @@ -46,9 +46,10 @@ async def resolve(reference, context, consumer): old = ReportingReceiptHandler(InMemoryReportingReceiptStore(), resolve_account=resolve) new = ReportingReceiptHandler(store, resolve_account=resolve) - assert await old.get_reporting_status(request) == await ADCPHandler().get_reporting_status( + feed_condition_1 = await old.get_reporting_status( request - ) + ) == await ADCPHandler().get_reporting_status(request) + assert feed_condition_1 assert {t["name"] for t in get_tools_for_handler(old)} == { "get_adcp_capabilities", "sync_reporting_receipts", diff --git a/tests/conformance/reporting/_feed_process.py b/tests/conformance/reporting/_feed_process.py index bc6b9c4b9..713cf44b8 100644 --- a/tests/conformance/reporting/_feed_process.py +++ b/tests/conformance/reporting/_feed_process.py @@ -34,7 +34,8 @@ async def main(settings): async def pause(point, **evidence): if settings.get("pause") == point: print(json.dumps({"point": point, **evidence}), flush=True) - assert json.loads(await asyncio.to_thread(sys.stdin.readline))["continue"] + feed_operation_1 = await asyncio.to_thread(sys.stdin.readline) + assert json.loads(feed_operation_1)["continue"] receipt_only = settings.get("receipt_only", False) if receipt_only: @@ -61,9 +62,11 @@ async def _save_feed_snapshot_on(self, connection, stored): if settings["action"] == "install": await store.create_schema() await store.create_schema() - assert await store.receipt_ingestion_ready() + feed_operation_2 = await store.receipt_ingestion_ready() + assert feed_operation_2 if not receipt_only: - assert await store.reporting_feed_ready() + feed_operation_3 = await store.reporting_feed_ready() + assert feed_operation_3 if settings.get("legacy_status_schema"): from adcp.reporting.ledger import PgReportingReconciliationStore from adcp.reporting.outbox import PgStatusNotificationStore @@ -118,18 +121,23 @@ async def _save_feed_snapshot_on(self, connection, stored): expected.add("get_reporting_status") assert {tool["name"] for tool in inventory["tools"]} == expected for path in ("/.well-known/agent.json", "/.well-known/agent-card.json"): - assert {s["id"] for s in (await client.get(path)).json()["skills"]} == expected + feed_condition_4 = { + s["id"] for s in (await client.get(path)).json()["skills"] + } == expected + assert feed_condition_4 if settings["action"] == "receipt": _, response = await mount.mcp(client, settings["request"]) assert response["results"] for v1 in (False, True): - assert (await mount.a2a(client, settings["request"], v1=v1))[1] == response + feed_operation_6 = await mount.a2a(client, settings["request"], v1=v1) + assert (feed_operation_6)[1] == response mount.grants.clear() for call in (mount.mcp, mount.a2a): - assert ( + feed_condition_7 = ( transport.error_code((await call(client, settings["request"]))[1]) == "UNAUTHORIZED" ) + assert feed_condition_7 result = response else: @@ -163,12 +171,13 @@ def reporting(wire): pages[0]["ledger_snapshot_id"], caller=caller ) mount.grants.clear() - assert ( + feed_condition_5 = ( transport.error_code( (await mount.a2a(client, request, mutate_wire=reporting))[1] ) == "UNAUTHORIZED" ) + assert feed_condition_5 result = { "pages": pages, "binding": snapshot.binding, diff --git a/tests/conformance/reporting/_feed_support.py b/tests/conformance/reporting/_feed_support.py index 44407329a..3ef9ecfea 100644 --- a/tests/conformance/reporting/_feed_support.py +++ b/tests/conformance/reporting/_feed_support.py @@ -66,7 +66,7 @@ async def mixed_case(h, **kwargs): def feed_request(s, *, limit=1, **kwargs): return { - "adcp_version": "3.2-rc.3", + "adcp_version": "3.2-rc.6", "view": "periods", "account": {"account_id": s.obligation.account_id}, "pagination": {"max_results": limit}, @@ -215,7 +215,7 @@ def mcp_http(**kwargs): clients[protocol] = await stack.enter_async_context( ADCPClient( config, - adcp_version="3.2-rc.3", + adcp_version="3.2-rc.6", force_a2a_version=a2a_version if protocol == "a2a" else None, httpx_client_factory=mcp_http if protocol == "mcp" else None, ) diff --git a/tests/conformance/reporting/_receipt_frozen.py b/tests/conformance/reporting/_receipt_frozen.py index cca0b3764..870367657 100644 --- a/tests/conformance/reporting/_receipt_frozen.py +++ b/tests/conformance/reporting/_receipt_frozen.py @@ -63,7 +63,8 @@ async def main(settings): await PgStatusNotificationStore( ledger.PgReportingReconciliationStore(pool=pool, notifications=True) ).create_schema() - assert await store.materializer_ready() + feed_operation_1 = await store.materializer_ready() + assert feed_operation_1 manifest = json.loads( files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() ) @@ -169,7 +170,8 @@ async def project_ordinary_status(): assert readiness == (settings["artifact"] != "a") materializer = None if settings["artifact"] in {"b21", "b22"}: - assert await store.materializer_ready() + feed_operation_2 = await store.materializer_ready() + assert feed_operation_2 boundaries = await store.read_materializer_boundaries(caller=caller) assert len(boundaries) == 1 assert boundaries[0].to_storage()["version"] == 1 diff --git a/tests/conformance/reporting/test_reporting_feed_contract.py b/tests/conformance/reporting/test_reporting_feed_contract.py index f79f61d9a..fbc832e76 100644 --- a/tests/conformance/reporting/test_reporting_feed_contract.py +++ b/tests/conformance/reporting/test_reporting_feed_contract.py @@ -67,10 +67,10 @@ async def test_allowlist_exact_totals_dependency_order_and_final_only_checkpoint ): assert secret not in text assert without_feed(await h.image()) == before - assert ( - await h.store.ingest_receipt_batch(receipt_request, caller=s.binding.principal) - == receipt_response + feed_operation_1 = await h.store.ingest_receipt_batch( + receipt_request, caller=s.binding.principal ) + assert feed_operation_1 == receipt_response async def test_incremental_receipt_replays_old_exact_revision_materialization_adjustment_and_owner( @@ -139,7 +139,7 @@ async def test_foreign_consumer_writes_do_not_change_open_snapshot_or_visible_ve store, request, s.binding.principal, first=first, consumer_status_enabled=not feedback ) assert pages[-1]["pagination"]["total_count"] == 6 - assert ( + feed_condition_2 = ( rows["receipts"][0]["received_at"] != ( await store.read_reporting_feed( @@ -148,6 +148,7 @@ async def test_foreign_consumer_writes_do_not_change_open_snapshot_or_visible_ve )["receipts"][0]["received_at"] or h.pool is None ) + assert feed_condition_2 saved = await store.read_reporting_feed_snapshot( first["ledger_snapshot_id"], caller=s.binding.principal ) @@ -257,7 +258,8 @@ async def forbidden(*args, **kwargs): raise AssertionError("continuation consulted current projection") monkeypatch.setattr(store, "_capture_feed_on", forbidden) - assert await walk(store, req, s.binding.principal, first=first) == expected + feed_operation_3 = await walk(store, req, s.binding.principal, first=first) + assert feed_operation_3 == expected assert ( await store.read_reporting_feed_snapshot( first["ledger_snapshot_id"], caller=s.binding.principal diff --git a/tests/conformance/reporting/test_reporting_feed_history.py b/tests/conformance/reporting/test_reporting_feed_history.py index 6b84309bf..0f2c80341 100644 --- a/tests/conformance/reporting/test_reporting_feed_history.py +++ b/tests/conformance/reporting/test_reporting_feed_history.py @@ -80,7 +80,8 @@ async def test_snapshot_restatement_and_official_keep_exact_receipt_adjustment_t ): h = feeds case = await durable_case(h.store, count=3, reconciliation_mode="consumer_receipt") - assert (await case.service().run_once()).state == "verified" + feed_operation_1 = await case.service().run_once() + assert (feed_operation_1).state == "verified" request = { "account": {"account_id": case.config.account_id}, "view": "periods", @@ -91,7 +92,8 @@ async def test_snapshot_restatement_and_official_keep_exact_receipt_adjustment_t restatement = await case.publish( "restated-snapshot", finality="snapshot", supersedes=case.revision.reporting_revision_id ) - assert (await case.service().run_once()).state == "verified" + feed_operation_2 = await case.service().run_once() + assert (feed_operation_2).state == "verified" outcomes = await case.outcomes() retained_outcome = next( r for r in outcomes if r.reporting_revision_id == restatement.reporting_revision_id @@ -217,9 +219,10 @@ async def test_rejected_adjustment_digest_and_replacement_chain_are_preserved_wi ) assert response["results"][0]["result"] == "recorded" before = without_feed(await h.image()) - assert ( - await walk(await restart(h), feed_request(s), s.binding.principal, first=first) == original + feed_operation_3 = await walk( + await restart(h), feed_request(s), s.binding.principal, first=first ) + assert feed_operation_3 == original _, delta, _ = await walk( h.store, feed_request(s, changes_after=original[2]), s.binding.principal ) @@ -290,12 +293,10 @@ async def test_page_one_freezes_issue_waiver_status_replacement_configuration_an h.clock.now += timedelta(days=500) store = await restart(h) before = without_feed(await h.image()) - assert ( - await walk( - store, req, s.binding.principal, first=first, consumer_status_enabled=not feedback - ) - == expected + feed_operation_4 = await walk( + store, req, s.binding.principal, first=first, consumer_status_enabled=not feedback ) + assert feed_operation_4 == expected assert ( await store.read_reporting_feed_snapshot( first["ledger_snapshot_id"], caller=s.binding.principal @@ -401,7 +402,8 @@ async def test_late_committed_backdated_check_cannot_rewrite_receipt_admission(f ) ) before = without_feed(await h.image()) - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + feed_operation_5 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert feed_operation_5 == response fresh = await h.store.read_reporting_feed( feed_request(s, limit=100), caller=s.binding.principal ) @@ -445,7 +447,8 @@ async def test_conflicting_owner_in_retained_receipt_boundary_fails_new_capture_ ) store = await restart(h) before = await h.image() - assert (await walk(store, feed_request(s), s.binding.principal, first=first))[1]["receipts"] + feed_operation_6 = await walk(store, feed_request(s), s.binding.principal, first=first) + assert (feed_operation_6)[1]["receipts"] with pytest.raises(ReportingFeedError) as error: await store.read_reporting_feed(feed_request(s), caller=s.binding.principal) assert error.value.code == "REPORTING_FEED_HISTORY_CORRUPT" diff --git a/tests/conformance/reporting/test_reporting_feed_installed_pg.py b/tests/conformance/reporting/test_reporting_feed_installed_pg.py index a2067eb4b..463da2bc1 100644 --- a/tests/conformance/reporting/test_reporting_feed_installed_pg.py +++ b/tests/conformance/reporting/test_reporting_feed_installed_pg.py @@ -18,7 +18,7 @@ from ._durable_materializer_support import DurableHarness from ._feed_support import feed_request, mixed_case, second_consumer, walk, without_feed -from ._generation_support import assert_c_collated_rolling_database, isolated_reporting_pool +from ._generation_support import isolated_reporting_pool, require_rolling_database from ._reconciliation_support import Clock from .test_reporting_feed_packaging import ( ROOT, @@ -36,7 +36,7 @@ @pytest.fixture(scope="module", params=["vcs", "sdist"]) def installed_feed(request): - assert_c_collated_rolling_database() + require_rolling_database() root, wheels, assets = request.getfixturevalue("feed_wheels") interpreter = os.environ.get("ADCP_PYTHON310") or sys.executable environment = root / f"feed-pg-{request.param}" @@ -195,7 +195,8 @@ async def test_installed_python310_cold_continuation_freezes_mutable_history_and h, s, continuation, action="walk", transport="a2a", v1=v1, feedback=True, **options ) as child: done = await child.event("done") - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + feed_operation_2 = await asyncio.wait_for(child.process.wait(), 5) + assert feed_operation_2 == 0 assert done["result"]["pages"] == expected[0][1:] assert done["result"]["binding"] == original.binding assert done["result"]["version"] == 1 and done["result"]["ownership_mode"] == "absent" @@ -206,10 +207,10 @@ async def test_installed_python310_cold_continuation_freezes_mutable_history_and ) == original ) - assert ( - await h.store.ingest_receipt_batch(receipt_request, caller=s.binding.principal) - == receipt_response + feed_operation_1 = await h.store.ingest_receipt_batch( + receipt_request, caller=s.binding.principal ) + assert feed_operation_1 == receipt_response assert without_feed(await h.image()) == before print( json.dumps( diff --git a/tests/conformance/reporting/test_reporting_feed_migration.py b/tests/conformance/reporting/test_reporting_feed_migration.py index a3a333e87..886c997b1 100644 --- a/tests/conformance/reporting/test_reporting_feed_migration.py +++ b/tests/conformance/reporting/test_reporting_feed_migration.py @@ -8,7 +8,8 @@ from adcp.reporting.feed import PgReportingFeedStore, ReportingFeedError from adcp.reporting.materializer import ReportingMaterializerLease -from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects +from adcp.reporting.outbox.status_schema import REQUIRED_STATUS_OBJECTS from adcp.reporting.receipts import PgReportingReceiptStore from ._durable_materializer_support import DurableHarness, durable_case @@ -61,16 +62,32 @@ async def test_feed_migration_preserves_parent_catalog_receipts_pending_and_fair before = await h.image() async with pool.connection() as c: original = await schema_objects(c) - assert len(original) == 742 + waiver_objects = { + key: value + for key, value in REQUIRED_STATUS_OBJECTS.items() + if "reporting_issue_waiver_bindings" in key + } + assert len(waiver_objects) == 10 + parent_manifest = {**REQUIRED_OBJECTS, **waiver_objects} + for package in ("materializer", "receipts"): + parent_manifest.update( + json.loads( + files("adcp.reporting." + package).joinpath("required_schema.json").read_text() + ) + ) + assert len(parent_manifest) == 763 + assert original == parent_manifest new = PgReportingFeedStore(pool=pool, notifications=notifications) with pytest.raises(ReportingFeedError) as error: await new.reporting_feed_ready() assert error.value.code == "REPORTING_FEED_SCHEMA_UNREADY" await asyncio.gather(*(new.create_schema() for _ in range(3))) - assert await new.reporting_feed_ready() + feed_operation_1 = await new.reporting_feed_ready() + assert feed_operation_1 async with pool.connection() as c: actual = await schema_objects(c) assert {k: actual[k] for k in original} == original + assert actual == {**original, **MANIFEST} assert {k: v for k, v in actual.items() if "reporting_feed_" in k} == MANIFEST assert len(MANIFEST) == 33 for package, count in (("materializer", 187), ("receipts", 102)): @@ -89,9 +106,13 @@ async def test_feed_migration_preserves_parent_catalog_receipts_pending_and_fair await new.read_reporting_feed_snapshot(saved.snapshot_id, caller=s.binding.principal) == saved ) - assert await parent.ingest_receipt_batch(req, caller=s.binding.principal) == response + feed_operation_2 = await parent.ingest_receipt_batch(req, caller=s.binding.principal) + assert feed_operation_2 == response assert await fairness(pool) == turns - assert await parent.materializer_ready() and await parent.receipt_ingestion_ready() + feed_condition_3 = ( + await parent.materializer_ready() and await parent.receipt_ingestion_ready() + ) + assert feed_condition_3 async def test_interrupted_feed_migration_is_invisible_and_retry_retains_history(): @@ -128,10 +149,10 @@ async def migrate(): with pytest.raises(ReportingFeedError): await new.reporting_feed_ready() await new.create_schema() - assert ( - await parent.ingest_receipt_batch(request_for(s), caller=s.binding.principal) - == response + feed_operation_4 = await parent.ingest_receipt_batch( + request_for(s), caller=s.binding.principal ) + assert feed_operation_4 == response assert without_feed(await h.image()) == before @@ -141,10 +162,14 @@ async def migrate(): "ALTER TABLE reporting_feed_snapshots DISABLE TRIGGER reporting_feed_immutable", "ALTER TABLE reporting_feed_snapshots ALTER COLUMN document DROP NOT NULL", "DROP TABLE reporting_feed_snapshots", - "CREATE OR REPLACE FUNCTION reporting_feed_immutable() RETURNS TRIGGER" - " LANGUAGE plpgsql AS $body$ BEGIN RETURN NEW; END $body$", - "ALTER TABLE reporting_receipt_ingestion_results" - " DISABLE TRIGGER reporting_receipt_ingestion_result", + ( + "CREATE OR REPLACE FUNCTION reporting_feed_immutable() RETURNS TRIGGER" + " LANGUAGE plpgsql AS $body$ BEGIN RETURN NEW; END $body$" + ), + ( + "ALTER TABLE reporting_receipt_ingestion_results" + " DISABLE TRIGGER reporting_receipt_ingestion_result" + ), "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", ], ) diff --git a/tests/conformance/reporting/test_reporting_feed_process.py b/tests/conformance/reporting/test_reporting_feed_process.py index a9f496643..5fa622055 100644 --- a/tests/conformance/reporting/test_reporting_feed_process.py +++ b/tests/conformance/reporting/test_reporting_feed_process.py @@ -64,13 +64,15 @@ async def test_process_crash_preserves_receipts_and_cold_mcp_a2a_pages(point, no assert count == int(point == "committed") await child.kill() assert without_feed(await h.image()) == before - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + feed_operation_1 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert feed_operation_1 == response if point == "committed": first = boundary["result"] else: async with feed_process(h, s, feed_request(s)) as child: first = (await child.event("done"))["result"]["pages"][0] - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + feed_operation_2 = await asyncio.wait_for(child.process.wait(), 5) + assert feed_operation_2 == 0 expected = await walk(h.store, feed_request(s), s.binding.principal, first=first) continuation = feed_request( s, pagination={"max_results": 1, "cursor": first["pagination"]["cursor"]} @@ -80,7 +82,8 @@ async def test_process_crash_preserves_receipts_and_cold_mcp_a2a_pages(point, no h, s, continuation, action="walk", transport="a2a", v1=v1, feedback=True ) as child: result = (await child.event("done"))["result"] - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + feed_operation_3 = await asyncio.wait_for(child.process.wait(), 5) + assert feed_operation_3 == 0 assert result["pages"] == expected[0][1:] assert result["version"] == 1 and result["ownership_mode"] == "absent" assert without_feed(await h.image()) == before diff --git a/tests/conformance/reporting/test_reporting_feed_rolling.py b/tests/conformance/reporting/test_reporting_feed_rolling.py index 3bf508f75..4c946b13d 100644 --- a/tests/conformance/reporting/test_reporting_feed_rolling.py +++ b/tests/conformance/reporting/test_reporting_feed_rolling.py @@ -30,8 +30,8 @@ from .test_reporting_materializer_rolling import ARTIFACTS, build_frozen, frozen_call from .test_reporting_receipt_rolling import B21, immutable_parent_rows, receipt_probe -B22 = "74b338d81653f4f65bc1520e638da10225f199f3" -B22_TREE = "b694de7cb216b1ea8456f44ea9ccc9fb66257c79" +B22 = "09fd87f79a746665d828dea66b3a1dd9d1fc189e" +B22_TREE = "76c64bb94d8b92faed644158317bb5372b64fcb9" @pytest.fixture(scope="module") @@ -175,7 +175,8 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou ) as child: produced = await child.event("done") assert produced["state"] == "verified" - assert await asyncio.wait_for(child.process.wait(), 5) == 0 + feed_operation_3 = await asyncio.wait_for(child.process.wait(), 5) + assert feed_operation_3 == 0 outcome = (await case.outcomes())[0] second = await other_artifact_consumer(parent, case, outcome) evidence = outcome.verification @@ -201,7 +202,7 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou ), ) request = { - "adcp_version": "3.2-rc.3", + "adcp_version": "3.2-rc.6", "account": {"account_id": case.config.account_id}, "idempotency_key": "frozen-feed-mixed-batch", "receipts": [receipt_to_wire(receipt)], @@ -278,7 +279,8 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou assert second_result["result"] != admitted["result"] image = without_feed(await h.image()) fresh_reader = PgReportingFeedStore(pool=pool, notifications=notifications) - assert await walk(fresh_reader, query, case.scope.principal, first=first) == expected + feed_operation_1 = await walk(fresh_reader, query, case.scope.principal, first=first) + assert feed_operation_1 == expected assert ( await fresh_reader.read_reporting_feed_snapshot( frozen.snapshot_id, caller=case.scope.principal @@ -289,9 +291,10 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou assert await parent.read_receipt_boundaries(caller=case.scope.principal) == captures assert await immutable_parent_rows(pool) == saved assert await h.queue() == queue - assert await PgReportingMaterializerStore( + feed_operation_2 = await PgReportingMaterializerStore( pool=pool, notifications=notifications ).materializer_ready() + assert feed_operation_2 assert before["ordinary_core"] and after["ordinary_core"] assert ( before["ordinary_materializer"] diff --git a/tests/conformance/reporting/test_reporting_feed_transactions.py b/tests/conformance/reporting/test_reporting_feed_transactions.py index 906dbc866..69cffe0ab 100644 --- a/tests/conformance/reporting/test_reporting_feed_transactions.py +++ b/tests/conformance/reporting/test_reporting_feed_transactions.py @@ -159,7 +159,8 @@ async def reauthorize(): assert len(changes["periods"]) == len(changes["revisions"]) == 1 if writer_kind == "receipt": assert changes["receipts"] == [result["results"][0]["receipt"]] - assert await fresh.ingest_receipt_batch(request_for(s), caller=caller) == result + feed_operation_8 = await fresh.ingest_receipt_batch(request_for(s), caller=caller) + assert feed_operation_8 == result else: assert len(changes["materializations"]) == 1 assert changes["receipts"] == [] @@ -269,7 +270,8 @@ async def fail_after_insert(*args, **kwargs): await asyncio.gather(reader, return_exceptions=True) assert await h.image() == written await restart(h) - assert await h.store.ingest_receipt_batch(request_for(s), caller=caller) == response + feed_operation_7 = await h.store.ingest_receipt_batch(request_for(s), caller=caller) + assert feed_operation_7 == response assert await h.image() == written @@ -323,7 +325,8 @@ async def reauthorize(): assert "private-" not in repr(error.value) assert await h.image() == before await restart(h) - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response + feed_operation_1 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert feed_operation_1 == response assert await h.image() == before @@ -341,7 +344,8 @@ async def test_snapshot_identity_collision_cannot_replace_an_open_walk(feeds, mo await h.store.read_reporting_feed(req, caller=s.binding.principal) assert error.value.code == "REPORTING_FEED_STORAGE_UNAVAILABLE" assert await h.image() == before - assert await walk(h.store, req, s.binding.principal, first=first) == expected + feed_operation_2 = await walk(h.store, req, s.binding.principal, first=first) + assert feed_operation_2 == expected @pytest.mark.parametrize("damage", ["missing", "record-kind", "signature"]) @@ -454,10 +458,10 @@ async def failing(*args, **kwargs): assert await h.image() == before monkeypatch.undo() await restart(h) - assert await h.store.ingest_receipt_batch(request, caller=s.binding.principal) == response - assert (await walk(h.store, feed_request(s), s.binding.principal))[0][-1]["pagination"][ - "total_count" - ] == 6 + feed_operation_3 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert feed_operation_3 == response + feed_operation_4 = await walk(h.store, feed_request(s), s.binding.principal) + assert (feed_operation_4)[0][-1]["pagination"]["total_count"] == 6 assert without_feed(await h.image()) == without_feed(before) @@ -511,7 +515,8 @@ async def test_missing_references_fail_new_snapshot_but_never_rebuild_persisted_ await c.execute(query, (value,)) store = await restart(h) before = await h.image() - assert await walk(store, request, s.binding.principal, first=first) == expected + feed_operation_5 = await walk(store, request, s.binding.principal, first=first) + assert feed_operation_5 == expected with pytest.raises(ReportingFeedError) as error: await store.read_reporting_feed(request, caller=s.binding.principal) assert error.value.code == "REPORTING_FEED_HISTORY_CORRUPT" @@ -523,7 +528,8 @@ async def test_verified_finish_capture_and_epoch_zero_queue_are_frozen_without_r ): h = feeds case = await durable_case(h.store, count=3) - assert (await case.service().run_once()).state == "verified" + feed_operation_6 = await case.service().run_once() + assert (feed_operation_6).state == "verified" boundaries = await h.store.read_materializer_boundaries(caller=case.scope.principal) assert len(boundaries) == 1 request = { diff --git a/tests/conformance/reporting/test_reporting_feed_transports.py b/tests/conformance/reporting/test_reporting_feed_transports.py index 4abee6914..d6c4789a6 100644 --- a/tests/conformance/reporting/test_reporting_feed_transports.py +++ b/tests/conformance/reporting/test_reporting_feed_transports.py @@ -354,9 +354,10 @@ async def get_active(self, **kwargs): s, _, _ = await mixed_case(h) mounted = MountedFeed(h) mounted.authorize(s) - assert await mounted.handler.get_adcp_capabilities({}) == ( + feed_condition_1 = await mounted.handler.get_adcp_capabilities({}) == ( await ADCPHandler().get_adcp_capabilities({}) ) + assert feed_condition_1 before = without_feed(await h.image()) async with mounted.client() as client: for transport in (mounted.mcp, mounted.a2a): @@ -493,7 +494,7 @@ async def test_mounted_malformed_scoped_positions_and_generic_cache_cannot_bypas assert error_code(denied) in {"UNAUTHORIZED", "INVALID_REQUEST"} -@pytest.mark.parametrize("version", [None, "3.2.0-rc.3"]) +@pytest.mark.parametrize("version", [None, "3.2.0-rc.6"]) async def test_actual_inventory_and_fallback_schemas_bound_positions_without_mutating_legacy( feeds, version ): @@ -631,7 +632,8 @@ async def test_paired_handler_instances_keep_mcp_a2a_inventory_calls_and_authori feed.authorize(other_account, token="token-account") receipts.authorize(legacy) unsupported = await ADCPHandler().get_reporting_status(feed_request(s)) - assert await receipts.handler.get_reporting_status(feed_request(s)) == unsupported + feed_operation_2 = await receipts.handler.get_reporting_status(feed_request(s)) + assert feed_operation_2 == unsupported async with AsyncExitStack() as stack: clients = { name: await stack.enter_async_context(mounts[name].client()) for name in order @@ -722,6 +724,7 @@ async def test_paired_handler_instances_keep_mcp_a2a_inventory_calls_and_authori assert error_code(denied) == "UNAUTHORIZED" _, replay = await receipts.mcp(rc, request_for(legacy)) assert replay == legacy_response - assert await receipts.handler.get_reporting_status(feed_request(s)) == unsupported + feed_operation_3 = await receipts.handler.get_reporting_status(feed_request(s)) + assert feed_operation_3 == unsupported _, inventory = await receipts.mcp(rc, inventory=True) assert {t["name"] for t in inventory["tools"]} == expected["receipts"]