From a89f9900f9e69827161d51c6cd4a4afec74c9107 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Wed, 16 Sep 2026 23:06:54 +0000 Subject: [PATCH 1/5] feat(reporting): add destination verification and strict revision selection B1 of B1/B2: immutable writer/resolver contracts, SDK-owned verification, a non-production reference writer, strict whole-history revision selection, and a fenced C checkpoint semantics migration. No materializer persistence or Managed readiness is added. Refs #1167 --- .github/workflows/ci.yml | 9 +- MANIFEST.in | 1 + docs/reporting-destination-writer.md | 163 +++ docs/reporting-status-notifications.md | 53 +- examples/reporting_destination_writer.py | 77 ++ pyproject.toml | 1 + src/adcp/reporting/__init__.py | 14 +- src/adcp/reporting/_consumer.py | 46 +- src/adcp/reporting/_reconcile.py | 75 +- src/adcp/reporting/ledger/__init__.py | 14 + src/adcp/reporting/ledger/consumer_status.py | 55 +- src/adcp/reporting/ledger/delivery_models.py | 2 +- src/adcp/reporting/ledger/health.py | 95 +- src/adcp/reporting/ledger/pg.py | 6 +- src/adcp/reporting/ledger/producer.py | 62 +- .../reporting_status_selector_version.sql | 66 + .../reporting/ledger/status_projection.py | 51 +- src/adcp/reporting/ledger/status_snapshot.py | 32 +- src/adcp/reporting/ledger/store.py | 43 +- src/adcp/reporting/materializer/__init__.py | 93 ++ src/adcp/reporting/materializer/_json.py | 154 +++ .../assets/reference-canonicalization.json | 42 + .../assets/reference-definition.json | 48 + .../assets/reference-row-schema.json | 38 + src/adcp/reporting/materializer/contracts.py | 646 ++++++++++ src/adcp/reporting/materializer/reference.py | 368 ++++++ .../reporting/materializer/verification.py | 1107 +++++++++++++++++ src/adcp/reporting/outbox/__init__.py | 2 + src/adcp/reporting/outbox/_capabilities.py | 4 +- .../required_status_selector_schema.json | 42 + src/adcp/reporting/outbox/status.py | 23 +- src/adcp/reporting/outbox/status_memory.py | 115 +- src/adcp/reporting/outbox/status_pg.py | 147 ++- src/adcp/reporting/outbox/status_schema.py | 9 + src/adcp/reporting/outbox/status_service.py | 12 +- src/adcp/reporting/revision_selection.py | 191 +++ .../conformance/reporting/_frozen_status_c.py | 111 ++ .../reporting/_materializer_installed.py | 188 +++ .../reporting/_materializer_support.py | 178 +++ .../reporting/_reliable_support.py | 25 +- .../test_reporting_materializer_contracts.py | 214 ++++ .../test_reporting_materializer_lifecycle.py | 373 ++++++ .../test_reporting_materializer_packaging.py | 129 ++ .../test_reporting_materializer_readers.py | 426 +++++++ ...est_reporting_materializer_verification.py | 600 +++++++++ .../test_reporting_notification_packaging.py | 3 + .../test_reporting_notification_readiness.py | 3 +- .../test_reporting_revision_selection.py | 236 ++++ .../test_reporting_status_migration.py | 13 +- .../test_reporting_status_process_matrix.py | 7 +- .../test_reporting_status_selector_epoch.py | 417 +++++++ .../test_reporting_status_selector_pg.py | 416 +++++++ .../reporting_destination_writer.py | 134 ++ 53 files changed, 7193 insertions(+), 186 deletions(-) create mode 100644 docs/reporting-destination-writer.md create mode 100644 examples/reporting_destination_writer.py create mode 100644 src/adcp/reporting/ledger/reporting_status_selector_version.sql create mode 100644 src/adcp/reporting/materializer/__init__.py create mode 100644 src/adcp/reporting/materializer/_json.py create mode 100644 src/adcp/reporting/materializer/assets/reference-canonicalization.json create mode 100644 src/adcp/reporting/materializer/assets/reference-definition.json create mode 100644 src/adcp/reporting/materializer/assets/reference-row-schema.json create mode 100644 src/adcp/reporting/materializer/contracts.py create mode 100644 src/adcp/reporting/materializer/reference.py create mode 100644 src/adcp/reporting/materializer/verification.py create mode 100644 src/adcp/reporting/outbox/required_status_selector_schema.json create mode 100644 src/adcp/reporting/revision_selection.py create mode 100644 tests/conformance/reporting/_frozen_status_c.py create mode 100644 tests/conformance/reporting/_materializer_installed.py create mode 100644 tests/conformance/reporting/_materializer_support.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_contracts.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_lifecycle.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_packaging.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_readers.py create mode 100644 tests/conformance/reporting/test_reporting_materializer_verification.py create mode 100644 tests/conformance/reporting/test_reporting_revision_selection.py create mode 100644 tests/conformance/reporting/test_reporting_status_selector_epoch.py create mode 100644 tests/conformance/reporting/test_reporting_status_selector_pg.py create mode 100644 tests/type_checks/reporting_destination_writer.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f56ed5b7f..ad1ebff25 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,7 +4,7 @@ on: push: branches: [main, python-adcp-sdk-setup] pull_request: - branches: [main, conductor/reporting-webhook-activity-1168b] + branches: [main, conductor/reporting-webhook-activity-1168b, conductor/reporting-status-notifications-1168c] # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -88,7 +88,7 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' @@ -201,12 +201,13 @@ jobs: steps: - uses: actions/checkout@v6 - - name: Fetch exact reviewed A and B compatibility artifacts + - name: Fetch exact reviewed A, B and C compatibility artifacts timeout-minutes: 1 run: | git fetch --no-tags --depth=1 origin \ 21bf443e7d850d1800ec8a6f2e4abec1c8f85541 \ - 198d50e61c74fb82aedbf2c77e06a0e200b91db6 + 198d50e61c74fb82aedbf2c77e06a0e200b91db6 \ + ea150fabd5ad90e3abf93f89729d2919f1c61798 - name: Set up Python 3.12 uses: actions/setup-python@v6 diff --git a/MANIFEST.in b/MANIFEST.in index 476408259..432e859c6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include README.md include LICENSE include MIGRATION*.md recursive-include src/adcp py.typed +recursive-include src/adcp/reporting/materializer/assets *.json # Bundled AdCP JSON schemas. ``scripts/bundle_schemas.py`` mirrors # ``schemas/cache/`` into ``src/adcp/_schemas/`` before ``python -m # build`` so the validator ships with the wheel. Keep distributions on diff --git a/docs/reporting-destination-writer.md b/docs/reporting-destination-writer.md new file mode 100644 index 000000000..f2c12112a --- /dev/null +++ b/docs/reporting-destination-writer.md @@ -0,0 +1,163 @@ +# Verified destination I/O — #1167B1 + +**B1 of B1/B2**, stacked on reviewed #1168C at +`ea150fabd5ad90e3abf93f89729d2919f1c61798`. Refs #1167. + +B1 supplies immutable public contracts, whole-history revision selection, +SDK-owned source/destination verification, and a deterministic development +destination. Import from `adcp.reporting.materializer` and +`adcp.reporting.revision_selection`; PostgreSQL is optional. The strict +[adopter example](../examples/reporting_destination_writer.py) accepts existing +frozen ledger records and returns verification observations. + +There is no durable materializer, discovery queue, lease manager, retry +allocator, final outcome transaction, or materializer migration in B1. +`ReferenceReportingDestinationWriter.production_eligible` is always `False`, +including at the type level. It cannot be promoted by configuration or +subclassing. Configuring it, completing its readback, or freezing a destination +binding never advertises `managed_delivery`, `reconciled_billing`, or +`reporting.delivery_ready`. The real outbox readiness helper suppresses that +claim until B2 can prove a complete durable materializer. The producer's `extra` +argument rejects SDK-owned task, tier and notification keys. + +## Trusted contracts and lifecycle + +`ReportingDestinationRequest` includes the exact account and canonical consumer +(including HTTPS BuyerAgent identities), configuration generation, trusted +destination and binding references, binding fingerprint, obligation, revision, +materialization ID and attempt. Its verification key includes the complete +frozen definition/schema, canonicalization and capability tuples. Aliases must +be resolved by trusted middleware before this boundary. Credentials are never +protocol arguments, request fields, locators, verification records or metadata. + +`ReportingWriterCapability` closes method, transport, format, profile, readback +path, immutable-location/native-version mode, SHA-256 checksum and conditional +or idempotent write semantics. An unsupported tuple fails before resolver or +writer I/O. The immutable registry holds explicitly installed contract bytes; +it has no registration mutation, plugin callback or network fallback. + +A resolver synchronously constructs an **unopened** SDK +`ReportingDestinationSession`. Resource acquisition and fresh authorization +belong in `_open`; all credentials and partial acquisitions belong to that +redacted, non-persistable session. `_close` must release every acquired resource +and any adopter spool, including after an interrupted `_open`. Override the +protected hooks and I/O methods, preserving the SDK-owned context manager, +redaction and exactly-once close. Read methods must resolve locators inside the +session's trusted destination namespace and reject another tenant's paths. + +Write and readback open separate sessions and independently authorize the same +frozen request, so rotation/revocation between phases is observable. The SDK +checks the resolved request before opening and before I/O. `ReportingIOContext` +carries an absolute UTC deadline, cancellation event and optional service-owned +heartbeat checkpoint. It uses Python-3.10-compatible primitives, joins canceled +tasks, shields bounded cleanup and re-raises a clean `CancelledError`. The +optional heartbeat is a call boundary; B1 starts no lease-extension loop. + +Writers receive SDK-produced immutable canonical row bytes inside +`ReportingPreparedRevision`, never mutable row dictionaries or arbitrary +metadata. They return `ReportingDestinationLocator` claims. Counts, checksums, +manifest hashes and native commit IDs supplied by a writer establish where and +what to read; they cannot establish verification. + +`ReportingWriterFailure` contains only a closed code, retry instruction, +retry-after seconds and external-effect state (`not_started`, `applied`, +`unknown`). Unexpected provider failures become closed diagnostics without +provider exception chains. Do not construct messages from provider text or +log session internals, signed URLs, credentials or provider bodies. + +The external idempotency identity includes all tenant, consumer, binding, +revision and attempt coordinates. Resuming the same pending attempt preserves +it; equal attempt numbers on different revisions do not collide. Unknown +effects cannot request a new attempt. **Public foundation persistence still +allows N+1 after any immutable terminal outcome.** B2's autonomous retry rule +will be narrower: N+1 only after its own known terminal failure, never while N +is pending. Consumer receipt rejection is unchanged. Before activating B2, +drain legacy materialization writers and explicitly recover/import legacy +pending identities; their external-effect history cannot be inferred. + +## Installed canonicalizer and destination verification + +The small built-in implementation supports pinned `adcp_jcs_rows_v1` contracts, +Draft 2020-12 schemas with local fragment references, direct sum metrics and +complete integer/fixed-scale decimal control totals. Decimal values are strings; +JSON integers must be exactly typed and within the JavaScript safe range. +The pinned schema's `x-adcp-control-total` annotation specifies value type, +optional unit and decimal scale. All declared metrics must be represented. +The reference definition, row schema and canonicalization golden vectors are +bundled package assets with exact byte hashes in both distribution paths. + +This deliberately bounded JSON subset rejects floats, subclasses, tuples, +Decimal/datetime/bytes/set values, duplicate JSON members, invalid UTF-8 and lone +surrogates. It preserves Unicode without normalization and applies JCS UTF-16 +member ordering. Primary-key row ordering is by canonical scalar-key-array +bytes; duplicate primary keys fail. Golden vectors must test empty content, +nontrivial row ordering and member ordering. SHA-256 hexadecimal evidence is +validated and compared semantically, accepting uppercase and producing lowercase. + +Preparation reads **every** frozen source page, including zero rows and 501+ +rows, rederives the Core digest and the canonical digest, and recomputes typed +totals before destination authorization. SDK source cursors bind revision and +offset; custom row readers return the same `ReportingRowPage` identity/cursor +contract. Stable totals, cursor progress, cycles, `has_more` pairing and final +count are enforced. Source/destination walks bound bytes, recursive items, +nesting, rows, pages, objects and chunks. + +Readback independently verifies every logical destination row in order. File +verification also reads the exact manifest bytes and its closed schema, +identities, period, creation time, complete typed totals, ordered object +inventory, and every streamed JSONL object's checksum, length, row count and +content. Native verification observes the pinned version, location and required +consumer/destination path before and after all pages; each page repeats that +version. A native commit ID alone cannot satisfy canonical-digest verification. + +| Method | B1 reference format | Supported profiles | Required readback | +| --- | --- | --- | --- | +| File transfer | JSONL, uncompressed | canonical digest, manifest checksums | logical pages + exact manifest + every object | +| Dataset share | logical typed rows | canonical digest, native commit | representative-consumer rows + pinned native observations | +| Warehouse materialization | logical typed rows | canonical digest, native commit | destination rows + pinned native observations | + +The verifier returns `ReportingVerifiedDestination` only after these reads. +Corruption returns a closed failure and no reusable verified result. The public +foundation's materialization transition validator checks retained claims; it is +not an independent destination reader. Its persisted `MaterializationFailure` +variants and record shapes are unchanged. + +## Current revision and B2 finish seam + +The neutral selector returns `selected`, `not_ready` or `corrupt`. It validates +ownership, duplicate IDs, every predecessor, finality edges, connected snapshot +history, forks, cycles and multiple officials before selecting. Only empty +history or absence of a required official is ordinary not-ready. A unique +official wins over an intact retained snapshot chain; otherwise snapshot +finality selects its one unsuperseded leaf. Readability and materialization +availability never select a revision or allow fallback. + +Core health, producer acquisition, status projection/validation/lifecycle, +consumer planning and reconciliation use that selector. Producer corruption +fails before source or adapter I/O. Status emits a stable `HISTORY_UNAVAILABLE` +issue. The exported `current_required_revision(...) -> record | None` remains +source compatible; internal selection consumes the typed result. + +`validate_materialization_target` is a pure seam for B2's locked finish path. +It checks the selected/readable exact revision and frozen binding against the +prepared input. It provides no transaction or fencing claim. B2 must reselect +on the same account-locked connection and co-commit immutable outcome, +reconciliation feed, work acknowledgment, C dirty and readiness notification. +A stale-after-I/O attempt must close as a compatible public safe failure with +its richer reason isolated in B2 state; B1 does not add a persisted +`CURRENT_REVISION_CHANGED` variant. Neither B1 nor B2 merges autonomously. + +## C selector-epoch cutover + +B1's only SQL addition is the separately manifested **C checkpoint** migration +`reporting_status_selector_version.sql`. See the +[C rollout instructions](reporting-status-notifications.md#selector-epoch-cutover-1167b1). +Stop and drain old C projectors and sweepers before enabling v2 turns. A/B/C +Core and notification writers remain compatible; legacy materialization writers +must separately be drained before B2 activation. + +The frozen-C process gates exercise populated checkpoints, old `claim_due`, +old schema recreation, competing v2 projectors/sweepers, pool-local marker +cleanup and retained physical rows. Shared memory/PG vectors cover interrupted +fence, checkpoint/event and final-mark commits, ordered boundaries, late clocks, +retained scopes, unchanged fingerprints and once-only restart convergence. diff --git a/docs/reporting-status-notifications.md b/docs/reporting-status-notifications.md index 9d33f468f..6ff932c3c 100644 --- a/docs/reporting-status-notifications.md +++ b/docs/reporting-status-notifications.md @@ -105,12 +105,63 @@ PostgreSQL. PostgreSQL exports load lazily and construction without the extra raises an actionable `adcp[pg]` installation hint. `PgStatusNotificationStore.create_schema()` executes the six existing ledger/A/B -steps plus `reporting_status_notifications.sql` in one serialized transaction. +steps plus `reporting_status_notifications.sql` and +`reporting_status_selector_version.sql` in one serialized transaction. External migration tools must execute that same chain atomically. The original `required_schema.json`, A/B objects, functions and constraints are unchanged. `required_status_schema.json` independently validates C status and C activity objects. Missing C DDL suppresses status without changing B readiness. +## Selector epoch cutover (#1167B1) + +B1 changes whole-history revision selection and directional feed issue scopes. +Install the additive checkpoint migration, then **stop and drain old C +projectors and sweepers before scheduling v2 turns**. Core/source, A/B outbox and +C HTTP/activity writers remain usable. Installing schema alone does not fence +old C; account cutover is explicit, persisted and separate from its completion. + +Drive `ReportingStatusProjector.rebuild_once()` (or +`ReportingStatusService.rebuild_selector_once()`) until idle. This uses indexed +discovery of every populated stale account compatible with the store's original +escalation policy; no adopter account list or full periodic scan is needed. +The existing service's `drain()` includes these turns. Use the same escalation +policy as the original baseline; an explicitly targeted mismatched policy fails +closed. Ordinary `project_one(account_id=...)` also resumes that account's +interrupted cutover. + +Each first turn takes the existing account advisory lock, locks its checkpoints +and commits a checkpoint-local v2 writer floor plus an account transitioning +policy. A separate `selector_semantics_version` remains stale until projection +has committed. The new guard examines only the checkpoint and a +transaction-local v2 marker, avoiding an account-row lookup from old row-only +due claims. Old pending lease identities are retained, but old claims, +completions, projectors and readiness fail closed after the fence. Migration +does not wait for those leases to expire. Transaction-local markers are cleared +when connections return to the pool. Old named guards and their manifests are +unchanged, so old `create_schema()` cannot remove the independent v2 guard. + +Subsequent short transactions drain captured source boundaries in `through` +order, replay overdue semantic deadlines chronologically, then project current +state and mark the account complete atomically. Checkpoint/event failure rolls +back the entire turn. The scope union includes all retained checkpoints, +including those no longer returned by current scope discovery; absent source +history has a stable `HISTORY_UNAVAILABLE` result and no obsolete due deadline. + +No baseline, scope key, event, queue or activity history is deleted/reset. The +six-column scope identity, generation, baseline highwater, dirty cursor and +leases retain their meaning. Selector epoch is non-key metadata; canonical +fingerprints retain `version: 1`. Unchanged canonical health/issues only advance +the selector epoch and emit no event. Changed topology or issue membership +emits the corrected status with its existing previous health and next monotone +generation. Newly baselined accounts start at v2 without migration events. + +Readiness requires current schema, baseline, target epoch and zero stale or +incomplete checkpoint migrations. Account transitions are isolated. The memory +reference imports old shared-state images as v1 and performs the same restartable +transition. These objects and `required_status_selector_schema.json` belong only +to C checkpoint semantics; B1 adds no materializer persistence. See +[B1 destination I/O](reporting-destination-writer.md) for the B2 rollout dependency. + Default-off Core lifecycle writes also work before the A outbox migration. That schema has no issue-scope table: reads derive scope from retained status evidence until migration makes scope persistence available. Notification-enabled writers diff --git a/examples/reporting_destination_writer.py b/examples/reporting_destination_writer.py new file mode 100644 index 000000000..647209384 --- /dev/null +++ b/examples/reporting_destination_writer.py @@ -0,0 +1,77 @@ +"""B1 development destination: verify frozen content without advertising Managed. + +The trusted application supplies a frozen binding and its existing attempt. +Keep this object across retries to retain the reference writer's in-process +idempotency history. It is intentionally not durable or production eligible. +There is no scheduler, retry allocation, persistence, or readiness publication. +""" + +from __future__ import annotations + +import asyncio +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import datetime + +from adcp.reporting.ledger import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationBinding, + ReportingDestinationIO, + ReportingIOContext, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingRevisionRowReader, + ReportingRevisionVerifier, + ReportingRevisionVerifierRegistry, + ReportingVerifiedDestination, + reference_verifier, +) + + +@dataclass(frozen=True) +class DevelopmentDestination: + verifier: ReportingRevisionVerifier + registry: ReportingRevisionVerifierRegistry + resolver: ReferenceReportingResolver + writer: ReferenceReportingDestinationWriter + + async def verify_revision( + self, + *, + reader: ReportingRevisionRowReader, + binding: ReportingDestinationBinding, + delivery: ReportingObligationDeliveryRecord, + obligation: ReportingObligationRecord, + revisions: Sequence[ReportingRevisionRecord], + attempt: ReportingMaterializationAttempt, + deadline_at: datetime, + cancel: asyncio.Event, + ) -> ReportingVerifiedDestination: + # Complete source verification happens before the resolver can open a + # destination session. The same frozen attempt keeps its external ID. + prepared = await self.registry.prepare( + key=self.verifier.key, + reader=reader, + binding=binding, + delivery=delivery, + obligation=obligation, + revisions=revisions, + attempt=attempt, + context=ReportingIOContext(deadline_at, cancel), + ) + io = ReportingDestinationIO(self.registry, self.resolver) + locator = await io.write(prepared, context=ReportingIOContext(deadline_at, cancel)) + # A fresh session reauthorizes readback, including any intervening + # revocation/credential rotation. Credentials never enter these records. + return await io.verify(prepared, locator, context=ReportingIOContext(deadline_at, cancel)) + + +def development_destination(binding: ReportingDestinationBinding) -> DevelopmentDestination: + """Install the exact bundled JSONL example definition/canonicalization.""" + verifier = reference_verifier() + registry = ReportingRevisionVerifierRegistry((verifier,)) + writer = ReferenceReportingDestinationWriter((verifier.key.capability,)) + resolver = ReferenceReportingResolver(writer, registry, (binding,)) + return DevelopmentDestination(verifier, registry, resolver, writer) diff --git a/pyproject.toml b/pyproject.toml index 6ed19f860..8913ecfc1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -191,6 +191,7 @@ adcp = [ "decisioning/pg/*.sql", "reporting/ledger/*.sql", "reporting/outbox/*.json", + "reporting/materializer/assets/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/src/adcp/reporting/__init__.py b/src/adcp/reporting/__init__.py index b2f68881b..875f43e6e 100644 --- a/src/adcp/reporting/__init__.py +++ b/src/adcp/reporting/__init__.py @@ -75,10 +75,22 @@ from adcp.reporting import fixtures as fixtures from adcp.reporting import inline_source as inline_source from adcp.reporting import ledger as ledger + from adcp.reporting import materializer as materializer + from adcp.reporting import revision_selection as revision_selection from adcp.reporting import source as source _LAZY_SUBMODULES = frozenset( - {"canonical_json", "conformance", "currency", "fixtures", "inline_source", "ledger", "source"} + { + "canonical_json", + "conformance", + "currency", + "fixtures", + "inline_source", + "ledger", + "materializer", + "revision_selection", + "source", + } ) diff --git a/src/adcp/reporting/_consumer.py b/src/adcp/reporting/_consumer.py index d21f171fc..ba7b98ec8 100644 --- a/src/adcp/reporting/_consumer.py +++ b/src/adcp/reporting/_consumer.py @@ -40,6 +40,7 @@ from datetime import datetime, timedelta, timezone from typing import Any, Literal, Protocol +from adcp.reporting.revision_selection import RevisionHistoryEntry, select_reporting_revision from adcp.types import ( GetReportingStatusRequest, ReportingObligation, @@ -544,6 +545,9 @@ def add(intent: ConsumerStatusIntent | None) -> None: continue reading = readings.get(obligation_id) + history = obligation_revisions.get(obligation_id) + if history is not None: + _has_required_revision(obligation, history) # Reject damage even when a reading exists. status: ConsumerStatusValue mismatch: MismatchCode | None = None failure: ReportingFailureCode | None = None @@ -795,22 +799,36 @@ def _expected_at_of(period: Any) -> datetime | None: def _has_required_revision( obligation: ReportingObligation, revisions: Sequence[ReportingRevision] | None ) -> bool: - """Whether the seller has published a revision meeting the required finality. - - Prefers the supplied revisions, because ``required_finality`` matters: an - obligation needing ``official`` is not satisfied by snapshots. Falls back to - ``revision_count``, which the spec defines as the number of distinct - revision records for this obligation in the snapshot, so a caller that did - not pass revisions still gets the coarse answer rather than a wrong one. - """ + """Validate the caller's complete obligation partition before choosing finality.""" if revisions is not None: - required = str(getattr(obligation.required_finality, "value", obligation.required_finality)) - return any( - required == "snapshot" - or str(getattr(item.finality, "value", item.finality)) == "official" - for item in revisions + result = select_reporting_revision( + tuple( + RevisionHistoryEntry( + "wire", + obligation.reporting_obligation_id, + item.reporting_revision_id, + str(getattr(item.finality, "value", item.finality)), + item.supersedes_reporting_revision_id, + ) + for item in revisions + ), + account_id="wire", + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=str( + getattr(obligation.required_finality, "value", obligation.required_finality) + ), + ) + if result.kind == "corrupt" or len(revisions) != obligation.revision_count: + raise ConsumerStatusPlanError( + "the complete revision history requires repair before planning a consumer status" + ) + return result.kind == "selected" + if obligation.revision_count: + raise ConsumerStatusPlanError( + "the obligation advertises revisions but no reading was supplied; " + "a complete revision history is required to select finality" ) - return bool(obligation.revision_count) + return False def _source_timezone(obligation: ReportingObligation) -> str: diff --git a/src/adcp/reporting/_reconcile.py b/src/adcp/reporting/_reconcile.py index 65c539730..3868b53bb 100644 --- a/src/adcp/reporting/_reconcile.py +++ b/src/adcp/reporting/_reconcile.py @@ -40,6 +40,7 @@ post_consumer_statuses, resolve_checkpointed_leaves, ) +from adcp.reporting.revision_selection import RevisionHistoryEntry, select_reporting_revision from adcp.types import ( GetReportingStatusRequest, GetReportingStatusResponse, @@ -489,54 +490,50 @@ def _select_current( history_incomplete = True if history_incomplete: reasons.append("ASSOCIATED_HISTORY_INCOMPLETE") - superseded = { - item.supersedes_reporting_revision_id - for item in candidates - if item.supersedes_reporting_revision_id - } - candidate_ids = {item.reporting_revision_id for item in candidates} if any(not _revision_matches_obligation(item, obligation) for item in candidates) or any( item.reporting_revision_id in revision_ids and item.reporting_obligation_id != obligation.reporting_obligation_id for item in ledger.materializations ): reasons.append("REVISION_SCOPE_MISMATCH") - by_id = {item.reporting_revision_id: item for item in candidates} - leaves = [item for item in candidates if item.reporting_revision_id not in superseded] - # Walk every leaf back through its predecessors. A supersession cycle leaves - # its members unreachable, so a broken history cannot hide behind an official - # close the way a leaf-only count would let it. - reachable: set[str] = set() - for leaf in leaves: - node: ReportingRevision | None = leaf - while node is not None and node.reporting_revision_id not in reachable: - reachable.add(node.reporting_revision_id) - predecessor = node.supersedes_reporting_revision_id - node = by_id.get(predecessor) if predecessor else None - if ( - any( - item.supersedes_reporting_revision_id - and item.supersedes_reporting_revision_id not in candidate_ids + selection = select_reporting_revision( + tuple( + RevisionHistoryEntry( + ledger.account_id, + obligation.reporting_obligation_id, + item.reporting_revision_id, + _enum(item.finality), + item.supersedes_reporting_revision_id, + ) for item in candidates + ), + account_id=ledger.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=_enum(obligation.required_finality), + ) + if selection.kind == "corrupt": + reasons.append( + "AMBIGUOUS_REVISION_CHAIN" + if selection.reason + in { + "multiple_officials", + "forked_snapshot_history", + "disconnected_snapshot_history", + "duplicate_revision_id", + } + else "INCOMPLETE_REVISION_CHAIN" ) - or reachable != candidate_ids - ): - reasons.append("INCOMPLETE_REVISION_CHAIN") - # Publication selection precedes destination selection. An official close - # coexists with retained snapshots; it does not supersede their histories. - # A newer unmaterialized publication must never reveal an older snapshot as - # the current deliverable merely because that snapshot has a ready resource. - official = [item for item in candidates if _enum(item.finality) == "official"] - # Snapshot topology is judged on its own. Selecting the official close must - # never excuse a forked snapshot history the buyer cannot reconcile. - snapshot_leaves = [item for item in leaves if _enum(item.finality) != "official"] - if official and len(snapshot_leaves) > 1: - reasons.append("AMBIGUOUS_REVISION_CHAIN") - current = official or snapshot_leaves - if len(current) != 1: - reasons.append("MISSING_CURRENT_REVISION" if not current else "AMBIGUOUS_REVISION_CHAIN") return None, None, reasons - revision = current[0] + if selection.kind == "not_ready": + reasons.append( + "FINALITY_NOT_MET" + if selection.reason == "official_required" + else "MISSING_CURRENT_REVISION" + ) + return None, None, reasons + revision = next( + r for r in candidates if r.reporting_revision_id == selection.revision.reporting_revision_id + ) if ( not _coverage_is_full(obligation.coverage, obligation.media_buy_ids) or obligation.coverage.evaluated_at != obligation.scope_resolved_at diff --git a/src/adcp/reporting/ledger/__init__.py b/src/adcp/reporting/ledger/__init__.py index 71fb0c0e5..d273652a7 100644 --- a/src/adcp/reporting/ledger/__init__.py +++ b/src/adcp/reporting/ledger/__init__.py @@ -184,6 +184,14 @@ issue_is_retirable, reject_reserved_authoritative_party, ) +from adcp.reporting.revision_selection import ( + REPORTING_SELECTOR_VERSION, + ReportingRevisionCorrupt, + ReportingRevisionNotReady, + ReportingRevisionSelected, + ReportingRevisionSelection, + select_reporting_revision, +) if TYPE_CHECKING: from adcp.reporting.ledger.delivery_pg import PgReportingReconciliationStore @@ -194,6 +202,12 @@ ) __all__ = [ + "REPORTING_SELECTOR_VERSION", + "ReportingRevisionCorrupt", + "ReportingRevisionNotReady", + "ReportingRevisionSelected", + "ReportingRevisionSelection", + "select_reporting_revision", "ReportingStatusCallerResolver", "ReportingStatusNotificationHandler", "ReportingStatusSnapshot", diff --git a/src/adcp/reporting/ledger/consumer_status.py b/src/adcp/reporting/ledger/consumer_status.py index afe0bc3e8..b2ef9b949 100644 --- a/src/adcp/reporting/ledger/consumer_status.py +++ b/src/adcp/reporting/ledger/consumer_status.py @@ -68,7 +68,7 @@ from typing import Any, Literal, cast from adcp.reporting.canonical_json import canonical_json_utf8_v1 -from adcp.reporting.ledger.health import current_required_revision, issue_id_for +from adcp.reporting.ledger.health import issue_id_for from adcp.reporting.ledger.models import ( ConsumerStatusRecord, ConsumerStatusValue, @@ -82,6 +82,7 @@ ReportingRevisionRecord, ) from adcp.reporting.ledger.store import LedgerConflictError, ReportingLedgerStore +from adcp.reporting.revision_selection import select_reporting_revision __all__ = [ "CONSUMER_STATUS_ENABLED", @@ -244,6 +245,14 @@ async def _open_issue_on_first_observation(self, stored: ConsumerStatusRecord) - account_id=stored.account_id, reporting_obligation_id=stored.reporting_obligation_id, ) + if obligation is None: + obligation = await self.store.find_obligation( + account_id=stored.account_id, + delivery_config_id=stored.delivery_config_id, + delivery_config_version=stored.delivery_config_version, + period_start=stored.period_start, + period_end=stored.period_end, + ) revisions = ( await self.store.list_revisions( account_id=stored.account_id, @@ -252,7 +261,18 @@ async def _open_issue_on_first_observation(self, stored: ConsumerStatusRecord) - if obligation is not None else () ) - current = current_required_revision(obligation, revisions) if obligation else None + current = None + if obligation is not None: + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + return # Corruption cannot establish or resolve consumer disagreement. + if selection.kind == "selected": + current = selection.revision if not consumer_statement_conflicts( current=stored, current_revision=current, revisions=revisions ): @@ -352,6 +372,32 @@ async def _resolve_named_records(self, record: ConsumerStatusRecord) -> None: "this statement", ) + if obligation is None: + obligation = await self.store.find_obligation( + account_id=record.account_id, + delivery_config_id=record.delivery_config_id, + delivery_config_version=record.delivery_config_version, + period_start=record.period_start, + period_end=record.period_end, + ) + required = None + if obligation is not None: + revisions = await self.store.list_revisions( + account_id=record.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "the revision history requires repair" + ) + if selection.kind == "selected": + required = selection.revision if record.reporting_revision_id is None: return @@ -377,11 +423,6 @@ async def _resolve_named_records(self, record: ConsumerStatusRecord) -> None: # recording it would degrade the caller's own view over bytes neither # party stands behind any more. The buyer's move there is to re-read and # either accept or dispute the current revision. - revisions = await self.store.list_revisions( - account_id=record.account_id, - reporting_obligation_id=obligation.reporting_obligation_id, - ) - required = current_required_revision(obligation, revisions) if required is None or required.reporting_revision_id != record.reporting_revision_id: raise LedgerConflictError( "REVISION_NOT_CURRENTLY_REQUIRED", diff --git a/src/adcp/reporting/ledger/delivery_models.py b/src/adcp/reporting/ledger/delivery_models.py index b75bbfcdb..a7d693982 100644 --- a/src/adcp/reporting/ledger/delivery_models.py +++ b/src/adcp/reporting/ledger/delivery_models.py @@ -71,7 +71,7 @@ def _freeze(value: Any, annotation: Any) -> Any: except ValueError: continue elif origin is Literal: - if value in args and type(value) is type(args[0]): + if type(value) is type(args[0]) and value in args: return value elif origin is tuple and isinstance(value, (tuple, list)): if len(args) == 2 and args[1] is Ellipsis: diff --git a/src/adcp/reporting/ledger/health.py b/src/adcp/reporting/ledger/health.py index ed109c967..429e4e434 100644 --- a/src/adcp/reporting/ledger/health.py +++ b/src/adcp/reporting/ledger/health.py @@ -39,6 +39,7 @@ ReportingProductionStatus, ReportingRevisionRecord, ) +from adcp.reporting.revision_selection import select_reporting_revision __all__ = [ "ObligationProjection", @@ -107,12 +108,44 @@ def project_obligation_health( ) -> ObligationProjection: """Classify one obligation's immutable evidence at the snapshot's clock.""" boundary = _utc(ledger_as_of) - qualifying = [ - revision - for revision in revisions - if obligation.required_finality == "snapshot" or revision.finality == "official" - ] - current = _current_revision(qualifying) + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + current = selection.revision if selection.kind == "selected" else None + + if selection.kind == "corrupt": + return ObligationProjection( + health="action_required", + production_status="published" if revisions else "pending", + issues=( + ReportingIssue( + issue_id=issue_id_for( + "core-revision-history-corrupt-v2", + obligation.account_id, + obligation.reporting_obligation_id, + ), + code="HISTORY_UNAVAILABLE", + severity="action_required", + responsible_party="seller", + recommended_action="contact_seller", + reporting_obligation_id=obligation.reporting_obligation_id, + delivery_config_id=obligation.delivery_config_id, + delivery_config_version=obligation.delivery_config_version, + feed_purpose=obligation.feed_purpose, + media_buy_ids=obligation.media_buy_ids, + period_start=obligation.period.start, + period_end=obligation.period.end, + message=( + "The retained revision history is inconsistent; seller repair is required." + ), + ), + ), + satisfied=False, + current_revision=None, + ) if obligation.currency is None: return ObligationProjection( @@ -201,48 +234,18 @@ def current_required_revision( obligation: ReportingObligationRecord, revisions: Sequence[ReportingRevisionRecord], ) -> ReportingRevisionRecord | None: - """The revision the seller currently requires for this obligation. + """Source-compatible wrapper. Corrupt and not-ready histories both return None. - Shared by the health projection and the ``sync_reporting_status`` ingest on - purpose. If the two computed "current" differently, a buyer could file a - ``content_mismatch`` the ingest accepts and the projection then treats as - naming a superseded revision -- a statement permanently stuck disputing - bytes nobody stands behind. - - Applies the obligation's ``required_finality`` first, then takes the - unsuperseded leaf: an official revision is terminal so it wins outright, - and among snapshots the current one is whichever no other supersedes. + New callers should consume ``select_reporting_revision``'s discriminated + result so corruption can be parked for repair instead of retried as absence. """ - qualifying = [ - revision - for revision in revisions - if obligation.required_finality == "snapshot" or revision.finality == "official" - ] - return _current_revision(qualifying) - - -def _current_revision( - revisions: Sequence[ReportingRevisionRecord], -) -> ReportingRevisionRecord | None: - """The unsuperseded leaf of a revision chain. - - An official revision is terminal, so it wins outright. Among snapshots, - the current one is whichever no other snapshot supersedes. - """ - if not revisions: - return None - official = [item for item in revisions if item.finality == "official"] - if official: - return max(official, key=lambda item: (_utc(item.created_at), item.reporting_revision_id)) - superseded = { - item.supersedes_reporting_revision_id - for item in revisions - if item.supersedes_reporting_revision_id - } - leaves = [item for item in revisions if item.reporting_revision_id not in superseded] - if not leaves: - return None - return max(leaves, key=lambda item: (_utc(item.created_at), item.reporting_revision_id)) + result = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + return result.revision if result.kind == "selected" else None def _overdue_issue( diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index 043557379..6c4a64d92 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -124,7 +124,6 @@ ReportingRowPage, check_issue_state_transition, configuration_lifecycle, - decode_cursor, encode_cursor, issue_is_retirable, managed_revision_metadata, @@ -898,7 +897,9 @@ async def read_revision_rows( cursor: str | None = None, limit: int = 500, ) -> ReportingRowPage: - offset = int(decode_cursor(cursor).get("offset", 0)) if cursor else 0 + from adcp.reporting.ledger.store import revision_row_offset + + offset = revision_row_offset(cursor, reporting_revision_id, limit) async with self._connection() as connection: owned = await ( await connection.execute( @@ -920,6 +921,7 @@ async def read_revision_rows( total = int(owned[0]) has_more = offset + limit < total return ReportingRowPage( + reporting_revision_id=reporting_revision_id, rows=tuple(row[0] for row in rows), total_count=total, has_more=has_more, diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index ea180f5a9..3fce9fb71 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -57,6 +57,7 @@ LedgerConflictError, ReportingLedgerStore, ) +from adcp.reporting.revision_selection import select_reporting_revision from adcp.reporting.source import ( MediaBuyConstituentV1, ReportingConstituent, @@ -304,6 +305,20 @@ def advertised_reporting_delivery( payload["consumer_status_task"] = "sync_reporting_status" payload.update(self._escalation.to_wire()) if extra: + reserved = { + *payload, + "consumer_status_task", + "managed_delivery", + "reconciled_billing", + "reporting.delivery_ready", + "ledger_notification", + "readiness_notification", + "status_notification", + "supports_webhook_activity", + "receipt_task", + } + if any(key in reserved or key.endswith(("_task", "_notification")) for key in extra): + raise ValueError("extra cannot override SDK-owned reporting capabilities") payload.update(extra) return payload @@ -523,11 +538,20 @@ async def acquire_obligation( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - if any(item.finality == "official" for item in revisions): + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") + current = selection.revision if selection.kind == "selected" else None + if current is not None and current.finality == "official": # An official close is terminal. A later source correction is an # adjustment, never another acquisition. return None - satisfied = any(item.readable for item in revisions) + satisfied = current is not None and current.readable if satisfied and not restate: return None # Everything below needs the frozen code: the slice request carries it, @@ -654,9 +678,20 @@ async def commit_revision_from_manifest( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) - supersedes = None - if finality == "snapshot": - supersedes = self._current_snapshot_leaf(existing) + selection = select_reporting_revision( + existing, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality="snapshot", + ) + if selection.kind == "corrupt": + raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") + current = selection.revision if selection.kind == "selected" else None + supersedes = ( + current.reporting_revision_id + if finality == "snapshot" and current is not None and current.finality == "snapshot" + else None + ) control_totals = tuple((total.name, total.value) for total in manifest.control_totals) revision_id = f"rpr_{manifest.publication_id[4:44]}" @@ -756,23 +791,6 @@ def _validate_definition_binding( "the source definition differs from the obligation's pin", ) - @staticmethod - def _current_snapshot_leaf(revisions: Sequence[ReportingRevisionRecord]) -> str | None: - snapshots = [item for item in revisions if item.finality == "snapshot"] - if not snapshots: - return None - superseded = { - item.supersedes_reporting_revision_id - for item in snapshots - if item.supersedes_reporting_revision_id - } - leaves = [item for item in snapshots if item.reporting_revision_id not in superseded] - if not leaves: - return None - return max( - leaves, key=lambda item: (_utc(item.created_at), item.reporting_revision_id) - ).reporting_revision_id - # -- slice construction ---------------------------------------------- def _build_slice( diff --git a/src/adcp/reporting/ledger/reporting_status_selector_version.sql b/src/adcp/reporting/ledger/reporting_status_selector_version.sql new file mode 100644 index 000000000..9e31cce8f --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_status_selector_version.sql @@ -0,0 +1,66 @@ +-- #1167B1: C checkpoint semantics only. No materializer persistence. +-- Drain old C projectors/sweepers before cutover. A/B/C source and outbox +-- writers do not update checkpoints and remain compatible with this guard. +DO $selector$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + ALTER TABLE reporting_status_scope_checkpoints + ADD COLUMN IF NOT EXISTS selector_semantics_version INTEGER NOT NULL DEFAULT 1; + ALTER TABLE reporting_status_scope_checkpoints + ADD COLUMN IF NOT EXISTS selector_writer_floor INTEGER NOT NULL DEFAULT 1; + ALTER TABLE reporting_status_accounts + ADD COLUMN IF NOT EXISTS selector_target_version INTEGER NOT NULL DEFAULT 1; + ALTER TABLE reporting_status_accounts + ADD COLUMN IF NOT EXISTS selector_transition TEXT NOT NULL DEFAULT 'pending'; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_status_scope_checkpoints'::regclass + AND conname = 'reporting_status_selector_versions') THEN + ALTER TABLE reporting_status_scope_checkpoints + ADD CONSTRAINT reporting_status_selector_versions CHECK ( + selector_semantics_version IN (1,2) AND selector_writer_floor IN (1,2) + AND selector_semantics_version <= selector_writer_floor); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid = 'reporting_status_accounts'::regclass + AND conname = 'reporting_status_selector_account_transition') THEN + ALTER TABLE reporting_status_accounts + ADD CONSTRAINT reporting_status_selector_account_transition CHECK ( + (selector_target_version = 1 AND selector_transition = 'pending') OR + (selector_target_version = 2 AND selector_transition IN ('transitioning','complete'))); + END IF; + CREATE INDEX IF NOT EXISTS reporting_status_selector_rebuild + ON reporting_status_scope_checkpoints (account_id) + WHERE selector_semantics_version <> 2 OR selector_writer_floor <> 2; + CREATE INDEX IF NOT EXISTS reporting_status_selector_accounts + ON reporting_status_accounts (account_id) + WHERE baseline_complete AND (selector_target_version <> 2 OR selector_transition <> 'complete'); +END +$selector$; + +CREATE OR REPLACE FUNCTION reporting_status_selector_writer_guard_v2() RETURNS trigger +LANGUAGE plpgsql AS $guard$ +BEGIN + -- Check only this checkpoint. An account lookup here would invert the + -- account -> checkpoint lock order against old row-lock-only due claims. + IF (NEW.selector_writer_floor >= 2 OR + (TG_OP = 'UPDATE' AND OLD.selector_writer_floor >= 2)) AND + (current_setting('adcp.reporting.selector_semantics_version', true) IS DISTINCT FROM '2' + OR NEW.selector_writer_floor < 2 + OR (TG_OP = 'UPDATE' AND NEW.selector_semantics_version < OLD.selector_semantics_version)) THEN + RAISE EXCEPTION 'reporting_status_selector_writer_drain_required'; + END IF; + RETURN NEW; +END +$guard$; + +DO $selector_guard$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid = 'reporting_status_scope_checkpoints'::regclass + AND tgname = 'reporting_status_selector_writer_v2') THEN + CREATE TRIGGER reporting_status_selector_writer_v2 + BEFORE INSERT OR UPDATE ON reporting_status_scope_checkpoints + FOR EACH ROW EXECUTE FUNCTION reporting_status_selector_writer_guard_v2(); + END IF; +END +$selector_guard$; diff --git a/src/adcp/reporting/ledger/status_projection.py b/src/adcp/reporting/ledger/status_projection.py index 3b969e38f..481969b9b 100644 --- a/src/adcp/reporting/ledger/status_projection.py +++ b/src/adcp/reporting/ledger/status_projection.py @@ -24,7 +24,6 @@ from adcp.reporting.ledger.health import ( ObligationProjection, aggregate_reporting_health, - current_required_revision, issue_id_for, issue_id_for_occurrence, project_obligation_health, @@ -46,6 +45,7 @@ ReportingStatusScope, validate_scope_refinement, ) +from adcp.reporting.revision_selection import select_reporting_revision @dataclass(frozen=True) @@ -180,7 +180,20 @@ def lifecycle_intents(snapshot: ReportingStatusSnapshot) -> tuple[StatusLifecycl for r in snapshot.revisions if owner is not None and r.reporting_obligation_id == owner.reporting_obligation_id ) - required = current_required_revision(owner, revisions) if owner else None + required = None + if owner is not None: + selection = select_reporting_revision( + revisions, + account_id=owner.account_id, + reporting_obligation_id=owner.reporting_obligation_id, + required_finality=owner.required_finality, + ) + if selection.kind == "corrupt": + # Keep prior consumer occurrences intact until seller history + # can establish agreement. Health exposes HISTORY_UNAVAILABLE. + continue + if selection.kind == "selected": + required = selection.revision conflicts = consumer_statement_conflicts( current=status, current_revision=required, revisions=revisions ) @@ -324,6 +337,11 @@ def _selected(scope: ReportingStatusScope, target: ReportingStatusScope) -> bool return ( scope.account_id == target.account_id and (scope.consumer_id is None or scope.consumer_id == target.consumer_id) + and ( + scope.feed_purpose is None + or target.feed_purpose is None + or scope.feed_purpose == target.feed_purpose + ) and ( scope.generation_key is None or target.generation_key is None @@ -416,6 +434,7 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ c for c in snapshot.configurations if (scope.generation_key is None or c.generation_key == scope.generation_key) + and (scope.feed_purpose is None or c.feed_purpose == scope.feed_purpose) and configuration_selected( c, delivery_config_ids=value.delivery_config_ids, @@ -431,6 +450,7 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ # restored/custom stores. Never erase its health by filtering through # only the extant configuration rows; apply the same typed selection. if (scope.generation_key is None or o.generation_key == scope.generation_key) + and (scope.feed_purpose is None or o.feed_purpose == scope.feed_purpose) and (not value.delivery_config_ids or o.delivery_config_id in value.delivery_config_ids) and (not value.feed_purposes or o.feed_purpose in value.feed_purposes) and ( @@ -451,6 +471,33 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ projected: list[StatusObligationProjection] = [] candidates: set[datetime] = set() pending = 0 + if (scope.reporting_obligation_id is not None and not obligations) or ( + scope.generation_key is not None and not configurations and not obligations + ): + # A retained checkpoint can outlive the registry/obligation that made + # its scope discoverable. Keep the row and its event identity, publish + # the absence honestly, and leave no obsolete clock deadline behind. + issues.append( + ReportingIssue( + issue_id=issue_id_for( + "retained-scope-history-unavailable-v2", + canonical_json_utf8_v1(asdict(scope)).hex(), + ), + code="HISTORY_UNAVAILABLE", + severity="action_required", + responsible_party="seller", + recommended_action="contact_seller", + reporting_obligation_id=scope.reporting_obligation_id, + delivery_config_id=( + scope.generation_key.delivery_config_id if scope.generation_key else None + ), + delivery_config_version=( + scope.generation_key.delivery_config_version if scope.generation_key else None + ), + feed_purpose=scope.feed_purpose, + message="The retained reporting scope has no available source history.", + ) + ) retained_from = status_retained_from(configurations, snapshot.as_of) if value.period_start is not None and value.period_start < retained_from: for configuration in configurations: diff --git a/src/adcp/reporting/ledger/status_snapshot.py b/src/adcp/reporting/ledger/status_snapshot.py index 4f3434d93..3ee1a9e1e 100644 --- a/src/adcp/reporting/ledger/status_snapshot.py +++ b/src/adcp/reporting/ledger/status_snapshot.py @@ -13,7 +13,9 @@ StatusLifecycleIntent, apply_intents_to_snapshot, lifecycle_intents, + status_matches_obligation, ) +from adcp.reporting.revision_selection import select_reporting_revision if TYPE_CHECKING: from adcp.reporting.ledger.pg import PgReportingLedgerStore @@ -40,7 +42,6 @@ def validate_status_evidence( consume only already captured evidence and never acquire a connection. """ from adcp.reporting.ledger.consumer_status import validate_consumer_status_timing - from adcp.reporting.ledger.health import current_required_revision from adcp.reporting.ledger.store import LedgerConflictError generation = next( @@ -57,11 +58,7 @@ def validate_status_evidence( "SELLER_SNAPSHOT_EVIDENCE_INCOMPLETE", "seller snapshot evidence is incomplete" ) obligation = next( - ( - o - for o in snapshot.obligations - if o.reporting_obligation_id == status.reporting_obligation_id - ), + (o for o in snapshot.obligations if status_matches_obligation(status, o)), None, ) if status.reporting_obligation_id is not None: @@ -79,6 +76,23 @@ def validate_status_evidence( ): raise LedgerConflictError("OBLIGATION_IDENTITY_MISMATCH", "status evidence differs") validate_consumer_status_timing(status, generation, as_of=snapshot.as_of) + required = None + if obligation is not None: + revisions = tuple( + r + for r in snapshot.revisions + if r.reporting_obligation_id == obligation.reporting_obligation_id + ) + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") + if selection.kind == "selected": + required = selection.revision if status.reporting_revision_id is None: return revision = next( @@ -91,12 +105,6 @@ def validate_status_evidence( ): raise LedgerConflictError("LOOKUP_UNAVAILABLE", "status evidence is unavailable") if status.consumer_status == "content_mismatch" and obligation is not None: - revisions = tuple( - r - for r in snapshot.revisions - if r.reporting_obligation_id == obligation.reporting_obligation_id - ) - required = current_required_revision(obligation, revisions) if required is None or required.reporting_revision_id != status.reporting_revision_id: raise LedgerConflictError( "REVISION_NOT_CURRENTLY_REQUIRED", "status revision is no longer required" diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index cb4721770..19949bc45 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -166,6 +166,7 @@ class ReportingRowPage: total_count: int has_more: bool cursor: str | None + reporting_revision_id: str | None = None def encode_cursor(payload: dict[str, Any]) -> str: @@ -182,17 +183,46 @@ def encode_cursor(payload: dict[str, Any]) -> str: def decode_cursor(cursor: str) -> dict[str, Any]: padded = cursor + "=" * (-len(cursor) % 4) + + def unique_pairs(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate cursor member") + result[key] = value + return result + try: - payload = json.loads(base64.urlsafe_b64decode(padded.encode("ascii"))) - except Exception as error: - raise LedgerConflictError( - "INVALID_CURSOR", "the pagination cursor is not readable" - ) from error + payload = json.loads( + base64.urlsafe_b64decode(padded.encode("ascii")), object_pairs_hook=unique_pairs + ) + except Exception: + payload = None if not isinstance(payload, dict): raise LedgerConflictError("INVALID_CURSOR", "the pagination cursor is not readable") return payload +def revision_row_offset(cursor: str | None, revision_id: str, limit: int) -> int: + """Reject another revision's cursor and invalid count types before a row read.""" + if type(limit) is not int or not 1 <= limit <= 500: + raise LedgerConflictError("INVALID_PAGE_SIZE", "row page size must be between 1 and 500") + if cursor is None: + return 0 + if type(cursor) is not str or len(cursor) > 2048: + raise LedgerConflictError("INVALID_CURSOR", "the cursor does not bind this revision") + payload = decode_cursor(cursor) + offset = payload.get("offset") + if ( + set(payload) != {"revision", "offset"} + or payload["revision"] != revision_id + or type(offset) is not int + or offset < 0 + ): + raise LedgerConflictError("INVALID_CURSOR", "the cursor does not bind this revision") + return offset + + @runtime_checkable class ReportingLedgerStore(Protocol): """Durable home for obligations, revisions, adjustments, and statuses.""" @@ -1013,10 +1043,11 @@ async def read_revision_rows( if revision is None: raise LedgerConflictError("REVISION_NOT_FOUND", "no such revision for this account") rows = self._rows.get(reporting_revision_id, ()) - offset = int(decode_cursor(cursor).get("offset", 0)) if cursor else 0 + offset = revision_row_offset(cursor, reporting_revision_id, limit) window = rows[offset : offset + limit] has_more = offset + limit < len(rows) return ReportingRowPage( + reporting_revision_id=reporting_revision_id, rows=tuple(deepcopy(row) for row in window), total_count=len(rows), has_more=has_more, diff --git a/src/adcp/reporting/materializer/__init__.py b/src/adcp/reporting/materializer/__init__.py new file mode 100644 index 000000000..165d6d3d8 --- /dev/null +++ b/src/adcp/reporting/materializer/__init__.py @@ -0,0 +1,93 @@ +"""B1 public destination contracts and verification; no durable Managed service. + +Use the immutable registry to prepare all frozen source rows, then invoke write +and verify explicitly with separate authorization sessions. The reference +writer is exclusively for tests/development. B2 owns durable work, fencing, +retry allocation, final target reselection, and readiness transactions. +""" + +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.materializer._json import ( + ReportingVerificationLimits, + parse_reporting_json, + strict_reporting_json, +) +from adcp.reporting.materializer.contracts import ( + ReportingCanonicalization, + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationResolver, + ReportingDestinationSession, + ReportingDestinationWriter, + ReportingExternalEffect, + ReportingHeartbeat, + ReportingIOContext, + ReportingIOPhase, + ReportingNativeObservation, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterCapability, + ReportingWriterError, + ReportingWriterFailure, + ReportingWriterFailureCode, + ReportingWriterRetry, +) +from adcp.reporting.materializer.reference import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + reference_digest, + reference_verifier, +) +from adcp.reporting.materializer.verification import ( + ReportingDestinationIO, + ReportingRevisionRowReader, + ReportingRevisionVerifier, + ReportingRevisionVerifierRegistry, + ReportingVerifiedDestination, + validate_materialization_target, +) + +__all__ = [ + "ReferenceReportingDestinationWriter", + "ReferenceReportingResolver", + "ReportingCanonicalization", + "ReportingDeliveryPrincipal", + "ReportingDestinationBinding", + "ReportingDestinationIO", + "ReportingDestinationLocator", + "ReportingDestinationPage", + "ReportingDestinationRequest", + "ReportingDestinationResolver", + "ReportingDestinationSession", + "ReportingDestinationWriter", + "ReportingExternalEffect", + "ReportingHeartbeat", + "ReportingIOContext", + "ReportingIOPhase", + "ReportingMaterializationAttempt", + "ReportingNativeObservation", + "ReportingObligationDeliveryRecord", + "ReportingPreparedRevision", + "ReportingRevisionRowReader", + "ReportingRevisionVerifier", + "ReportingRevisionVerifierRegistry", + "ReportingVerificationKey", + "ReportingVerificationLimits", + "ReportingVerifiedDestination", + "ReportingWriterCapability", + "ReportingWriterError", + "ReportingWriterFailure", + "ReportingWriterFailureCode", + "ReportingWriterRetry", + "parse_reporting_json", + "reference_digest", + "reference_verifier", + "strict_reporting_json", + "validate_materialization_target", +] diff --git a/src/adcp/reporting/materializer/_json.py b/src/adcp/reporting/materializer/_json.py new file mode 100644 index 000000000..01ab467ea --- /dev/null +++ b/src/adcp/reporting/materializer/_json.py @@ -0,0 +1,154 @@ +"""Bounded exact JSON values at the new verification boundary.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, fields +from typing import cast + +from adcp.reporting.canonical_json import MAX_SAFE_INTEGER, JsonValue, canonical_json_utf8_v1 +from adcp.reporting.materializer.contracts import failure + + +@dataclass(frozen=True, slots=True) +class ReportingVerificationLimits: + max_depth: int = 32 + max_value_bytes: int = 1024 * 1024 + max_total_bytes: int = 64 * 1024 * 1024 + max_items: int = 1_000_000 + max_rows: int = 100_000 + max_pages: int = 10_000 + max_objects: int = 10_000 + max_chunks: int = 1_000_000 + + def __post_init__(self) -> None: + if any( + type(getattr(self, f.name)) is not int or getattr(self, f.name) < 1 + for f in fields(self) + ): + raise ValueError("verification limits require positive integer bounds") + if self.max_depth > 128: + raise ValueError("verification depth cannot exceed 128") + + +DEFAULT_LIMITS = ReportingVerificationLimits() + + +def strict_reporting_json( + value: object, limits: ReportingVerificationLimits = DEFAULT_LIMITS +) -> bytes: + """Encode exact JSON types, without coercion, subclass hooks or Unicode normalization. + + Integers must be JS-safe; exact decimals are strings. Floats, tuples, + subclasses and lone surrogates are deliberately outside this SDK profile. + """ + stack = [(value, 0)] + count, encoded_bytes = 0, 0 + while stack: + item, depth = stack.pop() + count += 1 + if depth > limits.max_depth or count > limits.max_items: + raise failure("LIMIT_EXCEEDED") + kind = type(item) + if kind is str: + text = cast(str, item) + if len(text) > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + encoded_bytes += 2 # Opening/closing quotes, including escaped UTF-8. + for char in text: + code = ord(char) + if 0xD800 <= code <= 0xDFFF: + raise failure("SOURCE_INVALID") + if char in '\\"\b\t\n\f\r': + encoded_bytes += 2 + elif code < 0x20: + encoded_bytes += 6 + else: + encoded_bytes += ( + 1 if code < 0x80 else 2 if code < 0x800 else 3 if code < 0x10000 else 4 + ) + if encoded_bytes > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + elif kind is int: + if abs(cast(int, item)) > MAX_SAFE_INTEGER: + raise failure("SOURCE_INVALID") + encoded_bytes += len(str(item)) + elif item is None or kind is bool: + encoded_bytes += 5 if item is False else 4 + elif kind is list: + values = cast(list[object], item) + if len(values) + count + len(stack) > limits.max_items: + raise failure("LIMIT_EXCEEDED") + encoded_bytes += 2 + max(0, len(values) - 1) + stack.extend((v, depth + 1) for v in values) + elif kind is dict: + mapping = cast(dict[object, object], item) + if len(mapping) * 2 + count + len(stack) > limits.max_items: + raise failure("LIMIT_EXCEEDED") + encoded_bytes += 2 + max(0, 2 * len(mapping) - 1) + for key, child in mapping.items(): + if type(key) is not str: + raise failure("SOURCE_INVALID") + stack.extend(((key, depth + 1), (child, depth + 1))) + else: + raise failure("SOURCE_INVALID") + if encoded_bytes > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + result = canonical_json_utf8_v1(value) + if len(result) > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + return result + + +def parse_reporting_json( + payload: bytes, limits: ReportingVerificationLimits = DEFAULT_LIMITS +) -> JsonValue: + """Reject ambiguous JSON before it can be normalized into a Python mapping.""" + if type(payload) is not bytes: + raise failure("SOURCE_INVALID") + if len(payload) > limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + # Bound parser recursion before json.loads allocates a nested tree. + depth, quoted, escaped = 0, False, False + for byte in payload: + if quoted: + if escaped: + escaped = False + elif byte == 92: + escaped = True + elif byte == 34: + quoted = False + elif byte == 34: + quoted = True + elif byte in (91, 123): + depth += 1 + if depth > limits.max_depth: + raise failure("LIMIT_EXCEEDED") + elif byte in (93, 125): + depth -= 1 + + def pairs(values: list[tuple[str, JsonValue]]) -> dict[str, JsonValue]: + result: dict[str, JsonValue] = {} + for key, value in values: + if key in result: + raise ValueError + result[key] = value + return result + + def reject(value: str) -> JsonValue: + raise ValueError + + invalid = False + try: + value = json.loads( + payload.decode("utf-8"), + object_pairs_hook=pairs, + parse_constant=reject, + parse_float=reject, + ) + except (ValueError, UnicodeError, RecursionError): + invalid = True + if invalid: + raise failure("SOURCE_INVALID") + strict_reporting_json(value, limits) + return cast(JsonValue, value) diff --git a/src/adcp/reporting/materializer/assets/reference-canonicalization.json b/src/adcp/reporting/materializer/assets/reference-canonicalization.json new file mode 100644 index 000000000..27d9d867c --- /dev/null +++ b/src/adcp/reporting/materializer/assets/reference-canonicalization.json @@ -0,0 +1,42 @@ +{ + "contract_version": "1.0", + "media_type": "application/vnd.adcp.reporting-canonicalization+json", + "algorithm": "adcp_jcs_rows_v1", + "schema_sha256": "aa4927d97b5e5889c4d79bd5461da81657cafea75a61d266ab452bfa0fc5d799", + "primary_keys": [ + "row_id" + ], + "golden_vectors": { + "empty_report": { + "name": "empty", + "purpose": "empty_report", + "input_rows": [], + "canonical_utf8_base64": "W10=", + "sha256": "4f53cda18c2baa0c0354bb5f9a3ecbe5ed12ab4d8e11ba873c2f11161202b945" + }, + "ordering_encoding": { + "name": "unicode-order", + "purpose": "ordering_encoding", + "input_rows": [ + { + "spend": "2.50", + "impressions": 2, + "currency": "USD", + "row_id": "z", + "details": { + "": "é", + "😀": "é" + } + }, + { + "spend": "1.00", + "impressions": 1, + "currency": "USD", + "row_id": "a" + } + ], + "canonical_utf8_base64": "W3siY3VycmVuY3kiOiJVU0QiLCJpbXByZXNzaW9ucyI6MSwicm93X2lkIjoiYSIsInNwZW5kIjoiMS4wMCJ9LHsiY3VycmVuY3kiOiJVU0QiLCJkZXRhaWxzIjp7IvCfmIAiOiLDqSIsIu6AgCI6ImXMgSJ9LCJpbXByZXNzaW9ucyI6Miwicm93X2lkIjoieiIsInNwZW5kIjoiMi41MCJ9XQ==", + "sha256": "3fb72666e0a391ef931445b5cebe16f8a6a801594acc885d4f2b1f18fb9a8eff" + } + } +} diff --git a/src/adcp/reporting/materializer/assets/reference-definition.json b/src/adcp/reporting/materializer/assets/reference-definition.json new file mode 100644 index 000000000..ce87305dd --- /dev/null +++ b/src/adcp/reporting/materializer/assets/reference-definition.json @@ -0,0 +1,48 @@ +{ + "contract_version": "1.1", + "media_type": "application/vnd.adcp.reporting-definition+json", + "report_definition_id": "reference-report-v1", + "reporting_profile": "paid_media_delivery", + "grain": "row", + "source": { + "provider": { + "domain": "example.test" + }, + "system": "non-production-reference", + "api_version": "1", + "query_semantics": { + "fixture_only": true + } + }, + "calendar": { + "timezone_basis": "utc" + }, + "metrics": [ + { + "name": "impressions", + "source_expression": "impressions", + "aggregation": "sum" + }, + { + "name": "spend", + "source_expression": "spend", + "aggregation": "sum", + "unit": "USD" + } + ], + "dimensions": [ + "row_id" + ], + "restatement_policy": { + "source_requery_duration": "P1D", + "emit_only_on_content_change": true, + "official_correction_mode": "adjustments_only" + }, + "finality_policies": [ + { + "finality_policy_id": "reference-final", + "basis": "source_final", + "source_signal": "fixture_closed" + } + ] +} diff --git a/src/adcp/reporting/materializer/assets/reference-row-schema.json b/src/adcp/reporting/materializer/assets/reference-row-schema.json new file mode 100644 index 000000000..86f40809a --- /dev/null +++ b/src/adcp/reporting/materializer/assets/reference-row-schema.json @@ -0,0 +1,38 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": { + "row_id": { + "type": "string", + "minLength": 1 + }, + "impressions": { + "type": "integer", + "minimum": 0, + "maximum": 9007199254740991, + "x-adcp-control-total": { + "value_type": "integer" + } + }, + "spend": { + "type": "string", + "pattern": "^(0|[1-9][0-9]{0,37})\\.[0-9]{2}$", + "x-adcp-control-total": { + "value_type": "decimal", + "scale": 2, + "unit": "USD" + } + }, + "currency": { + "const": "USD" + }, + "details": {} + }, + "required": [ + "row_id", + "impressions", + "spend", + "currency" + ], + "additionalProperties": false +} diff --git a/src/adcp/reporting/materializer/contracts.py b/src/adcp/reporting/materializer/contracts.py new file mode 100644 index 000000000..be70ca58e --- /dev/null +++ b/src/adcp/reporting/materializer/contracts.py @@ -0,0 +1,646 @@ +"""B1 destination I/O contracts. No scheduling, persistence or capability claims. + +Resolvers construct a session synchronously; authorization and resource acquisition +happen in its protected ``_open`` method, inside the SDK's owned async lifecycle. +Allocate credentials only there, retain them only on the redacted session, and +release partial acquisitions in ``_close``. Protocol callers supply opaque IDs. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import math +import re +from abc import ABC, abstractmethod +from collections.abc import AsyncIterator, Awaitable, Callable +from dataclasses import asdict, dataclass, field, replace +from datetime import datetime, timezone +from types import TracebackType +from typing import Any, Literal, Protocol, TypeVar + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import ( + ReportingCanonicalDigest, + aware_utc, + destination_reference, + file_object_reference, + native_version_reference, + reporting_identifier, + sha256_value, +) +from adcp.reporting.ledger.delivery_models import ( + DeliveryMethod, + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingFormat, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingResourceRecord, + VerificationPath, + VerificationProfile, + _ClosedValue, + _freeze_fields, +) +from adcp.reporting.ledger.models import ( + ReportingConfigurationGenerationKey, + ReportingDefinitionBinding, + ReportingObligationRecord, + ReportingRevisionRecord, +) + +ReportingIOPhase = Literal["write", "readback"] +ReportingExternalEffect = Literal["not_started", "applied", "unknown"] +ReportingWriterRetry = Literal["never", "same_identity", "new_attempt"] +ReportingWriterFailureCode = Literal[ + "UNSUPPORTED_VERIFICATION", + "HISTORY_CORRUPT", + "REVISION_NOT_READY", + "CURRENT_REVISION_CHANGED", + "AUTHORIZATION_DENIED", + "BINDING_MISMATCH", + "SOURCE_INVALID", + "DESTINATION_CORRUPT", + "RESOURCE_UNAVAILABLE", + "WRITE_FAILED", + "DEADLINE_EXCEEDED", + "LEASE_LOST", + "LIMIT_EXCEEDED", +] + + +@dataclass(frozen=True, slots=True) +class ReportingWriterFailure(_ClosedValue): + """Safe diagnostics only. Unknown effects must retain the original identity. + + ``new_attempt`` is usable by B2 only for its own known terminal failure. + Public ledger persistence still allows N+1 after any immutable outcome. + Consumer receipt rejection is outside this contract. + """ + + code: ReportingWriterFailureCode + retry: ReportingWriterRetry = "never" + effect: ReportingExternalEffect = "not_started" + retry_after_seconds: int | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + if self.effect == "unknown" and self.retry == "new_attempt": + raise ValueError("unknown external effects require the original identity") + if self.retry_after_seconds is not None and self.retry_after_seconds < 0: + raise ValueError("retry delay must be nonnegative") + + +class ReportingWriterError(Exception): + """A closed failure; never pass provider prose or attach a provider cause.""" + + def __init__(self, failure: ReportingWriterFailure) -> None: + if type(failure) is not ReportingWriterFailure: + raise TypeError("writer errors require a closed failure") + self.failure = failure + super().__init__(failure.code) + + def __repr__(self) -> str: + return f"ReportingWriterError({self.failure.code})" + + +def failure(code: ReportingWriterFailureCode) -> ReportingWriterError: + return ReportingWriterError(ReportingWriterFailure(code)) + + +@dataclass(frozen=True, slots=True) +class ReportingWriterCapability(_ClosedValue): + method: DeliveryMethod + transport: str + format: ReportingFormat | None + verification_profile: VerificationProfile + verification_path: VerificationPath + immutability: Literal["immutable_location", "native_version"] + checksum: Literal["sha256"] + write_semantics: Literal["conditional_create", "idempotent"] + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.transport, maximum=64) + if re.fullmatch(r"[a-z][a-z0-9_.-]{0,63}", self.transport) is None: + raise ValueError("transport requires a public protocol label") + if self.method == "file_transfer" and self.format is None: + raise ValueError("file transfer requires a format") + if self.verification_profile == "manifest_checksums" and self.method != "file_transfer": + raise ValueError("manifest verification requires file transfer") + if ( + (self.method == "dataset_share" and self.verification_path != "representative_consumer") + or ( + self.method == "warehouse_materialization" + and self.verification_path != "destination" + ) + or ( + self.verification_profile == "native_commit" + and self.immutability != "native_version" + ) + or (self.immutability == "native_version" and self.verification_path == "producer") + ): + raise ValueError("capability requires its exact immutable observation path") + + +@dataclass(frozen=True, slots=True) +class ReportingCanonicalization(_ClosedValue): + canonicalization_id: str + canonicalization_uri: str + canonicalization_sha256: str + + def __post_init__(self) -> None: + _freeze_fields(self) + ReportingCanonicalDigest( + "0" * 64, + self.canonicalization_id, + self.canonicalization_uri, + self.canonicalization_sha256, + ) + object.__setattr__(self, "canonicalization_sha256", self.canonicalization_sha256.lower()) + + +@dataclass(frozen=True, slots=True) +class ReportingVerificationKey(_ClosedValue): + """Complete frozen identity, including method/format/profile/path and schema.""" + + report_definition_id: str + reporting_profile: str + definition: ReportingDefinitionBinding + canonicalization: ReportingCanonicalization + capability: ReportingWriterCapability + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.report_definition_id) + reporting_identifier(self.reporting_profile, maximum=128) + for value in (self.definition.report_definition_sha256, self.definition.schema_sha256): + if type(value) is not str: + raise ValueError("definition digests require exact strings") + sha256_value(value) + for uri in (self.definition.report_definition_uri, self.definition.schema_uri): + # Reuse the strict public HTTPS contract screen; never dereference it. + ReportingCanonicalDigest("0" * 64, "definition", uri, "0" * 64) + for value in (self.definition.schema_version, self.definition.schema_ref_policy): + reporting_identifier(value, maximum=128) + ReportingCanonicalDigest("0" * 64, "dialect", self.definition.schema_dialect, "0" * 64) + for units in ( + self.definition.monetary_metric_units, + self.definition.monetary_control_total_units, + ): + for name, unit in units: + reporting_identifier(name, maximum=128) + reporting_identifier(unit, maximum=32) + object.__setattr__( + self, + "definition", + replace( + self.definition, + report_definition_sha256=self.definition.report_definition_sha256.lower(), + schema_sha256=self.definition.schema_sha256.lower(), + ), + ) + + +def binding_fingerprint(binding: ReportingDestinationBinding) -> str: + value = asdict(binding) + value["created_at"] = binding.created_at.isoformat() + return hashlib.sha256(canonical_json_utf8_v1(value)).hexdigest() + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationRequest(_ClosedValue): + """Trusted resolver input. Aliases must already resolve to the canonical consumer.""" + + principal: ReportingDeliveryPrincipal + generation: ReportingConfigurationGenerationKey + destination_ref: str + trusted_binding_ref: str = field(repr=False) + binding_fingerprint: str + verification_key: ReportingVerificationKey + reporting_obligation_id: str + reporting_revision_id: str + reporting_materialization_id: str + attempt: int + + def __post_init__(self) -> None: + _freeze_fields(self) + if self.principal.account_id != self.generation.account_id or self.attempt < 1: + raise ValueError("destination request requires an exact account and attempt") + reporting_identifier(self.generation.delivery_config_id) + if ( + type(self.generation.delivery_config_version) is not int + or self.generation.delivery_config_version < 1 + ): + raise ValueError("destination request requires an exact generation") + destination_reference(self.destination_ref) + destination_reference(self.trusted_binding_ref) + sha256_value(self.binding_fingerprint) + object.__setattr__(self, "binding_fingerprint", self.binding_fingerprint.lower()) + for value in ( + self.reporting_obligation_id, + self.reporting_revision_id, + self.reporting_materialization_id, + ): + reporting_identifier(value) + + @classmethod + def from_binding( + cls, + binding: ReportingDestinationBinding, + attempt: ReportingMaterializationAttempt, + key: ReportingVerificationKey, + ) -> ReportingDestinationRequest: + cap = key.capability + if ( + binding.principal != attempt.scope.principal + or binding.generation_key != attempt.scope.generation_key + or (binding.method, binding.transport, binding.format, binding.verification_profile) + != (cap.method, cap.transport, cap.format, cap.verification_profile) + or (binding.success_status == "delivered" and cap.verification_path != "destination") + ): + raise failure("BINDING_MISMATCH") + return cls( + binding.principal, + binding.generation_key, + binding.destination_ref, + binding.trusted_binding_ref, + binding_fingerprint(binding), + key, + attempt.scope.reporting_obligation_id, + attempt.reporting_revision_id, + attempt.reporting_materialization_id, + attempt.attempt, + ) + + @property + def external_id(self) -> str: + """Stable across pending retries; isolated across tenants and revision attempts.""" + return "rwm_" + hashlib.sha256(canonical_json_utf8_v1(asdict(self))).hexdigest() + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationLocator(_ClosedValue): + """Writer claims identify what to read. They are never verification proof.""" + + external_id: str + binding_fingerprint: str + resource: ReportingResourceRecord + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.external_id) + sha256_value(self.binding_fingerprint) + object.__setattr__(self, "binding_fingerprint", self.binding_fingerprint.lower()) + + +@dataclass(frozen=True, slots=True) +class ReportingPreparedRevision(_ClosedValue): + """Bounded immutable canonical row bytes, produced before resolver I/O. + + This is not a durable reservation. Keep the original attempt identity when + an external effect is unknown. Never automatically import legacy pending + attempts: drain old writers and recover/import their identities explicitly. + """ + + request: ReportingDestinationRequest + obligation: ReportingObligationRecord = field(repr=False) + revision: ReportingRevisionRecord = field(repr=False) + delivery: ReportingObligationDeliveryRecord = field(repr=False) + binding: ReportingDestinationBinding = field(repr=False) + rows: tuple[bytes, ...] = field(repr=False) + + def __post_init__(self) -> None: + _freeze_fields(self) + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationPage(_ClosedValue): + reporting_revision_id: str + rows: tuple[bytes, ...] = field(repr=False) + total_count: int + has_more: bool + cursor: str | None + format: ReportingFormat | None + verification_path: VerificationPath + native_version_ref: str | None = None + + def __post_init__(self) -> None: + _freeze_fields(self) + reporting_identifier(self.reporting_revision_id) + if self.total_count < 0 or self.has_more != (self.cursor is not None): + raise ValueError("destination page requires a paired cursor and valid total") + if self.cursor is not None: + reporting_identifier(self.cursor, maximum=2048) + if self.native_version_ref is not None: + native_version_reference(self.native_version_ref) + + +@dataclass(frozen=True, slots=True) +class ReportingNativeObservation(_ClosedValue): + location: str + native_version_ref: str + verification_path: Literal["representative_consumer", "destination"] + + def __post_init__(self) -> None: + _freeze_fields(self) + from adcp.reporting.evidence import resource_location + + resource_location(self.location) + native_version_reference(self.native_version_ref) + + +def object_path(value: str) -> str: + file_object_reference(value) + if "%" in value or ":" in value or any(part in {"", ".", ".."} for part in value.split("/")): + raise ValueError("object references require decoded relative path components") + return value + + +class ReportingHeartbeat(Protocol): + """B2 owns any parallel lease heartbeat. B1 only calls this checkpoint.""" + + async def checkpoint(self) -> None: ... + + +T = TypeVar("T") + + +@dataclass(frozen=True, slots=True) +class ReportingIOContext: + deadline_at: datetime + cancel: asyncio.Event = field(repr=False) + heartbeat: ReportingHeartbeat | None = field(default=None, repr=False) + close_timeout_seconds: float = 5.0 + + def __post_init__(self) -> None: + object.__setattr__(self, "deadline_at", aware_utc(self.deadline_at)) + if not math.isfinite(self.close_timeout_seconds) or self.close_timeout_seconds <= 0: + raise ValueError("close timeout must be finite and positive") + + async def run( + self, call: Callable[[], Awaitable[T]], *, effect: ReportingExternalEffect = "not_started" + ) -> T: + if self.heartbeat is not None: + await self._run(self.heartbeat.checkpoint, effect=effect) + return await self._run(call, effect=effect) + + async def _run(self, call: Callable[[], Awaitable[T]], *, effect: ReportingExternalEffect) -> T: + if self.cancel.is_set(): + raise asyncio.CancelledError + remaining = (self.deadline_at - datetime.now(timezone.utc)).total_seconds() + if remaining <= 0: + raise ReportingWriterError( + ReportingWriterFailure("DEADLINE_EXCEEDED", "same_identity", effect) + ) + + async def invoke() -> T: + return await call() + + task = asyncio.create_task(invoke()) + canceled = asyncio.create_task(self.cancel.wait()) + problem: ReportingWriterFailure | None = None + was_canceled = False + try: + done, _ = await asyncio.wait( + (task, canceled), timeout=remaining, return_when=asyncio.FIRST_COMPLETED + ) + if canceled in done: + was_canceled = True + elif task not in done: + problem = ReportingWriterFailure("DEADLINE_EXCEEDED", "same_identity", effect) + else: + try: + result = task.result() + except ReportingWriterError as exc: + problem = exc.failure + if ( + effect == "unknown" + and problem.effect == "not_started" + and problem.code + in {"RESOURCE_UNAVAILABLE", "DEADLINE_EXCEEDED", "LEASE_LOST"} + ): + problem = replace(problem, effect="unknown", retry="same_identity") + except Exception: + problem = ReportingWriterFailure( + "RESOURCE_UNAVAILABLE", "same_identity", effect + ) + except asyncio.CancelledError: + was_canceled = True + finally: + task.cancel() + canceled.cancel() + if await _join_tasks(task, canceled): + was_canceled = True + if was_canceled: + raise asyncio.CancelledError + # Outside the except suite: no provider __context__, even when inspected. + if problem is not None: + raise ReportingWriterError(problem) + return result + + +async def _join_tasks(*tasks: asyncio.Task[Any]) -> bool: + """Finish cancellation cleanup even if the caller is canceled repeatedly.""" + joined = asyncio.gather(*tasks, return_exceptions=True) + canceled = False + while not joined.done(): + try: + await asyncio.shield(joined) + except asyncio.CancelledError: + canceled = True + joined.result() + return canceled + + +class ReportingDestinationSession(ABC): + """Single-use SDK-owned lifecycle around an adopter's private provider session. + + Override only _open/_close and I/O methods. _close must tolerate partial + _open, finish promptly, and remove any adopter-owned temporary spool. The + SDK invokes it exactly once, shields cancellation, and joins all its tasks. + No credentials may be placed on the public request, descriptor or results. + """ + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + owned = { + "__repr__", + "__str__", + "__reduce__", + "__reduce_ex__", + "__getstate__", + "__aenter__", + "__aexit__", + "aclose", + "request", + "phase", + "context", + } + if owned.intersection(cls.__dict__): + raise TypeError("destination session lifecycle and redaction belong to the SDK") + + def __init__( + self, + request: ReportingDestinationRequest, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> None: + if ( + type(request) is not ReportingDestinationRequest + or phase not in ("write", "readback") + or type(context) is not ReportingIOContext + ): + raise failure("BINDING_MISMATCH") + self._request, self._phase, self._context = request, phase, context + self._entered = False + self._closed = False + + def __repr__(self) -> str: + return "" + + __str__ = __repr__ + + @property + def request(self) -> ReportingDestinationRequest: + return self._request + + @property + def phase(self) -> ReportingIOPhase: + return self._phase + + @property + def context(self) -> ReportingIOContext: + return self._context + + def __reduce__(self) -> tuple[Any, ...]: + raise TypeError("destination sessions cannot be persisted") + + @abstractmethod + async def _open(self) -> None: ... + + @abstractmethod + async def _close(self) -> None: ... + + async def __aenter__(self) -> ReportingDestinationSession: + if self._entered or self._closed: + raise failure("BINDING_MISMATCH") + self._entered = True + problem: ReportingWriterFailure | None = None + canceled = False + try: + await self.context.run(self._open) + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + problem = exc.failure + if canceled or problem is not None: + try: + await self.aclose() + except asyncio.CancelledError: + canceled = True + except ReportingWriterError: + pass + if canceled or self.context.cancel.is_set(): + raise asyncio.CancelledError + assert problem is not None + raise ReportingWriterError(problem) + return self + + async def __aexit__( + self, + kind: type[BaseException] | None, + value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + try: + await self.aclose() + except ReportingWriterError: + if kind is None: + raise + if self.context.cancel.is_set(): + raise asyncio.CancelledError + if kind is None and datetime.now(timezone.utc) >= self.context.deadline_at: + raise ReportingWriterError( + ReportingWriterFailure("DEADLINE_EXCEEDED", "same_identity", "unknown") + ) + + async def aclose(self) -> None: + if self._closed: + return + self._closed = True + await _close_owned(self._close, self.context.close_timeout_seconds) + + async def write(self, content: ReportingPreparedRevision) -> ReportingDestinationLocator: + raise failure("UNSUPPORTED_VERIFICATION") + + async def read_rows( + self, locator: ReportingDestinationLocator, *, cursor: str | None, limit: int + ) -> ReportingDestinationPage: + raise failure("UNSUPPORTED_VERIFICATION") + + async def read_manifest(self, locator: ReportingDestinationLocator) -> bytes: + raise failure("UNSUPPORTED_VERIFICATION") + + async def list_objects(self, locator: ReportingDestinationLocator) -> tuple[str, ...]: + raise failure("UNSUPPORTED_VERIFICATION") + + def read_object( + self, locator: ReportingDestinationLocator, *, object_ref: str + ) -> AsyncIterator[bytes]: + raise failure("UNSUPPORTED_VERIFICATION") + + async def observe_native_version( + self, locator: ReportingDestinationLocator + ) -> ReportingNativeObservation: + raise failure("UNSUPPORTED_VERIFICATION") + + +class ReportingDestinationResolver(Protocol): + """Return an unopened session. _open reauthorizes each phase independently. + + Resolution must match every request coordinate, including consumer URL, + immutable trusted binding, definition/canonicalization and capability. A + resolver must never allocate resources before constructing the session. + """ + + def resolve( + self, + request: ReportingDestinationRequest, + *, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> ReportingDestinationSession: ... + + +class ReportingDestinationWriter(Protocol): + @property + def capabilities(self) -> tuple[ReportingWriterCapability, ...]: ... + + @property + def production_eligible(self) -> bool: ... + + +async def _close_owned(close: Callable[[], Awaitable[None]], timeout: float) -> None: + """Cancellation-safe joining shared by the session and object stream lifecycles.""" + + async def invoke() -> None: + await close() + + task = asyncio.create_task(asyncio.wait_for(invoke(), timeout)) + canceled = False + while not task.done(): + try: + await asyncio.shield(task) + except asyncio.CancelledError: + canceled = True + except Exception: + break + if canceled or task.cancelled(): + if not task.cancelled(): + task.exception() # Retrieve any failure while propagating a clean cancellation. + raise asyncio.CancelledError + if task.exception() is not None: + raise ReportingWriterError( + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "same_identity", "unknown") + ) diff --git a/src/adcp/reporting/materializer/reference.py b/src/adcp/reporting/materializer/reference.py new file mode 100644 index 000000000..fcfc09ae8 --- /dev/null +++ b/src/adcp/reporting/materializer/reference.py @@ -0,0 +1,368 @@ +"""Deterministic memory destination for tests/development. NEVER production support.""" + +from __future__ import annotations + +import hashlib +from collections.abc import AsyncIterator +from dataclasses import dataclass, field +from datetime import timedelta +from importlib.resources import files +from typing import Any, Literal, cast, final + +from adcp.reporting.evidence import ReportingCanonicalDigest +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingResourceRecord, +) +from adcp.reporting.ledger.models import ReportingDefinitionBinding +from adcp.reporting.materializer._json import strict_reporting_json +from adcp.reporting.materializer.contracts import ( + ReportingCanonicalization, + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationSession, + ReportingIOContext, + ReportingIOPhase, + ReportingNativeObservation, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterCapability, + ReportingWriterError, + ReportingWriterFailure, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.verification import ( + ReportingRevisionVerifier, + ReportingRevisionVerifierRegistry, +) + + +def reference_verifier( + capability: ReportingWriterCapability | None = None, +) -> ReportingRevisionVerifier: + """One installed example definition/canonicalization; no network resolution.""" + root = files("adcp.reporting.materializer").joinpath("assets") + definition = root.joinpath("reference-definition.json").read_bytes() + schema = root.joinpath("reference-row-schema.json").read_bytes() + contract = root.joinpath("reference-canonicalization.json").read_bytes() + capability = capability or ReportingWriterCapability( + "file_transfer", + "reference-memory", + "jsonl", + "canonical_digest", + "producer", + "immutable_location", + "sha256", + "conditional_create", + ) + return ReportingRevisionVerifier( + ReportingVerificationKey( + "reference-report-v1", + "paid_media_delivery", + ReportingDefinitionBinding( + "https://contracts.example.test/reference-definition.json", + hashlib.sha256(definition).hexdigest(), + "1.0.0", + "https://contracts.example.test/reference-row-schema.json", + hashlib.sha256(schema).hexdigest(), + monetary_metric_units=(("spend", "USD"),), + monetary_control_total_units=(("spend", "USD"),), + ), + ReportingCanonicalization( + "reference-jcs-rows-v1", + "https://contracts.example.test/reference-canonicalization.json", + hashlib.sha256(contract).hexdigest(), + ), + capability, + ), + definition, + schema, + contract, + ) + + +def reference_digest( + verifier: ReportingRevisionVerifier, rows: list[object] +) -> ReportingCanonicalDigest: + encoded, _ = verifier.canonicalize(rows) + contract = verifier.key.canonicalization + return ReportingCanonicalDigest( + hashlib.sha256(b"[" + b",".join(encoded) + b"]").hexdigest(), + contract.canonicalization_id, + contract.canonicalization_uri, + contract.canonicalization_sha256, + ) + + +@dataclass(frozen=True) +class _Artifact: + rows: tuple[bytes, ...] + manifest: bytes | None + objects: tuple[tuple[str, bytes], ...] + locator: ReportingDestinationLocator + + +@final +class ReferenceReportingDestinationWriter: + """Conditional, deterministic external identities in shared process memory. + + Losing this object loses its artifacts. No durability or Managed capability + follows from this type or its descriptors. The production flag is a constant + property, with no constructor/config override. + """ + + def __init_subclass__(cls, **kwargs: Any) -> None: + raise TypeError("the non-production reference writer cannot be promoted by subclassing") + + def __init__(self, capabilities: tuple[ReportingWriterCapability, ...]) -> None: + if type(capabilities) is not tuple or any( + type(c) is not ReportingWriterCapability for c in capabilities + ): + raise ValueError("reference capabilities require an immutable typed tuple") + self._capabilities = capabilities + self._artifacts: dict[str, _Artifact] = {} + self.write_effects = 0 + self.open_count = 0 + self.close_count = 0 + + @property + def capabilities(self) -> tuple[ReportingWriterCapability, ...]: + return self._capabilities + + @property + def production_eligible(self) -> Literal[False]: + return False + + def __repr__(self) -> str: + return "" + + +@dataclass(frozen=True) +class ReferenceReportingResolver: + writer: ReferenceReportingDestinationWriter + registry: ReportingRevisionVerifierRegistry + bindings: tuple[ReportingDestinationBinding, ...] = field(repr=False) + _revoked: set[ReportingDeliveryPrincipal] = field( + default_factory=set, init=False, repr=False, compare=False + ) + _rotations: list[int] = field( + default_factory=lambda: [0], init=False, repr=False, compare=False + ) + + def __post_init__(self) -> None: + if type(self.bindings) is not tuple or any( + type(b) is not ReportingDestinationBinding for b in self.bindings + ): + raise ValueError("reference resolver requires frozen bindings") + + def revoke(self, principal: ReportingDeliveryPrincipal) -> None: + self._revoked.add(principal) + + def rotate(self) -> None: + self._rotations[0] += 1 + + def resolve( + self, + request: ReportingDestinationRequest, + *, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> ReportingDestinationSession: + return _Session(self, request, phase, context) + + +class _Session(ReportingDestinationSession): + def __init__( + self, + resolver: ReferenceReportingResolver, + request: ReportingDestinationRequest, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> None: + super().__init__(request, phase, context) + self._resolver = resolver + self._credential: object | None = None + + async def _open(self) -> None: + resolver = self._resolver + resolver.writer.open_count += 1 + resolver.registry.require(self.request.verification_key) + if self.request.verification_key.capability not in resolver.writer.capabilities: + raise failure("UNSUPPORTED_VERIFICATION") + if self.request.principal in resolver._revoked or not any( + b.principal == self.request.principal + and b.generation_key == self.request.generation + and b.destination_ref == self.request.destination_ref + and b.trusted_binding_ref == self.request.trusted_binding_ref + and binding_fingerprint(b) == self.request.binding_fingerprint + for b in resolver.bindings + ): + raise failure("AUTHORIZATION_DENIED") + # A test credential exists only in the owned session and is discarded on + # every close. Real resolvers retrieve fresh credentials at this point. + self._credential = (object(), resolver._rotations[0]) + + async def _close(self) -> None: + self._credential = None + self._resolver.writer.close_count += 1 + + def _check(self, phase: ReportingIOPhase) -> None: + if self._credential is None or self._closed or self.phase != phase: + raise failure("AUTHORIZATION_DENIED") + + async def write(self, content: ReportingPreparedRevision) -> ReportingDestinationLocator: + self._check("write") + request = self.request + if content.request != request: + raise failure("BINDING_MISMATCH") + writer = self._resolver.writer + existing = writer._artifacts.get(request.external_id) + if existing is not None: + if existing.rows != content.rows: + raise ReportingWriterError( + ReportingWriterFailure("WRITE_FAILED", "never", "applied") + ) + return existing.locator + cap = request.verification_key.capability + prefix = request.external_id + objects: tuple[tuple[str, bytes], ...] = () + manifest: bytes | None = None + if cap.method == "file_transfer": + objects = tuple( + ( + f"{prefix}/part-{index // 250:06d}.jsonl", + b"".join(r + b"\n" for r in content.rows[index : index + 250]), + ) + for index in range(0, max(1, len(content.rows)), 250) + ) + period = content.obligation.period + manifest = strict_reporting_json( + { + "manifest_version": "1.0", + "complete": True, + "reporting_revision_id": request.reporting_revision_id, + "reporting_obligation_id": request.reporting_obligation_id, + "reporting_materialization_id": request.reporting_materialization_id, + "period": { + "start": period.start.isoformat(), + "end": period.end.isoformat(), + "source_timezone": period.source_timezone, + }, + "format": cap.format, + "compression": "none", + "files": [ + { + "object_ref": ref, + "size_bytes": len(raw), + "row_count": raw.count(b"\n"), + "sha256": hashlib.sha256(raw).hexdigest(), + } + for ref, raw in objects + ], + "total_size_bytes": sum(len(raw) for _, raw in objects), + "row_count": len(content.rows), + "control_totals": [ + t.to_wire() for t in content.revision.managed_control_totals or () + ], + "created_at": content.revision.created_at.isoformat(), + } + ) + resource = ReportingResourceRecord( + resource_ref=prefix, + kind=cast( + Literal["manifest", "dataset", "warehouse_relation"], + { + "file_transfer": "manifest", + "dataset_share": "dataset", + "warehouse_materialization": "warehouse_relation", + }[cap.method], + ), + location=f"{prefix}/manifest.json" if manifest is not None else f"{prefix}/table", + immutability=cap.immutability, + expires_at=max( + content.delivery.resource_retained_until, + self.context.deadline_at + timedelta(days=content.binding.resource_retention_days), + ), + native_version_ref=( + f"reference-{prefix}" if cap.immutability == "native_version" else None + ), + manifest_sha256=hashlib.sha256(manifest).hexdigest() if manifest is not None else None, + object_refs=tuple(ref for ref, _ in objects), + reader_compatibility=content.binding.reader_compatibility, + ) + locator = ReportingDestinationLocator( + request.external_id, request.binding_fingerprint, resource + ) + writer._artifacts[request.external_id] = _Artifact(content.rows, manifest, objects, locator) + writer.write_effects += 1 + return locator + + def _artifact(self, locator: ReportingDestinationLocator) -> _Artifact: + self._check("readback") + if ( + locator.external_id != self.request.external_id + or locator.binding_fingerprint != self.request.binding_fingerprint + ): + raise failure("BINDING_MISMATCH") + artifact = self._resolver.writer._artifacts.get(locator.external_id) + if artifact is None: + raise failure("RESOURCE_UNAVAILABLE") + if artifact.locator.resource.location != locator.resource.location: + raise failure("BINDING_MISMATCH") + return artifact + + async def read_rows( + self, locator: ReportingDestinationLocator, *, cursor: str | None, limit: int + ) -> ReportingDestinationPage: + artifact = self._artifact(locator) + offset = 0 + prefix = f"{self.request.external_id}:" + if cursor is not None: + if not cursor.startswith(prefix) or not cursor[len(prefix) :].isdigit(): + raise failure("BINDING_MISMATCH") + offset = int(cursor[len(prefix) :]) + window = artifact.rows[offset : offset + limit] + following = offset + len(window) + more = following < len(artifact.rows) + cap = self.request.verification_key.capability + return ReportingDestinationPage( + self.request.reporting_revision_id, + window, + len(artifact.rows), + more, + f"{prefix}{following}" if more else None, + cap.format, + cap.verification_path, + artifact.locator.resource.native_version_ref, + ) + + async def read_manifest(self, locator: ReportingDestinationLocator) -> bytes: + result = self._artifact(locator).manifest + if result is None: + raise failure("RESOURCE_UNAVAILABLE") + return result + + async def list_objects(self, locator: ReportingDestinationLocator) -> tuple[str, ...]: + return tuple(ref for ref, _ in self._artifact(locator).objects) + + async def read_object( + self, locator: ReportingDestinationLocator, *, object_ref: str + ) -> AsyncIterator[bytes]: + for ref, raw in self._artifact(locator).objects: + if ref == object_ref: + for start in range(0, len(raw), 4096): + yield raw[start : start + 4096] + return + raise failure("RESOURCE_UNAVAILABLE") + + async def observe_native_version( + self, locator: ReportingDestinationLocator + ) -> ReportingNativeObservation: + resource = self._artifact(locator).locator.resource + path = self.request.verification_key.capability.verification_path + if resource.native_version_ref is None or path == "producer": + raise failure("UNSUPPORTED_VERIFICATION") + return ReportingNativeObservation(resource.location, resource.native_version_ref, path) diff --git a/src/adcp/reporting/materializer/verification.py b/src/adcp/reporting/materializer/verification.py new file mode 100644 index 000000000..8cf7c9f9b --- /dev/null +++ b/src/adcp/reporting/materializer/verification.py @@ -0,0 +1,1107 @@ +"""SDK-owned verification of frozen source content and actual destination reads. + +The registry contains immutable, explicitly installed bytes. It never fetches a +URI or trusts a writer's digest as proof. B1 returns evidence only; B2 must +reselect and fence under its final account transaction before publishing it. +""" + +from __future__ import annotations + +import asyncio +import base64 +import hashlib +import re +from collections.abc import AsyncIterator, Sequence +from contextlib import asynccontextmanager +from dataclasses import dataclass, field, replace +from datetime import datetime, timedelta, timezone +from decimal import Decimal, localcontext +from typing import Any, Protocol, cast + +from jsonschema import Draft202012Validator + +from adcp.reporting.evidence import ReportingCanonicalDigest, ReportingControlTotalRecord +from adcp.reporting.ledger.delivery_models import ( + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingPhysicalChecksum, + ReportingResourceRecord, + ReportingVerificationRecord, + _ClosedValue, + _freeze_fields, +) +from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.ledger.producer import revision_content_sha256 +from adcp.reporting.ledger.store import LedgerConflictError, ReportingRowPage, revision_row_offset +from adcp.reporting.materializer._json import ( + ReportingVerificationLimits, + parse_reporting_json, + strict_reporting_json, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationResolver, + ReportingDestinationSession, + ReportingIOContext, + ReportingIOPhase, + ReportingNativeObservation, + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterError, + ReportingWriterFailure, + _close_owned, + binding_fingerprint, + failure, + object_path, +) +from adcp.reporting.revision_selection import select_reporting_revision + + +class ReportingRevisionRowReader(Protocol): + async def read_revision_rows( + self, + *, + account_id: str, + reporting_revision_id: str, + cursor: str | None = None, + limit: int = 500, + ) -> ReportingRowPage: ... + + +@dataclass(frozen=True, slots=True) +class ReportingVerifiedDestination(_ClosedValue): + """Immutable SDK observations. B2 alone owns atomic publication/readiness.""" + + request: ReportingDestinationRequest + resource: ReportingResourceRecord + verification: ReportingVerificationRecord + + def __post_init__(self) -> None: + _freeze_fields(self) + + +def validate_materialization_target( + prepared: ReportingPreparedRevision, + *, + binding: ReportingDestinationBinding, + revisions: Sequence[ReportingRevisionRecord], +) -> None: + """Pure B2 finish seam. Caller must supply its locked, complete current history.""" + selection = select_reporting_revision( + revisions, + account_id=prepared.request.principal.account_id, + reporting_obligation_id=prepared.request.reporting_obligation_id, + required_finality=prepared.obligation.required_finality, + ) + if selection.kind == "corrupt": + raise failure("HISTORY_CORRUPT") + if ( + selection.kind != "selected" + or not selection.revision.readable + or selection.revision != prepared.revision + or binding_fingerprint(binding) != prepared.request.binding_fingerprint + ): + raise failure("CURRENT_REVISION_CHANGED") + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionVerifier(_ClosedValue): + """One exact installed contract using the SDK's safe-integer JCS subset. + + Sum totals are declared on schema properties with ``x-adcp-control-total`` + (value_type, optional unit, decimal scale). This explicit pinned subset has + no expression evaluator, custom callbacks, mutable registration or network. + Unsupported formats and definition semantics fail during construction. + """ + + key: ReportingVerificationKey + definition_bytes: bytes = field(repr=False) + schema_bytes: bytes = field(repr=False) + canonicalization_bytes: bytes = field(repr=False) + limits: ReportingVerificationLimits = field(default_factory=ReportingVerificationLimits) + + def __post_init__(self) -> None: + _freeze_fields(self) + error = False + try: + self._runtime() + except Exception: + error = True + if error: + raise failure("UNSUPPORTED_VERIFICATION") + + def _runtime(self) -> _Canonicalizer: + key, definition = self.key, self.key.definition + if key.capability.format not in {"jsonl", None}: + raise failure("UNSUPPORTED_VERIFICATION") + if ( + key.capability.method == "file_transfer" + and key.capability.immutability != "immutable_location" + ): + raise failure("UNSUPPORTED_VERIFICATION") + if ( + definition.schema_dialect != "https://json-schema.org/draft/2020-12/schema" + or definition.schema_ref_policy != "local_fragment_only" + ): + raise failure("UNSUPPORTED_VERIFICATION") + for raw, expected in ( + (self.definition_bytes, definition.report_definition_sha256), + (self.schema_bytes, definition.schema_sha256), + (self.canonicalization_bytes, key.canonicalization.canonicalization_sha256), + ): + if hashlib.sha256(raw).hexdigest() != expected.lower(): + raise failure("UNSUPPORTED_VERIFICATION") + report = _mapping(parse_reporting_json(self.definition_bytes, self.limits)) + schema = _mapping(parse_reporting_json(self.schema_bytes, self.limits)) + contract = _mapping(parse_reporting_json(self.canonicalization_bytes, self.limits)) + if ( + report.get("report_definition_id") != key.report_definition_id + or report.get("reporting_profile") != key.reporting_profile + or schema.get("$schema") != definition.schema_dialect + or set(contract) + != { + "contract_version", + "media_type", + "algorithm", + "schema_sha256", + "primary_keys", + "golden_vectors", + } + or contract["contract_version"] != "1.0" + or contract["media_type"] != "application/vnd.adcp.reporting-canonicalization+json" + or contract["algorithm"] != "adcp_jcs_rows_v1" + or contract["schema_sha256"].lower() != definition.schema_sha256.lower() + ): + raise failure("UNSUPPORTED_VERIFICATION") + _local_schema(schema) + Draft202012Validator.check_schema(schema) + keys = contract["primary_keys"] + if ( + type(keys) is not list + or not keys + or any(type(k) is not str for k in keys) + or len(set(keys)) != len(keys) + ): + raise failure("UNSUPPORTED_VERIFICATION") + totals: list[_Total] = [] + properties = _mapping(schema.get("properties")) + metrics = report.get("metrics") + if type(metrics) is not list or not metrics: + raise failure("UNSUPPORTED_VERIFICATION") + for metric in metrics: + metric = _mapping(metric) + name = metric["name"] + prop = _mapping(properties[name]) + total = _mapping(prop["x-adcp-control-total"]) + if ( + metric.get("source_expression") != name + or metric.get("aggregation") != "sum" + or set(total) - {"value_type", "unit", "scale"} + or total.get("unit") != metric.get("unit") + or (total["value_type"], prop.get("type")) + not in {("integer", "integer"), ("decimal", "string")} + ): + raise failure("UNSUPPORTED_VERIFICATION") + scale = total.get("scale", 0) + if ( + type(scale) is not int + or not 0 <= scale <= 18 + or (total["value_type"] == "integer" and scale != 0) + ): + raise failure("UNSUPPORTED_VERIFICATION") + totals.append(_Total(name, total["value_type"], total.get("unit"), scale)) + if len({t.name for t in totals}) != len(totals): + raise failure("UNSUPPORTED_VERIFICATION") + if {t.name for t in totals} != { + name + for name, prop in properties.items() + if type(prop) is dict and "x-adcp-control-total" in prop + }: + raise failure("UNSUPPORTED_VERIFICATION") + if not set(keys).union(t.name for t in totals) <= set(schema.get("required", [])): + raise failure("UNSUPPORTED_VERIFICATION") + units = {t.name: t.unit for t in totals} + if any( + units.get(name) != unit + for name, unit in ( + *definition.monetary_metric_units, + *definition.monetary_control_total_units, + ) + ): + raise failure("UNSUPPORTED_VERIFICATION") + runtime = _Canonicalizer( + tuple(keys), tuple(totals), Draft202012Validator(schema), self.limits + ) + vectors = _mapping(contract["golden_vectors"]) + if not {"empty_report", "ordering_encoding"} <= set(vectors) or set(vectors) - { + "empty_report", + "ordering_encoding", + "additional", + }: + raise failure("UNSUPPORTED_VERIFICATION") + seen: set[str] = set() + for vector in [ + vectors["empty_report"], + vectors["ordering_encoding"], + *vectors.get("additional", []), + ]: + vector = _mapping(vector) + if ( + set(vector) != {"name", "purpose", "input_rows", "canonical_utf8_base64", "sha256"} + or vector["name"] in seen + ): + raise failure("UNSUPPORTED_VERIFICATION") + seen.add(vector["name"]) + rows, _ = runtime.canonicalize(vector["input_rows"]) + golden_bytes = base64.b64decode(vector["canonical_utf8_base64"], validate=True) + if ( + b"[" + b",".join(rows) + b"]" != golden_bytes + or hashlib.sha256(golden_bytes).hexdigest() != vector["sha256"].lower() + ): + raise failure("UNSUPPORTED_VERIFICATION") + if ( + vectors["empty_report"]["input_rows"] != [] + or vectors["empty_report"]["purpose"] != "empty_report" + ): + raise failure("UNSUPPORTED_VERIFICATION") + ordering = vectors["ordering_encoding"] + order_rows = ordering["input_rows"] + if ( + ordering["purpose"] != "ordering_encoding" + or len(order_rows) < 2 + or [runtime.order_key(r) for r in order_rows] + == sorted(runtime.order_key(r) for r in order_rows) + or not _unordered_members(order_rows) + ): + raise failure("UNSUPPORTED_VERIFICATION") + return runtime + + def canonicalize( + self, rows: Sequence[object] + ) -> tuple[tuple[bytes, ...], tuple[ReportingControlTotalRecord, ...]]: + """Useful to trusted publishers preparing expected evidence before storage.""" + return self._runtime().canonicalize(rows) + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionVerifierRegistry(_ClosedValue): + verifiers: tuple[ReportingRevisionVerifier, ...] + + def __post_init__(self) -> None: + _freeze_fields(self) + if len({v.key for v in self.verifiers}) != len(self.verifiers): + raise failure("UNSUPPORTED_VERIFICATION") + + def require(self, key: ReportingVerificationKey) -> ReportingRevisionVerifier: + for verifier in self.verifiers: + if verifier.key == key: + return verifier + raise failure("UNSUPPORTED_VERIFICATION") + + async def prepare( + self, + *, + key: ReportingVerificationKey, + binding: ReportingDestinationBinding, + delivery: ReportingObligationDeliveryRecord, + obligation: ReportingObligationRecord, + revisions: Sequence[ReportingRevisionRecord], + attempt: ReportingMaterializationAttempt, + reader: ReportingRevisionRowReader, + context: ReportingIOContext, + ) -> ReportingPreparedRevision: + verifier = self.require(key) # Unsupported tuples fail before any source/resolver I/O. + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + raise failure("HISTORY_CORRUPT") + if selection.kind != "selected": + raise failure("REVISION_NOT_READY") + revision = selection.revision + _revision_metadata(revision) + if not revision.readable: + raise failure("REVISION_NOT_READY") + request = ReportingDestinationRequest.from_binding(binding, attempt, key) + if ( + obligation.definition is None + or replace(key, definition=obligation.definition) != key + or (obligation.report_definition_id, obligation.reporting_profile) + != (key.report_definition_id, key.reporting_profile) + or request.generation != obligation.generation_key + or request.reporting_obligation_id != obligation.reporting_obligation_id + or request.reporting_revision_id != revision.reporting_revision_id + or delivery.scope != attempt.scope + or delivery.currency != obligation.currency + ): + raise failure("BINDING_MISMATCH") + _expected_digest(revision, key) + rows: list[dict[str, Any]] = [] + cursor: str | None = None + seen: set[str] = set() + budget = _Budget(verifier.limits) + while True: + budget.page() + page = await context.run( + lambda: reader.read_revision_rows( + account_id=obligation.account_id, + reporting_revision_id=revision.reporting_revision_id, + cursor=cursor, + limit=500, + ) + ) + if ( + type(page) is not ReportingRowPage + or type(page.rows) is not tuple + or page.reporting_revision_id != revision.reporting_revision_id + ): + raise failure("SOURCE_INVALID") + _page( + page.total_count, + page.has_more, + page.cursor, + len(page.rows), + len(rows), + revision.row_count, + seen, + ) + if page.cursor is not None: + cursor_invalid = False + try: + cursor_invalid = revision_row_offset( + page.cursor, revision.reporting_revision_id, 500 + ) != len(rows) + len(page.rows) + except (ValueError, LedgerConflictError): + cursor_invalid = True + if cursor_invalid: + raise failure("SOURCE_INVALID") + for row in page.rows: + encoded = strict_reporting_json(row, verifier.limits) + budget.add(encoded) + rows.append(_mapping(parse_reporting_json(encoded, verifier.limits))) + if not page.has_more: + break + cursor = page.cursor + if ( + revision_content_sha256( + reporting_revision_id=revision.reporting_revision_id, + row_count=revision.row_count, + control_totals=revision.control_totals, + reporting_rows=rows, + control_total_evidence=revision.managed_control_totals, + ) + != revision.revision_content_sha256.lower() + ): + raise failure("SOURCE_INVALID") + canonical_rows, totals = verifier.canonicalize(rows) + _verify_content(revision, key, canonical_rows, totals) + await context.run(_checkpoint) + return ReportingPreparedRevision( + request, obligation, revision, delivery, binding, canonical_rows + ) + + +@dataclass(frozen=True, slots=True) +class ReportingDestinationIO: + """Explicit one-shot write/readback operations; no coordinator or lease loop.""" + + registry: ReportingRevisionVerifierRegistry + resolver: ReportingDestinationResolver = field(repr=False) + + @asynccontextmanager + async def _session( + self, + prepared: ReportingPreparedRevision, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> AsyncIterator[ReportingDestinationSession]: + self.registry.require(prepared.request.verification_key) + problem = False + canceled = False + try: + session = self.resolver.resolve(prepared.request, phase=phase, context=context) + except asyncio.CancelledError: + canceled = True + except Exception: + problem = True + if canceled: + raise asyncio.CancelledError + if problem: + raise ReportingWriterError( + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "same_identity") + ) + if not isinstance(session, ReportingDestinationSession): + raise failure("BINDING_MISMATCH") + try: + _session_binding(session, prepared.request, phase, context) + except ReportingWriterError: + problem = True + if problem: + await session.aclose() + raise failure("BINDING_MISMATCH") + failure_record: ReportingWriterFailure | None = None + canceled = False + try: + async with session: + _session_binding(session, prepared.request, phase, context) + yield session + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + failure_record = exc.failure + except Exception: + failure_record = ReportingWriterFailure( + "RESOURCE_UNAVAILABLE", "same_identity", "unknown" + ) + if canceled: + raise asyncio.CancelledError + if failure_record is not None: + raise ReportingWriterError(failure_record) + + async def write( + self, prepared: ReportingPreparedRevision, *, context: ReportingIOContext + ) -> ReportingDestinationLocator: + self._validate_prepared(prepared) + problem: ReportingWriterFailure | None = None + canceled = False + try: + async with self._session(prepared, "write", context) as session: + _session_binding(session, prepared.request, "write", context) + locator = await context.run(lambda: session.write(prepared), effect="unknown") + invalid_locator = False + try: + _locator_binding(locator, prepared.request) + except ReportingWriterError: + invalid_locator = True + if invalid_locator: + raise ReportingWriterError( + ReportingWriterFailure("BINDING_MISMATCH", "same_identity", "unknown") + ) + return locator + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + problem = exc.failure + if canceled: + raise asyncio.CancelledError + assert problem is not None + raise ReportingWriterError(problem) + + def _validate_prepared(self, prepared: ReportingPreparedRevision) -> ReportingRevisionVerifier: + verifier = self.registry.require(prepared.request.verification_key) + request, obligation, binding = prepared.request, prepared.obligation, prepared.binding + cap = request.verification_key.capability + if ( + binding_fingerprint(binding) != request.binding_fingerprint + or request.principal != binding.principal + or request.generation != binding.generation_key + or request.destination_ref != binding.destination_ref + or request.trusted_binding_ref != binding.trusted_binding_ref + or request.generation != obligation.generation_key + or request.reporting_obligation_id != obligation.reporting_obligation_id + or request.reporting_revision_id != prepared.revision.reporting_revision_id + or prepared.revision.account_id != obligation.account_id + or prepared.revision.reporting_obligation_id != obligation.reporting_obligation_id + or not prepared.revision.readable + or prepared.delivery.scope.principal != request.principal + or prepared.delivery.scope.generation_key != request.generation + or prepared.delivery.scope.reporting_obligation_id != request.reporting_obligation_id + or prepared.delivery.currency != obligation.currency + or obligation.definition is None + or replace(verifier.key, definition=obligation.definition) != verifier.key + or (obligation.report_definition_id, obligation.reporting_profile) + != (verifier.key.report_definition_id, verifier.key.reporting_profile) + or (binding.method, binding.transport, binding.format, binding.verification_profile) + != (cap.method, cap.transport, cap.format, cap.verification_profile) + or (binding.success_status == "delivered" and cap.verification_path != "destination") + ): + raise failure("BINDING_MISMATCH") + rows, totals = verifier.canonicalize( + [parse_reporting_json(r, verifier.limits) for r in prepared.rows] + ) + if rows != prepared.rows: + raise failure("SOURCE_INVALID") + _verify_content(prepared.revision, verifier.key, rows, totals) + return verifier + + async def verify( + self, + prepared: ReportingPreparedRevision, + locator: ReportingDestinationLocator, + *, + context: ReportingIOContext, + ) -> ReportingVerifiedDestination: + verifier = self._validate_prepared(prepared) + _locator_binding(locator, prepared.request) + problem: ReportingWriterFailure | None = None + canceled = False + try: + async with self._session(prepared, "readback", context) as session: + _session_binding(session, prepared.request, "readback", context) + return await _verify_destination(verifier, prepared, locator, session, context) + except asyncio.CancelledError: + canceled = True + except ReportingWriterError as exc: + problem = exc.failure + if canceled: + raise asyncio.CancelledError + assert problem is not None + if problem.code == "SOURCE_INVALID": + problem = ReportingWriterFailure("DESTINATION_CORRUPT", "new_attempt", "applied") + raise ReportingWriterError(problem) + + +def _session_binding( + session: ReportingDestinationSession, + request: ReportingDestinationRequest, + phase: ReportingIOPhase, + context: ReportingIOContext, +) -> None: + if session.request != request or session.phase != phase or session.context is not context: + raise failure("BINDING_MISMATCH") + + +def _locator_binding( + locator: ReportingDestinationLocator, request: ReportingDestinationRequest +) -> None: + if ( + type(locator) is not ReportingDestinationLocator + or locator.external_id != request.external_id + or locator.binding_fingerprint != request.binding_fingerprint + ): + raise failure("BINDING_MISMATCH") + # Logical row readers may use these paths before reading the manifest. + # Reject traversal before authorization or any readback method can see it. + invalid = False + try: + for ref in locator.resource.object_refs: + object_path(ref) + except ValueError: + invalid = True + if invalid: + raise failure("BINDING_MISMATCH") + + +def _revision_metadata(revision: ReportingRevisionRecord) -> None: + # Foundation records are source compatible dataclasses, not the strict + # destination boundary. Never let Python's bool/int equality bless counts + # or readability flags from a custom/legacy reader. + if ( + type(revision) is not ReportingRevisionRecord + or type(revision.row_count) is not int + or revision.row_count < 0 + or type(revision.readable) is not bool + or type(revision.readable_at_commit) is not bool + or type(revision.revision_content_sha256) is not str + or re.fullmatch(r"[0-9a-fA-F]{64}", revision.revision_content_sha256) is None + ): + raise failure("SOURCE_INVALID") + + +def _expected_digest( + revision: ReportingRevisionRecord, key: ReportingVerificationKey +) -> ReportingCanonicalDigest: + _revision_metadata(revision) + digest, canonical = revision.canonical_content_digest, key.canonicalization + if digest is None or ( + digest.canonicalization_id, + digest.canonicalization_uri, + digest.canonicalization_sha256.lower(), + ) != ( + canonical.canonicalization_id, + canonical.canonicalization_uri, + canonical.canonicalization_sha256, + ): + raise failure("UNSUPPORTED_VERIFICATION") + return digest + + +def _verify_content( + revision: ReportingRevisionRecord, + key: ReportingVerificationKey, + rows: tuple[bytes, ...], + totals: tuple[ReportingControlTotalRecord, ...], +) -> None: + expected = _expected_digest(revision, key) + if ( + len(rows) != revision.row_count + or _digest_rows(rows) != expected.value.lower() + or totals != revision.managed_control_totals + ): + raise failure("SOURCE_INVALID") + + +def _digest_rows(rows: Sequence[bytes]) -> str: + digest = hashlib.sha256(b"[") + for index, row in enumerate(rows): + if index: + digest.update(b",") + digest.update(row) + digest.update(b"]") + return digest.hexdigest() + + +def _mapping(value: object) -> dict[str, Any]: + if type(value) is not dict: + raise failure("SOURCE_INVALID") + return cast(dict[str, Any], value) + + +def _local_schema(schema: dict[str, Any]) -> None: + stack: list[object] = [schema] + while stack: + value = stack.pop() + if isinstance(value, dict): + if any(k in value for k in ("$id", "$dynamicRef", "$recursiveRef", "$vocabulary")) or ( + "$ref" in value + and (type(value["$ref"]) is not str or not value["$ref"].startswith("#")) + ): + raise failure("UNSUPPORTED_VERIFICATION") + stack.extend(value.values()) + elif isinstance(value, list): + stack.extend(value) + + +@dataclass(frozen=True) +class _Total: + name: str + value_type: Any + unit: str | None + scale: int + + +@dataclass +class _Canonicalizer: + keys: tuple[str, ...] + totals: tuple[_Total, ...] + validator: Any + limits: ReportingVerificationLimits + + def order_key(self, row: object) -> bytes: + row = _mapping(row) + values = [row[k] for k in self.keys if k in row] + if len(values) != len(self.keys) or any( + type(v) not in {str, int, bool, type(None)} for v in values + ): + raise failure("SOURCE_INVALID") + return strict_reporting_json(values, self.limits) + + def canonicalize( + self, rows: Sequence[object] + ) -> tuple[tuple[bytes, ...], tuple[ReportingControlTotalRecord, ...]]: + budget = _Budget(self.limits) + ordered: dict[bytes, bytes] = {} + sums = [Decimal(0) for _ in self.totals] + for row in rows: + encoded = strict_reporting_json(row, self.limits) + budget.add(encoded) + value = _mapping(row) + valid = False + try: + valid = self.validator.is_valid(value) + except Exception: + valid = False # No schema resolver/validation diagnostics leave this boundary. + if not valid: + raise failure("SOURCE_INVALID") + key = self.order_key(value) + if key in ordered: + raise failure("SOURCE_INVALID") + ordered[key] = encoded + for index, total in enumerate(self.totals): + item = value.get(total.name) + if total.value_type == "integer": + if type(item) is not int: + raise failure("SOURCE_INVALID") + elif type(item) is not str or re_decimal(item, total.scale) is False: + raise failure("SOURCE_INVALID") + with localcontext() as ctx: + ctx.prec = 128 + sums[index] += Decimal(item) + totals = tuple( + ReportingControlTotalRecord( + rule.name, + format(value, f".{rule.scale}f"), + rule.value_type, + rule.unit, + ) + for rule, value in zip(self.totals, sums) + ) + return tuple(ordered[k] for k in sorted(ordered)), totals + + +def re_decimal(value: str, scale: int) -> bool: + return bool( + re.fullmatch(r"-?(?:0|[1-9][0-9]{0,37})" + (rf"\.[0-9]{{{scale}}}" if scale else ""), value) + ) + + +@dataclass +class _Budget: + limits: ReportingVerificationLimits + total_bytes: int = 0 + items: int = 0 + rows: int = 0 + pages: int = 0 + + def page(self) -> None: + self.pages += 1 + if self.pages > self.limits.max_pages: + raise failure("LIMIT_EXCEEDED") + + def add(self, encoded: bytes) -> None: + self.total_bytes += len(encoded) + self.rows += 1 + stack: list[object] = [parse_reporting_json(encoded, self.limits)] + while stack: + item = stack.pop() + self.items += 1 + if isinstance(item, dict): + self.items += len(item) + stack.extend(item.values()) + elif isinstance(item, list): + stack.extend(item) + if ( + self.total_bytes > self.limits.max_total_bytes + or self.items > self.limits.max_items + or self.rows > self.limits.max_rows + ): + raise failure("LIMIT_EXCEEDED") + + +def _page( + total: int, + has_more: bool, + cursor: str | None, + count: int, + before: int, + expected: int, + seen: set[str], +) -> None: + if ( + type(total) is not int + or total != expected + or type(has_more) is not bool + or has_more != (cursor is not None) + or count > 500 + or before + count > expected + or (has_more and (count == 0 or before + count >= expected)) + or (not has_more and before + count != expected) + ): + raise failure("SOURCE_INVALID") + if cursor is not None: + if type(cursor) is not str or not cursor or len(cursor) > 2048 or cursor in seen: + raise failure("SOURCE_INVALID") + seen.add(cursor) + + +async def _verify_destination( + verifier: ReportingRevisionVerifier, + prepared: ReportingPreparedRevision, + locator: ReportingDestinationLocator, + session: ReportingDestinationSession, + context: ReportingIOContext, +) -> ReportingVerifiedDestination: + cap, resource = verifier.key.capability, locator.resource + expected_kind = { + "file_transfer": "manifest", + "dataset_share": "dataset", + "warehouse_materialization": "warehouse_relation", + }[cap.method] + if ( + resource.kind != expected_kind + or resource.immutability != cap.immutability + or resource.expires_at + < max( + prepared.delivery.resource_retained_until, + datetime.now(timezone.utc) + timedelta(days=prepared.binding.resource_retention_days), + ) + or resource.reader_compatibility != prepared.binding.reader_compatibility + ): + raise failure("SOURCE_INVALID") + native: ReportingNativeObservation | None = None + if cap.immutability == "native_version": + native = await context.run( + lambda: session.observe_native_version(locator), effect="unknown" + ) + _native(native, resource, cap.verification_path) + cursor: str | None = None + seen: set[str] = set() + budget = _Budget(verifier.limits) + destination: list[object] = [] + while True: + budget.page() + page = await context.run( + lambda: session.read_rows(locator, cursor=cursor, limit=500), effect="unknown" + ) + if ( + type(page) is not ReportingDestinationPage + or page.reporting_revision_id != prepared.request.reporting_revision_id + or page.format != cap.format + or page.verification_path != cap.verification_path + or page.native_version_ref != (native.native_version_ref if native else None) + ): + raise failure("SOURCE_INVALID") + _page( + page.total_count, + page.has_more, + page.cursor, + len(page.rows), + len(destination), + len(prepared.rows), + seen, + ) + for raw in page.rows: + value = parse_reporting_json(raw, verifier.limits) + row_bytes = strict_reporting_json(value, verifier.limits) + budget.add(raw) + if row_bytes != prepared.rows[len(destination)]: + raise failure("SOURCE_INVALID") + destination.append(value) + if not page.has_more: + break + cursor = page.cursor + encoded, totals = verifier.canonicalize(destination) + _verify_content(prepared.revision, verifier.key, encoded, totals) + checksums: tuple[ReportingPhysicalChecksum, ...] = () + manifest_digest: str | None = None + if cap.method == "file_transfer": + checksums, manifest_digest = await _verify_files( + verifier, prepared, locator, session, context + ) + if native is not None: + observed = await context.run( + lambda: session.observe_native_version(locator), effect="unknown" + ) + _native(observed, resource, cap.verification_path) + if observed != native: + raise failure("SOURCE_INVALID") + at = datetime.now(timezone.utc) + canonical = verifier.key.canonicalization + verification = ReportingVerificationRecord( + verified_at=at, + verification_path=cap.verification_path, + verification_profile=cap.verification_profile, + row_count=len(encoded), + control_totals=totals, + canonical_content_digest=( + ReportingCanonicalDigest( + _digest_rows(encoded), + canonical.canonicalization_id, + canonical.canonicalization_uri, + canonical.canonicalization_sha256, + ) + if cap.verification_profile == "canonical_digest" + else None + ), + physical_checksums=checksums, + native_version_ref=native.native_version_ref if native else None, + native_observed_through=native.verification_path if native else None, + verified_format=cap.format, + ) + return ReportingVerifiedDestination( + prepared.request, replace(resource, manifest_sha256=manifest_digest), verification + ) + + +def _native( + observed: ReportingNativeObservation, resource: ReportingResourceRecord, path: str +) -> None: + if type(observed) is not ReportingNativeObservation or ( + observed.location, + observed.native_version_ref, + observed.verification_path, + ) != (resource.location, resource.native_version_ref, path): + raise failure("SOURCE_INVALID") + + +async def _verify_files( + verifier: ReportingRevisionVerifier, + prepared: ReportingPreparedRevision, + locator: ReportingDestinationLocator, + session: ReportingDestinationSession, + context: ReportingIOContext, +) -> tuple[tuple[ReportingPhysicalChecksum, ...], str]: + raw = await context.run(lambda: session.read_manifest(locator), effect="unknown") + if type(raw) is not bytes or len(raw) > verifier.limits.max_value_bytes: + raise failure("SOURCE_INVALID") + digest = hashlib.sha256(raw).hexdigest() + if ( + locator.resource.manifest_sha256 is None + or digest != locator.resource.manifest_sha256.lower() + ): + raise failure("SOURCE_INVALID") + manifest = _mapping(parse_reporting_json(raw, verifier.limits)) + required = { + "manifest_version", + "complete", + "reporting_revision_id", + "reporting_obligation_id", + "reporting_materialization_id", + "period", + "format", + "compression", + "files", + "total_size_bytes", + "row_count", + "control_totals", + "created_at", + } + period = prepared.obligation.period + if ( + set(manifest) != required + or manifest["manifest_version"] != "1.0" + or manifest["complete"] is not True + or manifest["reporting_revision_id"] != prepared.request.reporting_revision_id + or manifest["reporting_obligation_id"] != prepared.request.reporting_obligation_id + or manifest["reporting_materialization_id"] != prepared.request.reporting_materialization_id + or manifest["format"] != verifier.key.capability.format + or manifest["compression"] != "none" + or manifest["period"] + != { + "start": period.start.isoformat(), + "end": period.end.isoformat(), + "source_timezone": period.source_timezone, + } + or type(manifest["row_count"]) is not int + or manifest["row_count"] != len(prepared.rows) + or manifest["control_totals"] + != [t.to_wire() for t in prepared.revision.managed_control_totals or ()] + or type(manifest["total_size_bytes"]) is not int + or manifest["total_size_bytes"] < 0 + ): + raise failure("SOURCE_INVALID") + created = None + try: + if type(manifest["created_at"]) is str and re.fullmatch( + r"\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})", + manifest["created_at"], + ): + created = datetime.fromisoformat( + manifest["created_at"].replace("Z", "+00:00").replace("z", "+00:00") + ) + except ValueError: + pass + if ( + created is None + or created.tzinfo is None + or created < prepared.revision.created_at + or created > datetime.now(timezone.utc) + ): + raise failure("SOURCE_INVALID") + entries = manifest["files"] + if type(entries) is not list or not 1 <= len(entries) <= verifier.limits.max_objects: + raise failure("SOURCE_INVALID") + refs = tuple(_checked_object_path(_mapping(entry).get("object_ref")) for entry in entries) + inventory = await context.run(lambda: session.list_objects(locator), effect="unknown") + if ( + len(set(refs)) != len(refs) + or refs != locator.resource.object_refs + or type(inventory) is not tuple + or inventory != refs + ): + raise failure("SOURCE_INVALID") + size, ordinal, chunks = 0, 0, 0 + checksums: list[ReportingPhysicalChecksum] = [] + for entry, ref in zip(entries, refs): + if ( + set(entry) != {"object_ref", "size_bytes", "row_count", "sha256"} + or type(entry["size_bytes"]) is not int + or type(entry["row_count"]) is not int + or entry["size_bytes"] < 0 + or entry["row_count"] < 0 + ): + raise failure("SOURCE_INVALID") + hashed, actual, count = hashlib.sha256(), 0, 0 + pending = b"" + stream = session.read_object(locator, object_ref=ref) + try: + while True: + # StopAsyncIteration is a stream boundary, not a provider failure. + chunk = await context.run(lambda: _next_chunk(stream), effect="unknown") + if chunk is None: + break + if type(chunk) is not bytes or not chunk: + raise failure("SOURCE_INVALID") + chunks += 1 + size += len(chunk) + actual += len(chunk) + if ( + chunks > verifier.limits.max_chunks + or size > verifier.limits.max_total_bytes + or actual > entry["size_bytes"] + ): + raise failure("LIMIT_EXCEEDED") + hashed.update(chunk) + pending += chunk + while b"\n" in pending: + line, pending = pending.split(b"\n", 1) + if ( + ordinal >= len(prepared.rows) + or strict_reporting_json( + parse_reporting_json(line, verifier.limits), verifier.limits + ) + != prepared.rows[ordinal] + ): + raise failure("SOURCE_INVALID") + ordinal += 1 + count += 1 + if len(pending) > verifier.limits.max_value_bytes: + raise failure("LIMIT_EXCEEDED") + finally: + close = getattr(stream, "aclose", None) + if close is not None: + await _close_owned(close, context.close_timeout_seconds) + if ( + pending + or actual != entry["size_bytes"] + or count != entry["row_count"] + or type(entry["sha256"]) is not str + or hashed.hexdigest() != entry["sha256"].lower() + ): + raise failure("SOURCE_INVALID") + checksums.append(ReportingPhysicalChecksum(ref, "sha256", hashed.hexdigest())) + if size != manifest["total_size_bytes"] or ordinal != len(prepared.rows): + raise failure("SOURCE_INVALID") + return tuple(checksums), digest + + +async def _next_chunk(stream: Any) -> bytes | None: + try: + return cast(bytes, await stream.__anext__()) + except StopAsyncIteration: + return None + + +async def _checkpoint() -> None: + """One service-owned cancellation/deadline/heartbeat boundary; never a loop.""" + + +def _checked_object_path(value: object) -> str: + valid = False + if type(value) is str: + try: + object_path(value) + valid = True + except ValueError: + pass + if not valid: + raise failure("SOURCE_INVALID") + return cast(str, value) + + +def _unordered_members(value: object) -> bool: + """The mandatory golden vector must exercise JCS member ordering as well.""" + if type(value) is dict: + mapping = cast(dict[str, object], value) + if list(mapping) != sorted(mapping, key=lambda name: name.encode("utf-16-be")): + return True + return any(_unordered_members(item) for item in mapping.values()) + if type(value) is list: + return any(_unordered_members(item) for item in cast(list[object], value)) + return False diff --git a/src/adcp/reporting/outbox/__init__.py b/src/adcp/reporting/outbox/__init__.py index 33ddfe60f..f6a0bc090 100644 --- a/src/adcp/reporting/outbox/__init__.py +++ b/src/adcp/reporting/outbox/__init__.py @@ -48,6 +48,7 @@ StatusCheckpoint, StatusDueLease, StatusNotificationStore, + StatusSelectorRebuildStore, StatusTurn, ) from adcp.reporting.outbox.status_memory import ( @@ -78,6 +79,7 @@ "StatusCheckpoint", "StatusDueLease", "StatusNotificationStore", + "StatusSelectorRebuildStore", "StatusTurn", "ReportingStatusProjector", "ReportingStatusSweeper", diff --git a/src/adcp/reporting/outbox/_capabilities.py b/src/adcp/reporting/outbox/_capabilities.py index 9779d53af..540bf4542 100644 --- a/src/adcp/reporting/outbox/_capabilities.py +++ b/src/adcp/reporting/outbox/_capabilities.py @@ -121,5 +121,7 @@ async def advertised_notifications( or frozen.currency != obligation.currency ): raise ReportingNotificationError("notification_chain_unready") - result["readiness_notification"] = "reporting.delivery_ready" + # B1 contracts and a frozen binding are not a durable materializer. + # B2 must supply a concrete verified write/finish readiness proof before + # a positive delivery_ready capability can be added here. return result diff --git a/src/adcp/reporting/outbox/required_status_selector_schema.json b/src/adcp/reporting/outbox/required_status_selector_schema.json new file mode 100644 index 000000000..14075e031 --- /dev/null +++ b/src/adcp/reporting/outbox/required_status_selector_schema.json @@ -0,0 +1,42 @@ +{ + "column:reporting_status_accounts.selector_target_version": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "column:reporting_status_accounts.selector_transition": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_status_scope_checkpoints.selector_semantics_version": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "column:reporting_status_scope_checkpoints.selector_writer_floor": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "constraint:reporting_status_accounts.reporting_status_selector_account_transition": { + "enabled": true, + "fingerprint": "47b1f4b23693b8d717bbd0efbe388416c6d32a9bf15487b80402d79c7df40f47" + }, + "constraint:reporting_status_scope_checkpoints.reporting_status_selector_versions": { + "enabled": true, + "fingerprint": "27e9e3994ed88d04f38bd6272cef49110c92143746ceb5845cd183d0ce2ed146" + }, + "function:reporting_status_selector_writer_guard_v2()": { + "enabled": true, + "fingerprint": "6f9f6df5355a5cc64f411ebc601b30c1358d9969061f5b92f4339804f887e837" + }, + "index:reporting_status_accounts.reporting_status_selector_accounts": { + "enabled": true, + "fingerprint": "fd952ac0f3fb0dab96f35395e97e3e2493769e3dcc05bbba71d37a298027f281" + }, + "index:reporting_status_scope_checkpoints.reporting_status_selector_rebuild": { + "enabled": true, + "fingerprint": "c8b8633eebbfb756d627e9e8b19d6949770cbe5374f0d63eb54e457751ba02a2" + }, + "trigger:reporting_status_scope_checkpoints.reporting_status_selector_writer_v2": { + "enabled": true, + "fingerprint": "ad9a5b09fa436bb9bdd7d0e4f88f88eba60ed730864b099636e499a747d78eac" + } +} diff --git a/src/adcp/reporting/outbox/status.py b/src/adcp/reporting/outbox/status.py index e8ea0a5b0..a218400a7 100644 --- a/src/adcp/reporting/outbox/status.py +++ b/src/adcp/reporting/outbox/status.py @@ -8,7 +8,7 @@ from dataclasses import dataclass, field from datetime import datetime -from typing import Any, Protocol +from typing import Any, Protocol, runtime_checkable from uuid import uuid4 from adcp.reporting.ledger.models import ReportingDeliveryEscalation @@ -25,6 +25,7 @@ apply_intents_to_snapshot, lifecycle_intents, ) +from adcp.reporting.revision_selection import REPORTING_SELECTOR_VERSION @dataclass(frozen=True) @@ -47,6 +48,8 @@ class StatusCheckpoint: publishable: bool lease_token: str | None = field(default=None, repr=False) lease_expires_at: datetime | None = None + selector_semantics_version: int = REPORTING_SELECTOR_VERSION + selector_writer_floor: int = REPORTING_SELECTOR_VERSION @dataclass(frozen=True) @@ -85,6 +88,13 @@ async def release_due(self, lease: StatusDueLease) -> bool: ... async def checkpoints(self, *, account_id: str) -> tuple[StatusCheckpoint, ...]: ... +@runtime_checkable +class StatusSelectorRebuildStore(Protocol): + """Indexed, account-discovering C cutover seam; no materializer work queue.""" + + async def rebuild_one(self) -> StatusTurn: ... + + @dataclass(frozen=True) class ReportingStatusProjector: store: StatusNotificationStore @@ -92,6 +102,12 @@ class ReportingStatusProjector: async def run_once(self, *, account_id: str) -> StatusTurn: return await self.store.project_one(account_id=account_id) + async def rebuild_once(self) -> StatusTurn: + """Reproject one populated old scope's account, without an account list.""" + if not isinstance(self.store, StatusSelectorRebuildStore): + raise ReportingNotificationError("status_selector_rebuild_unsupported") + return await self.store.rebuild_one() + @dataclass(frozen=True) class ReportingStatusSweeper: @@ -171,4 +187,7 @@ def settled_replay(snapshot: ReportingStatusSnapshot) -> ReportingStatusSnapshot def escalation_identity(escalation: ReportingDeliveryEscalation | None) -> dict[str, Any]: - return escalation.to_wire() if escalation else {} + return { + **(escalation.to_wire() if escalation else {}), + "selector_semantics_version": REPORTING_SELECTOR_VERSION, + } diff --git a/src/adcp/reporting/outbox/status_memory.py b/src/adcp/reporting/outbox/status_memory.py index b20a094d7..e3a9b89d0 100644 --- a/src/adcp/reporting/outbox/status_memory.py +++ b/src/adcp/reporting/outbox/status_memory.py @@ -5,7 +5,7 @@ from dataclasses import dataclass, field, replace from datetime import timedelta from secrets import token_hex -from typing import Any +from typing import Any, Literal from adcp.reporting.ledger.models import ReportingDeliveryEscalation from adcp.reporting.ledger.notification_models import ReportingNotificationError @@ -27,6 +27,7 @@ escalation_identity, settled_replay, ) +from adcp.reporting.revision_selection import REPORTING_SELECTOR_VERSION @dataclass @@ -37,6 +38,7 @@ class _StatusMemoryState: ) outbox: NotificationState = field(default_factory=NotificationState) replay: dict[str, ReportingStatusSnapshot] = field(default_factory=dict) + selector_accounts: dict[str, Literal["transitioning", "complete"]] = field(default_factory=dict) class InMemoryReportingStatusOutbox(InMemoryReportingOutbox): @@ -58,6 +60,27 @@ def __init__( self.ledger, self.escalation = ledger, escalation if ledger._status_notification_state is None: ledger._status_notification_state = _StatusMemoryState() + state = ledger._status_notification_state + if not hasattr(state, "selector_accounts"): + state.selector_accounts = {} + for key, checkpoint in tuple(state.checkpoints.items()): + # An old shared-state image has no epoch fields. Decode by keyword + # rather than letting new dataclass class defaults label it v2. + if "selector_semantics_version" not in vars(checkpoint): + state.checkpoints[key] = StatusCheckpoint( + scope=checkpoint.scope, + fingerprint=checkpoint.fingerprint, + generation=checkpoint.generation, + snapshot=checkpoint.snapshot, + next_due_at=checkpoint.next_due_at, + source_sequence=checkpoint.source_sequence, + baseline=checkpoint.baseline, + publishable=checkpoint.publishable, + lease_token=checkpoint.lease_token, + lease_expires_at=checkpoint.lease_expires_at, + selector_semantics_version=1, + selector_writer_floor=1, + ) self.outbox = InMemoryReportingStatusOutbox(ledger) @property @@ -79,7 +102,8 @@ def _cursor(self, account_id: str) -> int: async def baseline(self, *, account_id: str) -> bool: async with self.ledger._mutation(): if account_id in self._state.accounts: - self._cursor(account_id) + if not self._needs_rebuild(account_id): + self._cursor(account_id) return False snapshot = settle_memory_snapshot(self.ledger, account_id) assert self.ledger._notification_state is not None @@ -94,21 +118,99 @@ async def baseline(self, *, account_id: str) -> bool: self._apply(snapshot, through=through, baseline=True) self._state.replay[account_id] = snapshot self._state.accounts[account_id] = (through, escalation_identity(self.escalation)) + self._state.selector_accounts[account_id] = "complete" return True async def baseline_ready(self, *, account_id: str) -> bool: async with self.ledger._mutation(): if account_id not in self._state.accounts: return False + if self._needs_rebuild(account_id): + return False self._cursor(account_id) return True + def _needs_rebuild(self, account_id: str) -> bool: + return account_id in self._state.accounts and ( + self._state.selector_accounts.get(account_id) != "complete" + or any( + c.scope.account_id == account_id + and ( + c.selector_semantics_version != REPORTING_SELECTOR_VERSION + or c.selector_writer_floor != REPORTING_SELECTOR_VERSION + ) + for c in self._state.checkpoints.values() + ) + ) + + def _rebuild(self, account_id: str) -> StatusTurn: + if not self._needs_rebuild(account_id): + return StatusTurn(False) + if self._state.selector_accounts.get(account_id) != "transitioning": + through, policy = self._state.accounts[account_id] + expected = escalation_identity(self.escalation) + if policy not in ( + expected, + {k: v for k, v in expected.items() if k != "selector_semantics_version"}, + ): + raise ReportingNotificationError("status_policy_conflict") + self._state.accounts[account_id] = (through, expected) + self._state.selector_accounts[account_id] = "transitioning" + for key, checkpoint in tuple(self._state.checkpoints.items()): + if checkpoint.scope.account_id == account_id: + self._state.checkpoints[key] = replace( + checkpoint, selector_writer_floor=REPORTING_SELECTOR_VERSION + ) + return StatusTurn(True) + turn = self._project(account_id) + if turn.did_work: + return turn + snapshot = settle_memory_snapshot(self.ledger, account_id) + deadlines = [ + c.next_due_at + for c in self._state.checkpoints.values() + if c.scope.account_id == account_id + and c.next_due_at is not None + and c.next_due_at <= snapshot.as_of + ] + if deadlines: + return StatusTurn( + True, + self._apply( + replace(snapshot, as_of=min(deadlines)), through=self._cursor(account_id) + ), + ) + count = self._apply(snapshot, through=self._cursor(account_id)) + self._state.replay[account_id] = snapshot + self._state.selector_accounts[account_id] = "complete" + return StatusTurn(True, count) + + async def rebuild_one(self) -> StatusTurn: + async with self.ledger._mutation(): + expected = escalation_identity(self.escalation) + legacy = {k: v for k, v in expected.items() if k != "selector_semantics_version"} + account_id = next( + ( + a + for a in sorted(self._state.accounts) + if self._needs_rebuild(a) and self._state.accounts[a][1] in (expected, legacy) + ), + None, + ) + return self._rebuild(account_id) if account_id is not None else StatusTurn(False) + def _apply( self, snapshot: ReportingStatusSnapshot, *, through: int, baseline: bool = False ) -> int: snapshot = settled_replay(snapshot) events = 0 - for scope in projection_scopes(snapshot): + scopes = {s.checkpoint_key: s for s in projection_scopes(snapshot)} + scopes.update( + (key, c.scope) + for key, c in self._state.checkpoints.items() + if c.scope.account_id == snapshot.account_id + ) + for _, scope in sorted(scopes.items()): result = project_status_scope(StatusProjectionInput(snapshot, scope, self.escalation)) checkpoint, event = advance_checkpoint( self._state.checkpoints.get(scope.checkpoint_key), @@ -146,6 +248,9 @@ def _project(self, account_id: str) -> StatusTurn: async def project_one(self, *, account_id: str) -> StatusTurn: async with self.ledger._mutation(): + rebuilt = self._rebuild(account_id) + if rebuilt.did_work: + return rebuilt return self._project(account_id) async def claim_due( @@ -154,6 +259,8 @@ async def claim_due( if lease_seconds <= 0: raise ValueError("lease_seconds must be positive") async with self.ledger._lock: + if self._needs_rebuild(account_id): + return None self._cursor(account_id) at = self.ledger._clock() for key, checkpoint in sorted(self._state.checkpoints.items()): @@ -183,6 +290,8 @@ async def complete_due(self, lease: StatusDueLease) -> StatusTurn: async def _complete_due(self, lease: StatusDueLease) -> StatusTurn: async with self.ledger._mutation(): + if self._needs_rebuild(lease.scope.account_id): + return StatusTurn(False) checkpoint = self._state.checkpoints.get(lease.scope.checkpoint_key) at = self.ledger._clock() if ( diff --git a/src/adcp/reporting/outbox/status_pg.py b/src/adcp/reporting/outbox/status_pg.py index 941fe9954..4ac861d90 100644 --- a/src/adcp/reporting/outbox/status_pg.py +++ b/src/adcp/reporting/outbox/status_pg.py @@ -58,7 +58,7 @@ ) _CHECKPOINT = ( "scope, fingerprint, generation, snapshot, next_due_at, source_sequence, baseline, publishable," - " lease_token, lease_expires_at, initialized" + " lease_token, lease_expires_at, selector_semantics_version, selector_writer_floor, initialized" ) _LIFECYCLE = TypeAdapter(ReportingIssueLifecycle) @@ -212,11 +212,19 @@ async def create_schema(self) -> None: .joinpath("reporting_status_notifications.sql") .read_text() ) + await connection.execute( + files("adcp.reporting.ledger") + .joinpath("reporting_status_selector_version.sql") + .read_text() + ) await validate_status_schema(connection) @asynccontextmanager async def _transaction(self, account_id: str) -> AsyncIterator[Any]: async with self.ledger._pool.connection() as connection, connection.transaction(): + await connection.execute( + "SELECT set_config('adcp.reporting.selector_semantics_version', '2', true)" + ) await self.ledger._lock_account(connection, account_id) yield connection @@ -240,8 +248,8 @@ async def _lock_scopes_on(self, connection: Any, snapshot: ReportingStatusSnapsh for scope in projection_scopes(snapshot): await connection.execute( f"INSERT INTO reporting_status_scope_checkpoints ({_KEY}, scope, fingerprint," # nosec B608 - " snapshot, source_sequence, baseline, publishable)" - ' VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s,\'{"health":"waiting"}\',0,FALSE,FALSE)' + " snapshot, source_sequence, baseline, publishable, selector_writer_floor)" + ' VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s,\'{"health":"waiting"}\',0,FALSE,FALSE,2)' f" ON CONFLICT ({_KEY}) DO NOTHING", # nosec B608 (*scope.checkpoint_key, json.dumps(asdict(scope)), "0" * 64), ) @@ -258,7 +266,8 @@ async def _write_on(self, connection: Any, checkpoint: StatusCheckpoint) -> None await connection.execute( "UPDATE reporting_status_scope_checkpoints SET scope=%s::jsonb, fingerprint=%s," " generation=%s, snapshot=%s::jsonb, next_due_at=%s, source_sequence=%s," - " baseline=%s, publishable=%s, initialized=TRUE" + " baseline=%s, publishable=%s, initialized=TRUE, selector_semantics_version=%s," + " selector_writer_floor=%s" f" WHERE {_WHERE}", # nosec B608 ( json.dumps(asdict(checkpoint.scope)), @@ -269,6 +278,8 @@ async def _write_on(self, connection: Any, checkpoint: StatusCheckpoint) -> None checkpoint.source_sequence, checkpoint.baseline, checkpoint.publishable, + checkpoint.selector_semantics_version, + checkpoint.selector_writer_floor, *checkpoint.scope.checkpoint_key, ), ) @@ -284,7 +295,15 @@ async def _apply_on( await self._lock_scopes_on(connection, snapshot) snapshot = settled_replay(snapshot) count = 0 - for scope in projection_scopes(snapshot): + rows = await ( + await connection.execute( + "SELECT scope FROM reporting_status_scope_checkpoints WHERE account_id=%s" + " ORDER BY account_id, consumer_namespace, delivery_config_id, version," + " scope_kind, obligation_namespace", + (snapshot.account_id,), + ) + ).fetchall() + for scope in (decode_status_scope(row[0]) for row in rows): row = await ( await connection.execute( f"SELECT {_CHECKPOINT} FROM reporting_status_scope_checkpoints WHERE {_WHERE}" # nosec B608 @@ -318,7 +337,8 @@ async def baseline(self, *, account_id: str) -> bool: ) ).fetchone() if row is not None and row[0]: - await self._account_on(connection, account_id) + if not await self._needs_rebuild_on(connection, account_id): + await self._account_on(connection, account_id) return False await connection.execute( "INSERT INTO reporting_status_accounts (account_id, policy) VALUES (%s,%s::jsonb)" @@ -338,6 +358,7 @@ async def baseline(self, *, account_id: str) -> bool: "UPDATE reporting_status_accounts SET baseline_complete=TRUE," " baseline_highwater=%s," " dirty_sequence=%s, baseline_at=%s, replay_lifecycles=%s::jsonb" + ", selector_target_version=2, selector_transition='complete'" " WHERE account_id=%s", (through, through, snapshot.as_of, _replay_storage(snapshot), account_id), ) @@ -355,7 +376,7 @@ async def baseline_ready(self, *, account_id: str) -> bool: (account_id,), ) ).fetchone() - if row is None or not row[0]: + if row is None or not row[0] or await self._needs_rebuild_on(connection, account_id): return False if row[1] != escalation_identity(self.escalation): raise ReportingNotificationError("status_policy_conflict") @@ -363,6 +384,108 @@ async def baseline_ready(self, *, account_id: str) -> bool: # account's current business health or waived issue occurrences. return True + async def _needs_rebuild_on(self, connection: Any, account_id: str) -> bool: + row = await ( + await connection.execute( + "SELECT selector_target_version <> 2 OR selector_transition <> 'complete'" + " OR EXISTS(SELECT 1 FROM reporting_status_scope_checkpoints c" + " WHERE c.account_id=a.account_id AND" + " (c.selector_semantics_version <> 2 OR c.selector_writer_floor <> 2))" + " FROM reporting_status_accounts a WHERE account_id=%s AND baseline_complete", + (account_id,), + ) + ).fetchone() + return bool(row and row[0]) + + async def _rebuild_on(self, connection: Any, account_id: str) -> StatusTurn: + if not await self._needs_rebuild_on(connection, account_id): + return StatusTurn(False) + row = await ( + await connection.execute( + "SELECT policy, selector_transition FROM reporting_status_accounts" + " WHERE account_id=%s FOR UPDATE", + (account_id,), + ) + ).fetchone() + expected = escalation_identity(self.escalation) + legacy = {k: v for k, v in expected.items() if k != "selector_semantics_version"} + if row[0] not in (expected, legacy): + raise ReportingNotificationError("status_policy_conflict") + if row[1] != "transitioning": + # Phase one commits a checkpoint-local writer floor. The guard + # never reads/locks an account, including for old due claimers. + await ( + await connection.execute( + "SELECT 1 FROM reporting_status_scope_checkpoints WHERE account_id=%s" + " ORDER BY account_id, consumer_namespace, delivery_config_id, version," + " scope_kind, obligation_namespace FOR UPDATE", + (account_id,), + ) + ).fetchall() + await connection.execute( + "UPDATE reporting_status_scope_checkpoints SET selector_writer_floor=2" + " WHERE account_id=%s AND selector_writer_floor <> 2", + (account_id,), + ) + await connection.execute( + "UPDATE reporting_status_accounts SET selector_target_version=2," + " selector_transition='transitioning', policy=%s::jsonb WHERE account_id=%s", + (json.dumps(expected), account_id), + ) + return StatusTurn(True) + # Phase two advances exactly one immutable boundary or deadline per + # transaction. A crash leaves the durable cursor at the last commit. + turn = await self._project_on(connection, account_id) + if turn.did_work: + return turn + snapshot = await settle_snapshot_on(self.ledger, connection, account_id=account_id) + through = await self._account_on(connection, account_id) + due = await ( + await connection.execute( + "SELECT min(next_due_at) FROM reporting_status_scope_checkpoints" + " WHERE account_id=%s AND next_due_at <= %s", + (account_id, snapshot.as_of), + ) + ).fetchone() + if due[0] is not None: + count = await self._apply_on( + connection, replace(snapshot, as_of=due[0]), through=through + ) + return StatusTurn(True, count) + count = await self._apply_on(connection, snapshot, through=through) + await connection.execute( + "UPDATE reporting_status_accounts SET selector_transition='complete'," + " replay_lifecycles=%s::jsonb WHERE account_id=%s", + (_replay_storage(snapshot), account_id), + ) + return StatusTurn(True, count) + + async def rebuild_one(self) -> StatusTurn: + """Discover incomplete C cutovers by index, without enumerating accounts.""" + expected = escalation_identity(self.escalation) + policies = ( + json.dumps(expected), + json.dumps({k: v for k, v in expected.items() if k != "selector_semantics_version"}), + ) + async with self.ledger._pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT account_id FROM reporting_status_accounts WHERE baseline_complete" + " AND (selector_target_version <> 2 OR selector_transition <> 'complete')" + " AND policy IN (%s::jsonb,%s::jsonb)" + " UNION SELECT c.account_id FROM reporting_status_scope_checkpoints c" + " JOIN reporting_status_accounts a ON a.account_id=c.account_id" + " WHERE (c.selector_semantics_version <> 2 OR c.selector_writer_floor <> 2)" + " AND a.baseline_complete AND a.policy IN (%s::jsonb,%s::jsonb)" + " ORDER BY account_id LIMIT 1", + (*policies, *policies), + ) + ).fetchone() + if row is None: + return StatusTurn(False) + async with self._transaction(row[0]) as connection: + return await self._rebuild_on(connection, row[0]) + async def _project_on(self, connection: Any, account_id: str) -> StatusTurn: through = await self._account_on(connection, account_id) row = await ( @@ -394,6 +517,9 @@ async def _project_on(self, connection: Any, account_id: str) -> StatusTurn: async def project_one(self, *, account_id: str) -> StatusTurn: async with self._transaction(account_id) as connection: + rebuilt = await self._rebuild_on(connection, account_id) + if rebuilt.did_work: + return rebuilt return await self._project_on(connection, account_id) async def claim_due( @@ -401,7 +527,10 @@ async def claim_due( ) -> StatusDueLease | None: if lease_seconds <= 0: raise ValueError("lease_seconds must be positive") - async with self.ledger._pool.connection() as connection, connection.transaction(): + async with self._transaction(account_id) as connection: + if await self._needs_rebuild_on(connection, account_id): + return None + await self._account_on(connection, account_id) at = await database_now(connection, self.ledger._clock) row = await ( await connection.execute( @@ -452,6 +581,8 @@ async def _ack_on(self, connection: Any, lease: StatusDueLease) -> bool: async def complete_due(self, lease: StatusDueLease) -> StatusTurn: try: async with self._transaction(lease.scope.account_id) as connection: + if await self._needs_rebuild_on(connection, lease.scope.account_id): + return StatusTurn(False) await self._account_on(connection, lease.scope.account_id) if not await self._held_on(connection, lease): return StatusTurn(False) diff --git a/src/adcp/reporting/outbox/status_schema.py b/src/adcp/reporting/outbox/status_schema.py index d1f4c636f..c83a17dc6 100644 --- a/src/adcp/reporting/outbox/status_schema.py +++ b/src/adcp/reporting/outbox/status_schema.py @@ -12,6 +12,9 @@ REQUIRED_STATUS_OBJECTS: dict[str, dict[str, Any]] = json.loads( files("adcp.reporting.outbox").joinpath("required_status_schema.json").read_text() ) +REQUIRED_STATUS_SELECTOR_OBJECTS: dict[str, dict[str, Any]] = json.loads( + files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").read_text() +) async def validate_status_schema( @@ -46,3 +49,9 @@ async def validate_status_schema( else: continue raise ReportingNotificationError(f"status_schema_unready:{reason}:{key}") + if status: + if not REQUIRED_STATUS_SELECTOR_OBJECTS: + raise ReportingNotificationError("status_selector_schema_unready:manifest_missing") + for key, expected in REQUIRED_STATUS_SELECTOR_OBJECTS.items(): + if installed.get(key) != expected: + raise ReportingNotificationError(f"status_selector_schema_unready:{key}") diff --git a/src/adcp/reporting/outbox/status_service.py b/src/adcp/reporting/outbox/status_service.py index f82fa09e5..03275fca2 100644 --- a/src/adcp/reporting/outbox/status_service.py +++ b/src/adcp/reporting/outbox/status_service.py @@ -5,7 +5,7 @@ from typing import Protocol from adcp.reporting.ledger.notification_models import ReportingNotificationError -from adcp.reporting.outbox.status import StatusTurn +from adcp.reporting.outbox.status import StatusSelectorRebuildStore, StatusTurn from adcp.reporting.outbox.status_support import ReportingStatusSupport @@ -68,6 +68,14 @@ async def project_dirty_once(self, *, account_id: str) -> StatusTurn: raise ReportingNotificationError("status_projector_unavailable") return await self.support.projector.run_once(account_id=account_id) + async def rebuild_selector_once(self) -> StatusTurn: + """Indexed C cutover, including retained accounts outside current registrations.""" + self._open() + store = self.support.store + if not isinstance(store, StatusSelectorRebuildStore): + return StatusTurn(False) # Existing custom lifecycle protocols stay compatible. + return await store.rebuild_one() + async def sweep_due_once(self, *, account_id: str) -> StatusTurn: self._open(account_id) if self.support.sweeper is None: @@ -87,7 +95,7 @@ async def drain(self, *, max_turns: int = 100) -> int: if max_turns < 1: raise ValueError("max_turns must be positive") for turn in range(max_turns): - worked = False + worked = (await self.rebuild_selector_once()).did_work for account_id in sorted(set(self.support.account_ids)): worked = (await self.project_dirty_once(account_id=account_id)).did_work or worked worked = (await self.sweep_due_once(account_id=account_id)).did_work or worked diff --git a/src/adcp/reporting/revision_selection.py b/src/adcp/reporting/revision_selection.py new file mode 100644 index 000000000..a09365081 --- /dev/null +++ b/src/adcp/reporting/revision_selection.py @@ -0,0 +1,191 @@ +"""Strict, pure publication selection shared by seller and buyer projections. + +Validate the *whole* obligation history before choosing finality. Destination +state, timestamps and readability never choose a publication. In particular, +an official close does not supersede (or excuse damage in) the snapshot chain. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from dataclasses import dataclass, field +from typing import Generic, Literal, Protocol, TypeVar + +__all__ = [ + "REPORTING_SELECTOR_VERSION", + "ReportingRevisionCorrupt", + "ReportingRevisionNotReady", + "ReportingRevisionSelected", + "ReportingRevisionSelection", + "RevisionHistoryEntry", + "select_reporting_revision", +] + +REPORTING_SELECTOR_VERSION = 2 + + +class _Revision(Protocol): + @property + def account_id(self) -> str: ... + + @property + def reporting_obligation_id(self) -> str: ... + + @property + def reporting_revision_id(self) -> str: ... + + @property + def finality(self) -> str: ... + + @property + def supersedes_reporting_revision_id(self) -> str | None: ... + + +R = TypeVar("R", bound=_Revision) + + +@dataclass(frozen=True, slots=True) +class RevisionHistoryEntry: + """Adapter for wire revisions whose ownership is established by their caller.""" + + account_id: str + reporting_obligation_id: str + reporting_revision_id: str + finality: str + supersedes_reporting_revision_id: str | None = None + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionSelected(Generic[R]): + revision: R + kind: Literal["selected"] = field(default="selected", init=False) + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionNotReady: + reason: Literal["empty_history", "official_required"] + kind: Literal["not_ready"] = field(default="not_ready", init=False) + + +@dataclass(frozen=True, slots=True) +class ReportingRevisionCorrupt: + # Closed diagnostics: no row data, credentials or provider errors. + reason: Literal[ + "ownership_mismatch", + "duplicate_revision_id", + "invalid_revision_identity", + "invalid_finality", + "missing_predecessor", + "cross_finality_edge", + "official_predecessor", + "forked_snapshot_history", + "disconnected_snapshot_history", + "revision_cycle", + "multiple_officials", + ] + kind: Literal["corrupt"] = field(default="corrupt", init=False) + + +ReportingRevisionSelection = ( + ReportingRevisionSelected[R] | ReportingRevisionNotReady | ReportingRevisionCorrupt +) + + +def select_reporting_revision( + revisions: Sequence[R], + *, + account_id: str, + reporting_obligation_id: str, + required_finality: str, +) -> ReportingRevisionSelection[R]: + """Linear-time, order-independent selection, including disconnected cycles. + + Only empty history or a missing required official is ordinary not-ready. + The caller supplies every retained revision belonging to the obligation, + without filtering by finality, readability, or existing materializations. + A selected unreadable revision remains selected: repair it, never fall back. + """ + if ( + type(account_id) is not str + or type(reporting_obligation_id) is not str + or any( + type(r.account_id) is not str + or type(r.reporting_obligation_id) is not str + or r.account_id != account_id + or r.reporting_obligation_id != reporting_obligation_id + for r in revisions + ) + ): + return ReportingRevisionCorrupt("ownership_mismatch") + if any( + type(r.reporting_revision_id) is not str + or not r.reporting_revision_id + or ( + r.supersedes_reporting_revision_id is not None + and type(r.supersedes_reporting_revision_id) is not str + ) + for r in revisions + ): + return ReportingRevisionCorrupt("invalid_revision_identity") + by_id = {r.reporting_revision_id: r for r in revisions} + if len(by_id) != len(revisions): + return ReportingRevisionCorrupt("duplicate_revision_id") + if ( + type(required_finality) is not str + or required_finality not in {"snapshot", "official"} + or any( + type(r.finality) is not str or r.finality not in {"snapshot", "official"} + for r in revisions + ) + ): + return ReportingRevisionCorrupt("invalid_finality") + if any( + r.supersedes_reporting_revision_id is not None + and r.supersedes_reporting_revision_id not in by_id + for r in revisions + ): + return ReportingRevisionCorrupt("missing_predecessor") + if any( + r.supersedes_reporting_revision_id is not None + and by_id[r.supersedes_reporting_revision_id].finality != r.finality + for r in revisions + ): + return ReportingRevisionCorrupt("cross_finality_edge") + if any( + r.finality == "official" and r.supersedes_reporting_revision_id is not None + for r in revisions + ): + return ReportingRevisionCorrupt("official_predecessor") + officials = [r for r in revisions if r.finality == "official"] + if len(officials) > 1: + return ReportingRevisionCorrupt("multiple_officials") + snapshots = [r for r in revisions if r.finality == "snapshot"] + successors: dict[str, str] = {} + for revision in snapshots: + predecessor = revision.supersedes_reporting_revision_id + if predecessor is not None: + if predecessor in successors: + return ReportingRevisionCorrupt("forked_snapshot_history") + successors[predecessor] = revision.reporting_revision_id + # Walk each component once. A unique-looking leaf cannot hide a cycle. + visited: set[str] = set() + for revision in snapshots: + path: set[str] = set() + node: str | None = revision.reporting_revision_id + while node is not None and node not in visited: + if node in path: + return ReportingRevisionCorrupt("revision_cycle") + path.add(node) + node = by_id[node].supersedes_reporting_revision_id + visited.update(path) + roots = [r for r in snapshots if r.supersedes_reporting_revision_id is None] + if snapshots and len(roots) != 1: + return ReportingRevisionCorrupt("disconnected_snapshot_history") + if not revisions: + return ReportingRevisionNotReady("empty_history") + if officials: + return ReportingRevisionSelected(officials[0]) + if required_finality == "official": + return ReportingRevisionNotReady("official_required") + leaf = next(r for r in snapshots if r.reporting_revision_id not in successors) + return ReportingRevisionSelected(leaf) diff --git a/tests/conformance/reporting/_frozen_status_c.py b/tests/conformance/reporting/_frozen_status_c.py new file mode 100644 index 000000000..d7c1dea63 --- /dev/null +++ b/tests/conformance/reporting/_frozen_status_c.py @@ -0,0 +1,111 @@ +"""JSON-line control for the exact reviewed C artifact, imported with python -I. + +This file deliberately imports no workspace SDK before choosing the frozen +source. It is a test driver, not a production migration or recovery service. +""" + +import asyncio +import json +import sys +from dataclasses import asdict +from datetime import datetime +from pathlib import Path + + +def emit(value): + print(json.dumps(value, default=lambda v: v.isoformat()), flush=True) + + +async def read(): + return json.loads(await asyncio.to_thread(sys.stdin.readline)) + + +async def main(): + config = await read() + root = Path(config["source"]).resolve() + sys.path.insert(0, str(root / "src")) + from psycopg import AsyncConnection, Error + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.outbox.status_pg as module + from adcp.reporting.ledger import PgReportingReconciliationStore + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox import PgStatusNotificationStore + + assert Path(module.__file__).is_relative_to(root) + gate = None + now = datetime.fromisoformat(config["now"]) + + class HeldConnection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + nonlocal gate + result = await super().execute(query, params, **kwargs) + if ( + gate + and isinstance(query, str) + and query.startswith("UPDATE reporting_status_scope_checkpoints SET") + ): + if gate == "project" or "SET lease_token=" in query: + gate = None + emit({"held": True}) + assert (await read())["action"] == "release_hold" + return result + + async with AsyncConnectionPool( + config["conninfo"], + kwargs=config["kwargs"], + min_size=1, + max_size=1, + connection_class=HeldConnection, + open=False, + ) as pool: + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: now) + status = PgStatusNotificationStore(ledger) + leases = [] + emit({"origin": str(module.__file__)}) + while True: + command = await read() + action = command["action"] + account = command.get("account", "acct_a") + if "now" in command: + now = datetime.fromisoformat(command["now"]) + gate = command.get("hold") + try: + if action == "stop": + emit({"stopped": True}) + return + if action == "schema": + await status.create_schema() + result = True + elif action == "baseline": + result = await status.baseline(account_id=account) + elif action == "ready": + result = await status.baseline_ready(account_id=account) + elif action == "project": + result = asdict(await status.project_one(account_id=account)) + elif action == "claim": + lease = await status.claim_due(account_id=account, lease_seconds=60) + leases.append(lease) + result = None if lease is None else asdict(lease) + elif action == "complete": + result = asdict(await status.complete_due(leases[command.get("lease", -1)])) + elif action == "release": + result = await status.release_due(leases[command.get("lease", -1)]) + elif action == "source": + await ledger.set_revision_readable( + account_id=account, + reporting_revision_id=command["revision"], + readable=command["readable"], + ) + result = True + else: + raise AssertionError("unknown test command") + emit({"result": result}) + except ReportingNotificationError as exc: + emit({"error": exc.code}) + except Error as exc: + emit({"error": "database_fence", "sqlstate": exc.sqlstate}) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py new file mode 100644 index 000000000..dc5f5c4f2 --- /dev/null +++ b/tests/conformance/reporting/_materializer_installed.py @@ -0,0 +1,188 @@ +"""Copied outside the checkout and executed with Python 3.10 -I against a wheel.""" + +import asyncio +import hashlib +import importlib +import importlib.util +import json +import sys +from datetime import datetime, timedelta, timezone +from importlib.resources import files +from pathlib import Path + + +async def main(): + config = json.load(sys.stdin) + assert sys.version_info[:2] == (3, 10) + assert importlib.util.find_spec("psycopg") is None + assert importlib.util.find_spec("psycopg_pool") is None + + assert "adcp.reporting.materializer" not in sys.modules + for name in ( + "adcp.reporting.materializer", + "adcp.reporting.revision_selection", + "adcp.reporting.ledger", + "adcp.reporting.outbox", + ): + module = importlib.import_module(name) + assert len(module.__all__) == len(set(module.__all__)) + for symbol in module.__all__: + getattr(module, symbol) + from adcp.reporting.ledger import ( + InMemoryReportingReconciliationStore, + ReportingConfiguration, + ReportingDeliveryScope, + ReportingObligationRecord, + ReportingRevisionRecord, + ReportingScheduleSpec, + derive_period, + revision_content_sha256, + ) + from adcp.reporting.materializer import ( + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + reference_digest, + reference_verifier, + ) + + example_path = Path(config["example"]) + spec = importlib.util.spec_from_file_location("installed_example", example_path) + example = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = example + spec.loader.exec_module(example) + verifier = reference_verifier() + actual = { + name: hashlib.sha256( + files("adcp.reporting.materializer").joinpath("assets", name).read_bytes() + ).hexdigest() + for name in config["assets"] + } + assert actual == config["assets"] + assert ( + files("adcp.reporting.ledger").joinpath("reporting_status_selector_version.sql").is_file() + ) + assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() + start = datetime(2026, 9, 1, tzinfo=timezone.utc) + schedule = ReportingScheduleSpec("PT1H", "PT1H", period_anchor=start) + period = derive_period(schedule, account_timezone="UTC", ordinal=0) + for count in (0, 501): + ledger = InMemoryReportingReconciliationStore() + configuration = ReportingConfiguration( + delivery_config_id="daily", + delivery_config_version=1, + account_id="account", + report_definition_id=verifier.key.report_definition_id, + reporting_profile=verifier.key.reporting_profile, + feed_purpose="analytics", + required_finality="snapshot", + schedule=schedule, + activated_at=start, + definition=verifier.key.definition, + ) + await ledger.put_configuration(configuration) + obligation = ReportingObligationRecord( + reporting_obligation_id="obligation", + account_id="account", + delivery_config_id="daily", + delivery_config_version=1, + report_definition_id=verifier.key.report_definition_id, + reporting_profile=verifier.key.reporting_profile, + feed_purpose="analytics", + period=period, + scope_resolved_at=period.end, + media_buy_ids=("buy",), + required_finality="snapshot", + automated_recovery_deadline_at=period.expected_at + timedelta(hours=1), + schedule=schedule, + definition=verifier.key.definition, + created_at=period.end, + currency="USD", + ) + await ledger.commit_obligation(obligation) + binding = ReportingDestinationBinding( + configuration.generation_key, + "https://buyer.example.test/agents/reporting", + "reference-destination", + "trusted-binding", + "file_transfer", + "reference-memory", + "canonical_digest", + "delivery_only", + "analytics", + 400, + start, + "jsonl", + ("reference-v1",), + ) + await ledger.put_destination_binding(binding) + scope = ReportingDeliveryScope( + configuration.generation_key, binding.consumer_id, obligation.reporting_obligation_id + ) + delivery = ReportingObligationDeliveryRecord( + scope, "USD", period.end + timedelta(days=400), period.end + ) + await ledger.bind_obligation_delivery(delivery) + rows = [ + {"row_id": f"{i:06d}", "impressions": i, "spend": "1.25", "currency": "USD"} + for i in range(count) + ] + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + "revision", + "account", + obligation.reporting_obligation_id, + "snapshot", + revision_content_sha256( + reporting_revision_id="revision", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + period.end, + period.end, + period.end, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + await ledger.commit_revision(revision, rows) + attempt = ReportingMaterializationAttempt( + scope, revision.reporting_revision_id, "materialization", 1, period.end + ) + await ledger.commit_materialization_attempt(attempt) + destination = example.development_destination(binding) + results = [] + for _ in range(2): + results.append( + await destination.verify_revision( + reader=ledger, + binding=binding, + delivery=delivery, + obligation=obligation, + revisions=(revision,), + attempt=attempt, + deadline_at=datetime.now(timezone.utc) + timedelta(seconds=60), + cancel=asyncio.Event(), + ) + ) + assert all(result.verification.row_count == count for result in results) + assert results[0].request.external_id == results[1].request.external_id + assert destination.writer.write_effects == 1 and not destination.writer.production_eligible + assert destination.writer.open_count == destination.writer.close_count == 4 + workspace = Path(config["workspace"]).resolve() + assert all(not Path(path).resolve().is_relative_to(workspace) for path in sys.path) + assert all( + not Path(module.__file__).resolve().is_relative_to(workspace) + for name, module in sys.modules.items() + if name == "adcp" or name.startswith("adcp.") + if getattr(module, "__file__", None) + ) + assert not any(name.startswith("psycopg") for name in sys.modules) + print(json.dumps({"python": "3.10", "rows": [0, 501], "installed": True, "assets": actual})) + + +asyncio.run(main()) diff --git a/tests/conformance/reporting/_materializer_support.py b/tests/conformance/reporting/_materializer_support.py new file mode 100644 index 000000000..dba8a9054 --- /dev/null +++ b/tests/conformance/reporting/_materializer_support.py @@ -0,0 +1,178 @@ +"""Frozen content shared by the B1 pure, memory and installed-artifact gates.""" + +import asyncio +from dataclasses import dataclass, replace +from datetime import datetime, timedelta, timezone + +from adcp.reporting.ledger import ( + InMemoryReportingReconciliationStore, + ReportingDeliveryScope, + ReportingRevisionRecord, + revision_content_sha256, +) +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationBinding, + ReportingDestinationIO, + ReportingIOContext, + ReportingMaterializationAttempt, + ReportingObligationDeliveryRecord, + ReportingRevisionVerifierRegistry, + reference_digest, + reference_verifier, +) + +from ._generation_support import END, START, configuration, obligation_for + + +def io_context(seconds=30): + return ReportingIOContext( + datetime.now(timezone.utc) + timedelta(seconds=seconds), asyncio.Event() + ) + + +def reference_rows(count): + return [ + { + "row_id": f"{i:06d}", + "impressions": i % 3, + "spend": "1.25", + "currency": "USD", + "details": {"active": True, "values": [1, None, "é", "e\u0301"]}, + } + for i in range(count) + ] + + +@dataclass +class Case: + store: object + verifier: object + registry: object + binding: object + delivery: object + obligation: object + revision: object + attempt: object + rows: object + prepared: object + writer: object + resolver: object + io: object + + async def prepare(self, **changes): + args = dict( + key=self.verifier.key, + binding=self.binding, + delivery=self.delivery, + obligation=self.obligation, + revisions=(self.revision,), + attempt=self.attempt, + reader=self.store, + context=io_context(), + ) + args.update(changes) + return await self.registry.prepare(**args) + + +async def materializer_case( + count=1, + *, + capability=None, + consumer="https://buyer.example.test/agents/reporting", + account="acct_a", + store=None, + finality="snapshot", +): + verifier = reference_verifier(capability) + registry = ReportingRevisionVerifierRegistry((verifier,)) + config = replace( + configuration(account), + definition=verifier.key.definition, + report_definition_id=verifier.key.report_definition_id, + required_finality=finality, + ) + store = store or InMemoryReportingReconciliationStore(notifications=True) + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + cap = verifier.key.capability + binding = ReportingDestinationBinding( + config.generation_key, + consumer, + "destination", + "trusted-reference-binding", + cap.method, + cap.transport, + cap.verification_profile, + "delivery_only", + "analytics", + 400, + START, + cap.format, + ("reference-v1",), + "delivered" if cap.method == "warehouse_materialization" else "available", + ) + await store.put_destination_binding(binding) + delivery = ReportingObligationDeliveryRecord( + ReportingDeliveryScope(config.generation_key, consumer, obligation.reporting_obligation_id), + "USD", + END + timedelta(days=400), + END, + ) + await store.bind_obligation_delivery(delivery) + rows = reference_rows(count) + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + "revision-first", + account, + obligation.reporting_obligation_id, + finality, + revision_content_sha256( + reporting_revision_id="revision-first", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + END, + END, + END, + finality_basis="source_final" if finality == "official" else None, + finality_policy_id="reference-final" if finality == "official" else None, + finalized_at=END if finality == "official" else None, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + await store.commit_revision(revision, rows) + attempt = ReportingMaterializationAttempt( + delivery.scope, + revision.reporting_revision_id, + "materialization-first", + 1, + END + timedelta(seconds=1), + ) + await store.commit_materialization_attempt(attempt) + writer = ReferenceReportingDestinationWriter((cap,)) + resolver = ReferenceReportingResolver(writer, registry, (binding,)) + io = ReportingDestinationIO(registry, resolver) + case = Case( + store, + verifier, + registry, + binding, + delivery, + obligation, + revision, + attempt, + rows, + None, + writer, + resolver, + io, + ) + case.prepared = await case.prepare() + return case diff --git a/tests/conformance/reporting/_reliable_support.py b/tests/conformance/reporting/_reliable_support.py index 76abbace0..21c3d4495 100644 --- a/tests/conformance/reporting/_reliable_support.py +++ b/tests/conformance/reporting/_reliable_support.py @@ -718,10 +718,27 @@ async def publication_records( ) await harness.store.put_destination_binding(binding) await harness.store.bind_obligation_delivery(delivery) - rows = await harness.store.read_revision_rows( - account_id=obligation.account_id, reporting_revision_id=revision.reporting_revision_id - ) - payload = b"".join(canonical_json_utf8_v1(row) + b"\n" for row in rows.rows) + all_rows = [] + cursor = None + seen = set() + while True: + page = await harness.store.read_revision_rows( + account_id=obligation.account_id, + reporting_revision_id=revision.reporting_revision_id, + cursor=cursor, + limit=500, + ) + assert page.reporting_revision_id == revision.reporting_revision_id + assert page.total_count == revision.row_count + assert page.has_more == (page.cursor is not None) + all_rows.extend(page.rows) + if not page.has_more: + break + assert page.rows and page.cursor not in seen + seen.add(page.cursor) + cursor = page.cursor + assert len(all_rows) == revision.row_count + payload = b"".join(canonical_json_utf8_v1(row) + b"\n" for row in all_rows) digest = await harness.destination.write( obligation.account_id, revision.reporting_revision_id, payload ) diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py new file mode 100644 index 000000000..7a2f8bbc7 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -0,0 +1,214 @@ +"""Public closed contracts, URL consumers, import safety and real capability gates.""" + +import asyncio +import importlib +import inspect +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from typing import get_args, get_type_hints + +import pytest +from pydantic import TypeAdapter + +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + ProducerOfferings, + ReportingDeliveryPrincipal, + ReportingProducer, +) +from adcp.reporting.ledger.delivery_models import MaterializationFailure +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReportingDestinationPage, + ReportingIOContext, + ReportingWriterError, + ReportingWriterFailure, + ReportingWriterFailureCode, + reference_verifier, +) + +from ._generation_support import UncalledSource +from ._materializer_support import io_context, materializer_case +from ._reliable_support import NotificationHarness, reliable_factory + + +@pytest.mark.parametrize( + "key", + [ + "managed_delivery", + "reconciled_billing", + "reporting.delivery_ready", + "readiness_notification", + "status_notification", + "ledger_notification", + "supports_webhook_activity", + "consumer_status_task", + "receipt_task", + "reconciliation_task", + "status_task", + "delivery_task", + ], +) +def test_producer_extra_cannot_inject_sdk_owned_readiness(key): + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=InMemoryReportingLedgerStore() + ) + with pytest.raises(ValueError, match="SDK-owned"): + producer.advertised_reporting_delivery( + consumer_status_task=False, + offerings=(), + automated_recovery_window=timedelta(hours=1), + status_retention_days=30, + extra={key: True}, + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_real_outbox_capability_path_never_promotes_a_verified_reference_writer(backend): + async with reliable_factory(backend, notifications=True) as h: + case = await materializer_case(store=h.store) + locator = await case.io.write(case.prepared, context=io_context()) + await case.io.verify(case.prepared, locator, context=io_context()) + fields = ( + await NotificationHarness(h) + .worker() + .advertised_notifications(h.store, account_id="acct_a", ready_scope=case.delivery.scope) + ) + assert fields == { + "ledger_notification": "reporting.ledger_changed", + "supports_webhook_activity": False, + } + assert not set(fields).intersection( + { + "managed_delivery", + "reconciled_billing", + "reporting.delivery_ready", + "readiness_notification", + } + ) + assert case.writer.production_eligible is False + + +async def test_url_consumer_round_trips_all_binding_resolver_reconciliation_and_delivery_paths(): + case = await materializer_case() + principal = case.binding.principal + assert "://" in principal.consumer_id + assert ( + TypeAdapter(ReportingDeliveryPrincipal).validate_json( + TypeAdapter(ReportingDeliveryPrincipal).dump_json(principal) + ) + == principal + ) + frozen = await case.store.get_destination_binding( + caller=principal, generation_key=case.binding.generation_key + ) + assert frozen == case.binding + assert case.prepared.request.principal == principal + assert (await case.store.get_obligation_delivery(case.delivery.scope)) == case.delivery + page = await case.store.read_reconciliation_changes(caller=principal) + assert page.caller == principal and page.changes + snapshot = await case.store.read_status_snapshot(account_id=principal.account_id) + assert principal.consumer_id in snapshot.consumer_ids + locator = await case.io.write(case.prepared, context=io_context()) + verified = await case.io.verify(case.prepared, locator, context=io_context()) + assert verified.request.principal == principal + + +def test_new_failures_do_not_change_reviewed_persisted_failure_enum(): + assert get_args(MaterializationFailure) == ( + "WRITE_FAILED", + "VERIFICATION_FAILED", + "CONTENT_CORRUPT", + "RESOURCE_UNAVAILABLE", + ) + assert "CURRENT_REVISION_CHANGED" in get_args(ReportingWriterFailureCode) + assert "CURRENT_REVISION_CHANGED" not in get_args(MaterializationFailure) + for code in get_args(ReportingWriterFailureCode): + assert str(ReportingWriterError(ReportingWriterFailure(code))) == code + for changes in ( + {"code": "provider prose"}, + {"retry": "eventually"}, + {"effect": "probably_written"}, + {"retry_after_seconds": True}, + {"retry_after_seconds": -1}, + ): + with pytest.raises(ValueError): + ReportingWriterFailure(**{"code": "WRITE_FAILED", **changes}) + + +def test_reference_writer_production_flag_is_not_configurable_or_subclass_promotable(): + with pytest.raises(TypeError): + ReferenceReportingDestinationWriter((), production_eligible=True) + writer = ReferenceReportingDestinationWriter(()) + with pytest.raises(AttributeError): + writer.production_eligible = True + with pytest.raises(TypeError): + type( + "PromotedReference", + (ReferenceReportingDestinationWriter,), + {"production_eligible": True}, + ) + + +def test_readback_page_repr_never_includes_unverified_provider_body(): + body = b"https://provider.example.test/private?token=credential-sentinel" + page = ReportingDestinationPage("revision", (body,), 1, False, None, "jsonl", "producer") + assert "credential-sentinel" not in str(page) + repr(page) + assert "https://provider" not in str(page) + repr(page) + + +@pytest.mark.parametrize( + "field", ["report_definition_uri", "schema_uri", "schema_dialect", "schema_version"] +) +def test_public_verification_key_rejects_credentials_in_every_definition_coordinate(field): + verifier = reference_verifier() + with pytest.raises(ValueError) as caught: + replace( + verifier.key, + definition=replace( + verifier.key.definition, + **{field: "https://provider.example.test/data?token=do-not-expose"}, + ), + ) + assert "do-not-expose" not in str(caught.value) + repr(caught.value) + + +async def test_service_heartbeat_is_a_checkpoint_only_and_fences_before_source_read(): + case = await materializer_case() + calls = 0 + + class Heartbeat: + async def checkpoint(self): + nonlocal calls + calls += 1 + raise ReportingWriterError( + ReportingWriterFailure("LEASE_LOST", "same_identity", "not_started") + ) + + context = ReportingIOContext( + datetime.now(timezone.utc) + timedelta(seconds=10), asyncio.Event(), Heartbeat() + ) + with pytest.raises(ReportingWriterError, match="LEASE_LOST"): + await case.prepare(context=context) + assert calls == 1 and case.writer.open_count == 0 + + +def test_curated_all_exports_resolve_and_do_not_add_materializer_sql(): + for name in ( + "adcp.reporting.materializer", + "adcp.reporting.revision_selection", + "adcp.reporting.ledger", + "adcp.reporting.outbox", + ): + module = importlib.import_module(name) + assert len(module.__all__) == len(set(module.__all__)) + for public in module.__all__: + assert getattr(module, public) is not None + module = importlib.import_module("adcp.reporting.materializer") + assert not any( + name.endswith(("Coordinator", "Service", "Lease", "WorkQueue")) for name in module.__all__ + ) + assert inspect.isclass(module.ReportingDestinationSession) + assert get_type_hints(module.ReportingDestinationSession.write)["content"] is ( + module.ReportingPreparedRevision + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_lifecycle.py b/tests/conformance/reporting/test_reporting_materializer_lifecycle.py new file mode 100644 index 000000000..636b19b41 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_lifecycle.py @@ -0,0 +1,373 @@ +"""Adversarial asynchronous ownership: safe errors, partial opens and cancellation.""" + +import asyncio +import pickle +import traceback +from dataclasses import replace + +import pytest + +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingDestinationSession, + ReportingWriterCapability, + ReportingWriterError, + ReportingWriterFailure, +) +from adcp.reporting.materializer.reference import _Session + +from ._materializer_support import io_context, materializer_case + +SECRET = "https://provider.example.test/private?token=credential-sentinel&signature=provider-body" + + +class Phases: + def __init__(self, case, tmp_path, stage, mode): + self.case, self.tmp_path, self.stage, self.mode = case, tmp_path, stage, mode + self.entered = asyncio.Event() + self.release = asyncio.Event() + self.sessions = [] + self.spools = [] + self.stream_closes = 0 + + async def hit(self, stage): + if stage != self.stage: + return + self.entered.set() + if self.mode == "error": + try: + raise RuntimeError(SECRET) + except RuntimeError as exc: + raise ValueError(SECRET) from exc + await self.release.wait() + + def resolve(self, request, *, phase, context): + owner = self + + class Session(_Session): + async def _open(self): + await super()._open() + self._credential = SECRET + self.spool = owner.tmp_path / f"{len(owner.spools)}.spool" + self.spool.write_text(SECRET) + owner.spools.append(self.spool) + await owner.hit("resolve" if self.phase == "write" else "readback-resolve") + + async def _close(self): + try: + await owner.hit("close") + finally: + if hasattr(self, "spool"): + self.spool.unlink(missing_ok=True) + await super()._close() + + async def write(self, content): + await owner.hit("write-before") + locator = await super().write(content) + await owner.hit("write-after") + return locator + + async def read_rows(self, locator, *, cursor, limit): + await owner.hit("rows") + return await super().read_rows(locator, cursor=cursor, limit=limit) + + async def read_manifest(self, locator): + await owner.hit("manifest") + return await super().read_manifest(locator) + + async def observe_native_version(self, locator): + self.native_reads = getattr(self, "native_reads", 0) + 1 + await owner.hit("native-before" if self.native_reads == 1 else "native-after") + return await super().observe_native_version(locator) + + async def list_objects(self, locator): + await owner.hit("inventory") + return await super().list_objects(locator) + + async def read_object(self, locator, *, object_ref): + try: + await owner.hit("object") + async for chunk in super().read_object(locator, object_ref=object_ref): + yield chunk + finally: + owner.stream_closes += 1 + await owner.hit("stream-close") + + session = Session(self.case.resolver, request, phase, context) + self.sessions.append(session) + return session + + +def safe_exception(error): + rendered = ( + str(error) + + repr(error) + + "".join(traceback.format_exception(type(error), error, error.__traceback__)) + ) + assert ( + SECRET not in rendered + and "credential-sentinel" not in rendered + and "provider-body" not in rendered + ) + assert error.__cause__ is None + if isinstance(error, asyncio.CancelledError): + # Python 3.10 Task adds a clean CancelledError context even to an + # unconditionally canceled coroutine. Inspect the entire chain: no + # provider exception or cancellation message may survive that wrapper. + seen = set() + while error is not None: + assert id(error) not in seen + seen.add(id(error)) + assert type(error) is asyncio.CancelledError and error.args == () + assert error.__cause__ is None + error = error.__context__ + else: + assert error.__context__ is None + + +@pytest.mark.parametrize( + "stage", + [ + "resolve", + "readback-resolve", + "write-before", + "write-after", + "rows", + "manifest", + "inventory", + "object", + "close", + "stream-close", + "native-before", + "native-after", + ], +) +@pytest.mark.parametrize("mode", ["cancel", "signal", "timeout", "error"]) +async def test_each_io_phase_closes_once_redacts_errors_and_leaves_no_tasks_or_spools( + tmp_path, caplog, stage, mode +): + capability = ( + ReportingWriterCapability( + "dataset_share", + "reference-memory", + None, + "canonical_digest", + "representative_consumer", + "native_version", + "sha256", + "conditional_create", + ) + if stage.startswith("native") + else None + ) + case = await materializer_case(capability=capability) + locator = await case.io.write(case.prepared, context=io_context()) + owner = Phases(case, tmp_path, stage, mode) + io = ReportingDestinationIO(case.registry, owner) + context = io_context(0.5 if mode == "timeout" else 20) + if stage in {"close", "stream-close"}: + context = replace(context, close_timeout_seconds=0.08 if mode == "timeout" else 5) + before = set(asyncio.all_tasks()) + writing = stage in {"resolve", "write-before", "write-after", "close"} + call = ( + io.write(case.prepared, context=context) + if writing + else io.verify(case.prepared, locator, context=context) + ) + task = asyncio.create_task(call) + await asyncio.wait_for(owner.entered.wait(), 10) + if mode == "cancel": + task.cancel(SECRET) + await asyncio.sleep(0) + task.cancel(SECRET) # Repeated cancellation cannot abandon cleanup. + elif mode == "signal": + context.cancel.set() + if mode in {"cancel", "signal"} and stage in {"close", "stream-close"}: + await asyncio.sleep(0) + owner.release.set() + expected = asyncio.CancelledError if mode in {"cancel", "signal"} else ReportingWriterError + with pytest.raises(expected) as caught: + await asyncio.wait_for(task, 10) + safe_exception(caught.value) + assert all(session._closed and session._credential is None for session in owner.sessions) + assert case.writer.open_count == case.writer.close_count == 2 + assert all(not p.exists() for p in owner.spools) + assert list(tmp_path.iterdir()) == [] + assert not (set(asyncio.all_tasks()) - before) + assert SECRET not in caplog.text + if stage in {"object", "stream-close"}: + assert owner.stream_closes == 1 + for session in owner.sessions: + await session.aclose() # A repeated close is physically idle. + assert case.writer.close_count == 2 + + +@pytest.mark.parametrize("mode", ["cancel", "timeout", "error"]) +async def test_source_read_cancellation_removes_its_spool_before_any_resolver_io(tmp_path, mode): + case = await materializer_case() + owner = Phases(case, tmp_path, "source", mode) + path = tmp_path / "source.spool" + closed = 0 + + class Reader: + async def read_revision_rows(self, **kwargs): + nonlocal closed + path.write_text(SECRET) + try: + await owner.hit("source") + return await case.store.read_revision_rows(**kwargs) + finally: + path.unlink() + closed += 1 + + before = set(asyncio.all_tasks()) + task = asyncio.create_task( + case.prepare(reader=Reader(), context=io_context(0.05 if mode == "timeout" else 20)) + ) + await asyncio.wait_for(owner.entered.wait(), 5) + if mode == "cancel": + task.cancel(SECRET) + with pytest.raises( + asyncio.CancelledError if mode == "cancel" else ReportingWriterError + ) as caught: + await task + safe_exception(caught.value) + assert closed == 1 and not path.exists() and case.writer.open_count == 0 + assert not (set(asyncio.all_tasks()) - before) + + +@pytest.mark.parametrize("phase", ["open", "write"]) +@pytest.mark.parametrize("mode", ["cancel", "signal"]) +async def test_cancellation_during_failed_operation_cleanup_takes_precedence(tmp_path, phase, mode): + case = await materializer_case() + closing, release = asyncio.Event(), asyncio.Event() + spool = tmp_path / "failure.spool" + + class Session(_Session): + async def _open(self): + await super()._open() + spool.write_text(SECRET) + if phase == "open": + raise ValueError(SECRET) + + async def write(self, content): + raise ValueError(SECRET) + + async def _close(self): + closing.set() + try: + await release.wait() + finally: + spool.unlink(missing_ok=True) + await super()._close() + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + before = set(asyncio.all_tasks()) + context = io_context() + task = asyncio.create_task( + ReportingDestinationIO(case.registry, Resolver()).write(case.prepared, context=context) + ) + await asyncio.wait_for(closing.wait(), 5) + if mode == "cancel": + task.cancel(SECRET) + await asyncio.sleep(0) + task.cancel(SECRET) + else: + context.cancel.set() + release.set() + with pytest.raises(asyncio.CancelledError) as caught: + await task + safe_exception(caught.value) + assert not spool.exists() and case.writer.open_count == case.writer.close_count == 1 + assert not (set(asyncio.all_tasks()) - before) + + +async def test_invalid_locator_after_write_retains_unknown_effect_and_original_identity(): + case = await materializer_case() + + class Session(_Session): + async def write(self, content): + locator = await super().write(content) + return replace(locator, external_id="other-tenant-identity") + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + with pytest.raises(ReportingWriterError) as caught: + await ReportingDestinationIO(case.registry, Resolver()).write( + case.prepared, context=io_context() + ) + assert caught.value.failure == ReportingWriterFailure( + "BINDING_MISMATCH", "same_identity", "unknown" + ) + safe_exception(caught.value) + assert case.writer.write_effects == 1 and case.writer.open_count == case.writer.close_count == 1 + + +@pytest.mark.parametrize("cancel", [False, True]) +async def test_resolver_factory_never_exposes_provider_exception_or_cancel_message(cancel): + case = await materializer_case() + + class BrokenResolver: + def resolve(self, *args, **kwargs): + raise asyncio.CancelledError(SECRET) if cancel else ValueError(SECRET) + + with pytest.raises(asyncio.CancelledError if cancel else ReportingWriterError) as caught: + await ReportingDestinationIO(case.registry, BrokenResolver()).write( + case.prepared, context=io_context() + ) + safe_exception(caught.value) + assert case.writer.open_count == 0 + + +async def test_session_binding_is_checked_before_open_and_lifecycle_is_sdk_owned(tmp_path): + case = await materializer_case() + owner = Phases(case, tmp_path, "unused", "error") + + class WrongResolver: + def resolve(self, request, *, phase, context): + return owner.resolve( + replace(request, destination_ref="wrong-destination"), phase=phase, context=context + ) + + with pytest.raises(ReportingWriterError, match="BINDING_MISMATCH"): + await ReportingDestinationIO(case.registry, WrongResolver()).write( + case.prepared, context=io_context() + ) + assert case.writer.open_count == 0 and case.writer.close_count == 1 + session = owner.sessions[0] + assert str(session) == repr(session) == "" + with pytest.raises(TypeError, match="cannot be persisted"): + pickle.dumps(session) + for name in ("request", "phase", "context"): + with pytest.raises(AttributeError): + setattr(session, name, SECRET) + for name in ("__repr__", "__aenter__", "__aexit__", "aclose", "__reduce__"): + with pytest.raises(TypeError, match="belong to the SDK"): + type("UnsafeSession", (ReportingDestinationSession,), {name: lambda *args: SECRET}) + + +async def test_closed_failures_preserve_known_failure_and_unknown_identity_contract(): + async def known(): + raise ReportingWriterError( + ReportingWriterFailure("WRITE_FAILED", "new_attempt", "not_started") + ) + + async def connection_loss(): + raise ConnectionError(SECRET) + + for operation, expected in ( + (known, ("WRITE_FAILED", "new_attempt", "not_started")), + (connection_loss, ("RESOURCE_UNAVAILABLE", "same_identity", "unknown")), + ): + with pytest.raises(ReportingWriterError) as caught: + await io_context().run(operation, effect="unknown") + record = caught.value.failure + assert (record.code, record.retry, record.effect) == expected + safe_exception(caught.value) + with pytest.raises(ValueError): + ReportingWriterFailure("RESOURCE_UNAVAILABLE", "new_attempt", "unknown") diff --git a/tests/conformance/reporting/test_reporting_materializer_packaging.py b/tests/conformance/reporting/test_reporting_materializer_packaging.py new file mode 100644 index 000000000..007a279bf --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_packaging.py @@ -0,0 +1,129 @@ +"""Non-editable VCS/sdist wheels, actual Python 3.10, strict adopter and no PG.""" + +import hashlib +import json +import os +import shutil +import sys +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from . import test_reporting_notification_packaging as distribution + +built_distribution = distribution.built_distribution +ROOT = distribution.ROOT +run_step = distribution.run_step +ASSETS = ROOT / "src/adcp/reporting/materializer/assets" + + +@pytest.fixture(scope="module") +def b1_wheels(built_distribution): + path, sdist_wheel, source = built_distribution + direct = path / "vcs-wheel" + # This build starts in the actual VCS checkout, including its build hook. + run_step( + [sys.executable, "-m", "build", "--wheel", "--outdir", str(direct), str(ROOT)], + label="b1-vcs-wheel", + cwd=ROOT, + timeout=180, + ) + vcs_wheel = next(direct.glob("*.whl")) + expected = {f.name: f.read_bytes() for f in ASSETS.glob("*.json")} + with ( + zipfile.ZipFile(vcs_wheel) as vcs, + zipfile.ZipFile(sdist_wheel) as wheel, + tarfile.open(source) as tar, + ): + prefix = tar.getnames()[0].split("/")[0] + for name, raw in expected.items(): + member = f"adcp/reporting/materializer/assets/{name}" + assert vcs.read(member) == wheel.read(member) == raw + assert tar.extractfile(f"{prefix}/src/{member}").read() == raw + for relative in ( + "ledger/reporting_status_selector_version.sql", + "outbox/required_status_selector_schema.json", + ): + assert ( + vcs.read(f"adcp/reporting/{relative}") + == wheel.read(f"adcp/reporting/{relative}") + == (ROOT / "src/adcp/reporting" / relative).read_bytes() + ) + assert not any( + "reporting_materializer_" in name and name.endswith(".sql") for name in vcs.namelist() + ) + return ( + path, + {"vcs": vcs_wheel, "sdist": sdist_wheel}, + {name: hashlib.sha256(raw).hexdigest() for name, raw in expected.items()}, + ) + + +@pytest.mark.parametrize("kind", ["vcs", "sdist"]) +def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter(b1_wheels, kind): + path, wheels, hashes = b1_wheels + interpreter = os.environ.get("ADCP_PYTHON310") or ( + sys.executable if sys.version_info[:2] == (3, 10) else None + ) + if interpreter is None: + pytest.skip( + "Python 3.10 matrix job runs this gate; ADCP_PYTHON310 enables it on other hosts" + ) + environment = path / f"b1-python310-{kind}" + run_step( + [interpreter, "-m", "venv", str(environment)], + label=f"b1-{kind}-python310-environment", + cwd=path, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, str(wheels[kind]), "mypy==1.20.2"], + label=f"b1-{kind}-noneditable-install", + cwd=path, + timeout=180, + ) + smoke, example, adopter = ( + path / f"{name}-{kind}.py" for name in ("smoke", "example", "adopter") + ) + shutil.copy2(Path(__file__).with_name("_materializer_installed.py"), smoke) + shutil.copy2(ROOT / "examples/reporting_destination_writer.py", example) + shutil.copy2(ROOT / "tests/type_checks/reporting_destination_writer.py", adopter) + result = json.loads( + run_step( + [str(python), "-I", str(smoke)], + label=f"b1-{kind}-isolated-python310-smoke", + cwd=path, + value={"workspace": str(ROOT), "example": str(example), "assets": hashes}, + timeout=120, + ) + ) + assert result == {"python": "3.10", "rows": [0, 501], "installed": True, "assets": hashes} + config = path / "mypy.ini" + config.write_text( + "[mypy]\npython_version = 3.10\nstrict = True\n" + "plugins = adcp.types.mypy_plugin\nfollow_imports = silent\n" + ) + run_step( + [ + str(python), + "-I", + "-m", + "mypy", + "--config-file", + str(config), + "--strict", + "--no-incremental", + str(adopter), + str(example), + ], + label=f"b1-{kind}-installed-adopter-types", + cwd=path, + timeout=120, + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_readers.py b/tests/conformance/reporting/test_reporting_materializer_readers.py new file mode 100644 index 000000000..ddd0c0ff4 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_readers.py @@ -0,0 +1,426 @@ +"""Hostile source/destination pagination, bounded streams and exact contract assets.""" + +import asyncio +import base64 +import hashlib +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.ledger import LedgerConflictError +from adcp.reporting.ledger.store import encode_cursor +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingDestinationPage, + ReportingRevisionVerifierRegistry, + ReportingVerificationLimits, + ReportingWriterCapability, + ReportingWriterError, + reference_verifier, + strict_reporting_json, +) +from adcp.reporting.materializer.reference import _Session + +from ._materializer_support import io_context, materializer_case +from ._reliable_support import reliable_factory + + +@pytest.mark.parametrize( + "field,value", + [ + ("row_count", True), + ("row_count", -1), + ("readable", 1), + ("readable", 0), + ("readable_at_commit", 1), + ("revision_content_sha256", "private-invalid-body"), + ], +) +async def test_foundation_count_flags_and_digest_are_strict_before_source_io(field, value): + case = await materializer_case(1) + + class Reader: + async def read_revision_rows(self, **kwargs): + pytest.fail("invalid frozen metadata must fail before source I/O") + + with pytest.raises(ReportingWriterError, match="SOURCE_INVALID") as caught: + await case.prepare(reader=Reader(), revisions=(replace(case.revision, **{field: value}),)) + assert "private-invalid-body" not in str(caught.value) + assert case.writer.open_count == 0 + + +@pytest.mark.parametrize( + "damage", + [ + "revision", + "cursor-revision", + "cursor-offset", + "cursor-cycle", + "cursor-duplicate-keys", + "cursor-type", + "total-change", + "bool-total", + "unpaired-cursor", + "empty-more", + "truncated", + "extra", + "second-page", + "wrong-page-type", + ], +) +async def test_source_walk_rejects_incomplete_or_substituted_revision_pages_before_destination_io( + damage, +): + case = await materializer_case(501) + calls = 0 + + class Reader: + async def read_revision_rows(self, **kwargs): + nonlocal calls + calls += 1 + page = await case.store.read_revision_rows(**kwargs) + if damage == "revision": + return replace(page, reporting_revision_id="other-revision") + if damage == "cursor-revision" and page.has_more: + return replace( + page, cursor=encode_cursor({"revision": "other-revision", "offset": 500}) + ) + if damage == "cursor-offset" and page.has_more: + return replace( + page, + cursor=encode_cursor( + {"revision": case.revision.reporting_revision_id, "offset": 0} + ), + ) + if damage == "cursor-cycle": + return replace( + page, + rows=(case.rows[0],), + has_more=True, + cursor=encode_cursor( + {"revision": case.revision.reporting_revision_id, "offset": 1} + ), + ) + if damage == "cursor-duplicate-keys": + raw = b'{"revision":"other","revision":"revision-first","offset":500}' + return replace(page, cursor=base64.urlsafe_b64encode(raw).decode()) + if damage == "cursor-type": + return replace(page, cursor=True) + if damage == "total-change" and calls == 2: + return replace(page, total_count=500) + if damage == "bool-total": + return replace(page, total_count=True) + if damage == "unpaired-cursor": + return replace(page, has_more=False) + if damage == "empty-more": + return replace(page, rows=()) + if damage == "truncated": + return replace(page, has_more=False, cursor=None) + if damage == "extra": + return replace(page, rows=(*page.rows, case.rows[0])) + if damage == "second-page" and calls == 2: + return replace(page, rows=({**case.rows[-1], "impressions": True},)) + if damage == "wrong-page-type": + return {"rows": page.rows, "provider": "private-response"} + return page + + with pytest.raises(ReportingWriterError, match="SOURCE_INVALID"): + await case.prepare(reader=Reader()) + assert case.writer.open_count == case.writer.write_effects == 0 + assert calls == (2 if damage in {"cursor-cycle", "total-change", "second-page"} else 1) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_real_store_cursor_binds_revision_and_all_501_rows(backend): + async with reliable_factory(backend, notifications=True) as h: + case = await materializer_case(501, store=h.store) + first = await case.store.read_revision_rows( + account_id="acct_a", reporting_revision_id=case.revision.reporting_revision_id + ) + second = await case.store.read_revision_rows( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + cursor=first.cursor, + ) + assert ( + first.reporting_revision_id + == second.reporting_revision_id + == case.revision.reporting_revision_id + ) + assert len(first.rows) == 500 and len(second.rows) == 1 + assert first.total_count == second.total_count == 501 and not second.has_more + for cursor in ( + encode_cursor({"revision": "another", "offset": 500}), + encode_cursor({"revision": case.revision.reporting_revision_id, "offset": True}), + encode_cursor({"revision": case.revision.reporting_revision_id, "offset": -1}), + ): + with pytest.raises(LedgerConflictError, match="does not bind"): + await case.store.read_revision_rows( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + cursor=cursor, + ) + for limit in (True, 0, 501): + with pytest.raises(LedgerConflictError, match="page size"): + await case.store.read_revision_rows( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + limit=limit, + ) + + +@pytest.mark.parametrize( + "damage", + ["replay", "cycle", "revision", "path", "format", "totals", "wrong-type", "empty", "extra"], +) +async def test_destination_pages_are_verifier_controlled(damage): + case = await materializer_case(3) + locator = await case.io.write(case.prepared, context=io_context()) + requests = [] + + class Session(_Session): + async def read_rows(self, locator, *, cursor, limit): + requests.append(cursor) + index = len(requests) - 1 + next_cursor = f"page-{index}" if index < 2 else None + page = ReportingDestinationPage( + case.revision.reporting_revision_id, + (case.prepared.rows[index],), + 3, + next_cursor is not None, + next_cursor, + "jsonl", + "producer", + ) + if damage == "replay": + return replace(page, rows=(case.prepared.rows[0],)) + if damage == "cycle": + return replace(page, has_more=True, cursor="repeat") + if damage == "revision": + return replace(page, reporting_revision_id="other") + if damage == "path": + return replace(page, verification_path="destination") + if damage == "format": + return replace(page, format="csv") + if damage == "totals": + return replace(page, total_count=2) + if damage == "wrong-type": + return object() + if damage == "empty": + return replace(page, rows=()) + if damage == "extra": + return replace(page, rows=tuple(case.prepared.rows) + (case.prepared.rows[0],)) + return page + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + with pytest.raises(ReportingWriterError, match="DESTINATION_CORRUPT"): + await ReportingDestinationIO(case.registry, Resolver()).verify( + case.prepared, locator, context=io_context() + ) + assert len(requests) <= 2 and case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize( + "bound", ["max_rows", "max_pages", "max_total_bytes", "max_items", "max_chunks", "max_objects"] +) +async def test_walks_and_repeating_streams_have_finite_budgets(bound): + case = await materializer_case(501 if bound in {"max_pages", "max_objects"} else 100) + locator = await case.io.write(case.prepared, context=io_context()) + limits = replace( + ReportingVerificationLimits(), + **{bound: 1 if bound in {"max_rows", "max_pages", "max_chunks", "max_objects"} else 1000}, + ) + # Registry assets have their own bounded construction. Item/byte budgets + # deliberately exceed the vectors but not the full source/destination. + if bound in {"max_rows", "max_total_bytes", "max_items"}: + limits = replace(limits, max_rows=2 if bound == "max_rows" else 100_000) + case = await materializer_case(100) + locator = await case.io.write(case.prepared, context=io_context()) + verifier = replace(case.verifier, limits=limits) + registry = ReportingRevisionVerifierRegistry((verifier,)) + if bound in {"max_rows", "max_pages", "max_total_bytes", "max_items"}: + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + await registry.prepare( + key=verifier.key, + binding=case.binding, + delivery=case.delivery, + obligation=case.obligation, + revisions=(case.revision,), + attempt=case.attempt, + reader=case.store, + context=io_context(), + ) + else: + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED|DESTINATION_CORRUPT"): + await ReportingDestinationIO(registry, case.resolver).verify( + case.prepared, locator, context=io_context() + ) + + +async def test_infinite_object_stream_is_bounded_and_closed_without_background_tasks(): + case = await materializer_case() + locator = await case.io.write(case.prepared, context=io_context()) + verifier = replace(case.verifier, limits=ReportingVerificationLimits(max_chunks=3)) + registry = ReportingRevisionVerifierRegistry((verifier,)) + chunks, closed = 0, 0 + + class Session(_Session): + async def read_object(self, locator, *, object_ref): + nonlocal chunks, closed + try: + while True: + chunks += 1 + yield b" " + finally: + closed += 1 + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + before = set(asyncio.all_tasks()) + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + await ReportingDestinationIO(registry, Resolver()).verify( + case.prepared, locator, context=io_context() + ) + assert chunks == 4 and closed == 1 + assert not (set(asyncio.all_tasks()) - before) + assert case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize("damage", ["page-version", "final-version", "final-path"]) +async def test_native_identity_is_pinned_across_pages_and_after_readback(damage): + capability = ReportingWriterCapability( + "dataset_share", + "reference-memory", + None, + "canonical_digest", + "representative_consumer", + "native_version", + "sha256", + "conditional_create", + ) + case = await materializer_case(501, capability=capability) + locator = await case.io.write(case.prepared, context=io_context()) + observations = 0 + + class Session(_Session): + async def read_rows(self, locator, *, cursor, limit): + result = await super().read_rows(locator, cursor=cursor, limit=limit) + if cursor is not None and damage == "page-version": + return replace(result, native_version_ref="different-version") + return result + + async def observe_native_version(self, locator): + nonlocal observations + observations += 1 + result = await super().observe_native_version(locator) + if observations == 2: + return replace( + result, + **{ + "native_version_ref" if damage == "final-version" else "location": "changed" + }, + ) + return result + + class Resolver: + def resolve(self, request, *, phase, context): + return Session(case.resolver, request, phase, context) + + with pytest.raises(ReportingWriterError, match="DESTINATION_CORRUPT"): + await ReportingDestinationIO(case.registry, Resolver()).verify( + case.prepared, locator, context=io_context() + ) + assert observations == (1 if damage == "page-version" else 2) + assert case.writer.open_count == case.writer.close_count == 2 + + +@pytest.mark.parametrize( + "mutation", + [ + "remote-ref", + "nested-id", + "missing-total", + "extra-total", + "expression", + "golden-digest", + "golden-bytes", + "golden-member-order", + "golden-row-order", + "golden-duplicate-name", + "canonical-algorithm", + ], +) +def test_installed_contracts_are_executable_complete_closed_and_network_free(mutation, monkeypatch): + verifier = reference_verifier() + definition, schema, contract = ( + json.loads(raw) + for raw in ( + verifier.definition_bytes, + verifier.schema_bytes, + verifier.canonicalization_bytes, + ) + ) + if mutation == "remote-ref": + schema["properties"]["details"] = {"$ref": "https://never-fetch.example.test/private"} + elif mutation == "nested-id": + schema["properties"]["details"] = {"$id": "https://never-fetch.example.test/private"} + elif mutation == "missing-total": + del schema["properties"]["spend"]["x-adcp-control-total"] + elif mutation == "extra-total": + schema["properties"]["extra"] = { + "type": "integer", + "x-adcp-control-total": {"value_type": "integer"}, + } + elif mutation == "expression": + definition["metrics"][0]["source_expression"] = "impressions * 2" + elif mutation == "golden-digest": + contract["golden_vectors"]["ordering_encoding"]["sha256"] = "0" * 64 + elif mutation == "golden-bytes": + contract["golden_vectors"]["ordering_encoding"]["canonical_utf8_base64"] = "e30=" + elif mutation == "golden-member-order": + contract["golden_vectors"]["ordering_encoding"]["input_rows"] = json.loads( + strict_reporting_json(contract["golden_vectors"]["ordering_encoding"]["input_rows"]) + ) + elif mutation == "golden-row-order": + contract["golden_vectors"]["ordering_encoding"]["input_rows"].reverse() + elif mutation == "golden-duplicate-name": + contract["golden_vectors"]["ordering_encoding"]["name"] = contract["golden_vectors"][ + "empty_report" + ]["name"] + else: + contract["algorithm"] = "provider-assertions-v1" + raw_schema = json.dumps(schema).encode() + contract["schema_sha256"] = hashlib.sha256(raw_schema).hexdigest() + raw_definition, raw_contract = json.dumps(definition).encode(), json.dumps(contract).encode() + key = replace( + verifier.key, + definition=replace( + verifier.key.definition, + report_definition_sha256=hashlib.sha256(raw_definition).hexdigest(), + schema_sha256=hashlib.sha256(raw_schema).hexdigest(), + ), + canonicalization=replace( + verifier.key.canonicalization, + canonicalization_sha256=hashlib.sha256(raw_contract).hexdigest(), + ), + ) + import socket + + def network(*args, **kwargs): + pytest.fail("canonicalizer attempted network fallback") + + monkeypatch.setattr(socket, "create_connection", network) + with pytest.raises(ReportingWriterError, match="UNSUPPORTED_VERIFICATION"): + replace( + verifier, + key=key, + schema_bytes=raw_schema, + definition_bytes=raw_definition, + canonicalization_bytes=raw_contract, + ) diff --git a/tests/conformance/reporting/test_reporting_materializer_verification.py b/tests/conformance/reporting/test_reporting_materializer_verification.py new file mode 100644 index 000000000..e82a885d1 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_materializer_verification.py @@ -0,0 +1,600 @@ +"""Actual bytes, whole walks, typed totals and exact immutable destination paths.""" + +import hashlib +import json +from dataclasses import FrozenInstanceError, replace +from datetime import datetime, timedelta +from decimal import Decimal +from itertools import product + +import pytest + +from adcp.reporting.ledger import ReportingDeliveryPrincipal, revision_content_sha256 +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationIO, + ReportingRevisionVerifierRegistry, + ReportingVerificationLimits, + ReportingWriterCapability, + ReportingWriterError, + parse_reporting_json, + reference_verifier, + strict_reporting_json, + validate_materialization_target, +) +from adcp.reporting.outbox import InMemoryReportingOutbox + +from ._materializer_support import io_context, materializer_case + +CAPABILITIES = [ + ReportingWriterCapability( + method, "reference-memory", fmt, profile, path, immutable, "sha256", "conditional_create" + ) + for method, fmt, path, immutable, profiles in ( + ( + "file_transfer", + "jsonl", + "producer", + "immutable_location", + ("canonical_digest", "manifest_checksums"), + ), + ( + "dataset_share", + None, + "representative_consumer", + "native_version", + ("canonical_digest", "native_commit"), + ), + ( + "warehouse_materialization", + None, + "destination", + "native_version", + ("canonical_digest", "native_commit"), + ), + ) + for profile in profiles +] + + +@pytest.mark.parametrize("count,capability", tuple(product((0, 1, 501), CAPABILITIES))) +async def test_entire_source_and_destination_walk_in_every_supported_profile(count, capability): + case = await materializer_case(count, capability=capability) + locator = await case.io.write(case.prepared, context=io_context()) + case.resolver.rotate() + result = await case.io.verify(case.prepared, locator, context=io_context()) + assert result.verification.row_count == count + assert result.verification.control_totals == case.revision.managed_control_totals + assert result.request.principal == ReportingDeliveryPrincipal( + "acct_a", "https://buyer.example.test/agents/reporting" + ) + assert result.verification.verification_path == capability.verification_path + assert await case.io.write(case.prepared, context=io_context()) == locator + assert case.writer.write_effects == 1 and case.writer.open_count == case.writer.close_count == 3 + assert case.writer.production_eligible is False + assert (await case.store.get_materialization(case.attempt.key)).outcome is None + assert len(await InMemoryReportingOutbox(case.store).list_events(account_id="acct_a")) >= 1 + assert all( + e.notification_type != "reporting.delivery_ready" + for e in await InMemoryReportingOutbox(case.store).list_events(account_id="acct_a") + ) + + +@pytest.mark.parametrize( + "damage", + [ + "rows", + "row-order", + "row-missing", + "row-extra", + "object", + "object-utf8", + "object-missing", + "object-extra", + "object-order", + "manifest", + "manifest-schema", + "manifest-count", + "manifest-total-type", + "manifest-total-unit", + "manifest-total-missing", + "manifest-period", + "manifest-object-checksum", + "manifest-object-count", + "native-version", + "native-path", + ], +) +async def test_writer_locators_cannot_prove_corrupt_or_changed_destination(damage): + cap = CAPABILITIES[2] if damage.startswith("native") else CAPABILITIES[0] + case = await materializer_case(501, capability=cap) + locator = await case.io.write(case.prepared, context=io_context()) + artifact = case.writer._artifacts[locator.external_id] + if damage == "rows": + row = json.loads(artifact.rows[0]) + row["details"]["active"] = 1 + artifact = replace(artifact, rows=(strict_reporting_json(row), *artifact.rows[1:])) + elif damage == "row-order": + artifact = replace(artifact, rows=artifact.rows[::-1]) + elif damage == "row-missing": + artifact = replace(artifact, rows=artifact.rows[:-1]) + elif damage == "row-extra": + artifact = replace(artifact, rows=(*artifact.rows, artifact.rows[0])) + elif damage in {"object", "object-utf8"}: + artifact = replace( + artifact, + objects=( + (artifact.objects[0][0], b"\xff\n" if damage == "object-utf8" else b"{}\n"), + *artifact.objects[1:], + ), + ) + elif damage == "object-missing": + artifact = replace(artifact, objects=artifact.objects[:-1]) + elif damage == "object-extra": + artifact = replace(artifact, objects=(*artifact.objects, ("extra.jsonl", b"{}\n"))) + elif damage == "object-order": + artifact = replace(artifact, objects=artifact.objects[::-1]) + elif damage.startswith("manifest"): + manifest = json.loads(artifact.manifest) + if damage == "manifest": + artifact = replace(artifact, manifest=artifact.manifest + b" ") + else: + if damage == "manifest-schema": + manifest["unexpected"] = "provider prose" + if damage == "manifest-count": + manifest["row_count"] = True + if damage == "manifest-total-type": + manifest["control_totals"][0]["value_type"] = "decimal" + if damage == "manifest-total-unit": + manifest["control_totals"][1]["unit"] = "EUR" + if damage == "manifest-total-missing": + manifest["control_totals"].pop() + if damage == "manifest-period": + manifest["period"]["end"] = manifest["period"]["start"] + if damage == "manifest-object-checksum": + manifest["files"][0]["sha256"] = "0" * 64 + if damage == "manifest-object-count": + manifest["files"][0]["row_count"] = 249 + raw = strict_reporting_json(manifest) + locator = replace( + locator, + resource=replace(locator.resource, manifest_sha256=hashlib.sha256(raw).hexdigest()), + ) + artifact = replace(artifact, manifest=raw) + elif damage == "native-version": + artifact = replace( + artifact, + locator=replace( + locator, resource=replace(locator.resource, native_version_ref="changed-version") + ), + ) + elif damage == "native-path": + artifact = replace( + artifact, + locator=replace( + locator, resource=replace(locator.resource, location="different/table") + ), + ) + case.writer._artifacts[locator.external_id] = artifact + with pytest.raises(ReportingWriterError): + await case.io.verify(case.prepared, locator, context=io_context()) + assert case.writer.open_count == case.writer.close_count == 2 + assert (await case.store.get_materialization(case.attempt.key)).outcome is None + + +@pytest.mark.parametrize( + "value", + [ + 1.0, + float("nan"), + float("inf"), + Decimal("1"), + datetime(2026, 1, 1), + b"abc", + {1}, + (1,), + {1: "value"}, + 2**53, + "\ud800", + ], +) +def test_exact_recursive_json_rejects_non_json_values(value): + with pytest.raises(ReportingWriterError): + strict_reporting_json({"nested": [value]}) + + +@pytest.mark.parametrize("base,value", [(int, 1), (str, "x"), (dict, {}), (list, [])]) +def test_json_subclasses_are_never_coerced(base, value): + subclass = type("NotExact", (base,), {}) + with pytest.raises(ReportingWriterError): + strict_reporting_json({"value": subclass(value)}) + + +@pytest.mark.parametrize( + "raw", + [ + b'{"a":1,"a":2}', + b'{"a":{"x":true,"x":1}}', + b"NaN", + b"Infinity", + b"-Infinity", + b"1.0", + b'"\xff"', + b'"\\udfff"', + ], +) +def test_parser_rejects_ambiguous_or_invalid_bytes(raw): + with pytest.raises(ReportingWriterError) as caught: + parse_reporting_json(raw) + assert caught.value.__context__ is None and caught.value.__cause__ is None + + +def test_type_distinction_unicode_and_bounded_json(): + assert strict_reporting_json(True) != strict_reporting_json(1) + assert strict_reporting_json("é") != strict_reporting_json("e\u0301") + assert strict_reporting_json({"\ue000": 1, "😀": 2}) == '{"😀":2,"\ue000":1}'.encode() + for value, limits in ( + ([[[[]]]], ReportingVerificationLimits(max_depth=2)), + ({"large": "x" * 30}, ReportingVerificationLimits(max_value_bytes=20)), + ([1, 2, 3], ReportingVerificationLimits(max_items=2)), + ): + with pytest.raises(ReportingWriterError): + strict_reporting_json(value, limits) + with pytest.raises(ReportingWriterError): + parse_reporting_json(b"[" * 1000 + b"]" * 1000) + + +@pytest.mark.parametrize("value", ['😀é\x00\n\\"', [True, False, None, -1], {"nested": [1, 2]}]) +def test_exact_utf8_and_json_escape_budget(value): + encoded = strict_reporting_json(value) + assert ( + strict_reporting_json(value, ReportingVerificationLimits(max_value_bytes=len(encoded))) + == encoded + ) + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + strict_reporting_json(value, ReportingVerificationLimits(max_value_bytes=len(encoded) - 1)) + + +def test_aggregate_byte_budget_rejects_before_allocating_canonical_output(monkeypatch): + import adcp.reporting.materializer._json as boundary + + def forbidden(value): + pytest.fail("oversized values must fail before canonical output allocation") + + monkeypatch.setattr(boundary, "canonical_json_utf8_v1", forbidden) + with pytest.raises(ReportingWriterError, match="LIMIT_EXCEEDED"): + boundary.strict_reporting_json( + ["x" * 1000] * 1000, ReportingVerificationLimits(max_value_bytes=2000) + ) + + +@pytest.mark.parametrize( + "part", + [ + "definition_id", + "profile", + "definition_uri", + "definition_hash", + "schema_version", + "schema_uri", + "schema_hash", + "dialect", + "ref_policy", + "canonical_id", + "canonical_uri", + "canonical_hash", + "method", + "transport", + "format", + "verification_profile", + "path", + "immutability", + "write_semantics", + ], +) +async def test_registry_key_is_the_complete_frozen_tuple(part): + case = await materializer_case() + key = case.verifier.key + if part in {"definition_id", "profile"}: + key = replace( + key, + **{ + ( + "report_definition_id" if part == "definition_id" else "reporting_profile" + ): "different" + }, + ) + elif part.startswith("canonical"): + field = { + "canonical_id": "canonicalization_id", + "canonical_uri": "canonicalization_uri", + "canonical_hash": "canonicalization_sha256", + }[part] + key = replace( + key, + canonicalization=replace( + key.canonicalization, + **{ + field: ( + "0" * 64 + if part.endswith("hash") + else ( + "https://different.example.test/contract" + if part.endswith("uri") + else "different" + ) + ) + }, + ), + ) + elif part in { + "method", + "transport", + "format", + "verification_profile", + "path", + "immutability", + "write_semantics", + }: + changes = { + "method": "warehouse_materialization", + "transport": "other", + "format": "csv", + "verification_profile": "manifest_checksums", + "verification_path": "destination", + "immutability": "native_version", + "write_semantics": "idempotent", + } + field = "verification_path" if part == "path" else part + values = {field: changes[field]} + if part in {"method", "immutability"}: + values["verification_path"] = "destination" + key = replace(key, capability=replace(key.capability, **values)) + else: + field = { + "definition_uri": "report_definition_uri", + "definition_hash": "report_definition_sha256", + "schema_hash": "schema_sha256", + "dialect": "schema_dialect", + "ref_policy": "schema_ref_policy", + }.get(part, part) + value = ( + "0" * 64 + if part.endswith("hash") + else ( + "https://different.example.test/schema" + if part.endswith("uri") or part == "dialect" + else "other" + ) + ) + key = replace(key, definition=replace(key.definition, **{field: value})) + with pytest.raises(ReportingWriterError, match="UNSUPPORTED_VERIFICATION"): + await case.prepare(key=key) + assert case.writer.open_count == 0 + + +async def test_uppercase_digest_evidence_is_semantically_equal_and_output_is_lowercase(): + case = await materializer_case() + revision = replace( + case.revision, + canonical_content_digest=replace( + case.revision.canonical_content_digest, + value=case.revision.canonical_content_digest.value.upper(), + canonicalization_sha256=case.revision.canonical_content_digest.canonicalization_sha256.upper(), + ), + ) + prepared = await case.prepare(revisions=(revision,)) + locator = await case.io.write(prepared, context=io_context()) + verified = await case.io.verify(prepared, locator, context=io_context()) + assert ( + verified.verification.canonical_content_digest.value + == case.revision.canonical_content_digest.value + ) + + +@pytest.mark.parametrize("claim", ["canonical", "core", "total-value", "total-missing"]) +async def test_validly_shaped_ledger_claims_are_recomputed_before_resolver_io(claim): + case = await materializer_case(501) + if claim == "canonical": + revision = replace( + case.revision, + canonical_content_digest=replace( + case.revision.canonical_content_digest, value="a" * 64 + ), + ) + elif claim == "core": + revision = replace(case.revision, revision_content_sha256="b" * 64) + else: + totals = case.revision.managed_control_totals + totals = ( + (replace(totals[0], value="0"), *totals[1:]) if claim == "total-value" else totals[:-1] + ) + pairs = tuple((total.name, total.value) for total in totals) + revision = replace( + case.revision, + managed_control_totals=totals, + control_totals=pairs, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=case.revision.reporting_revision_id, + row_count=case.revision.row_count, + control_totals=pairs, + reporting_rows=case.rows, + control_total_evidence=totals, + ), + ) + with pytest.raises(ReportingWriterError, match="SOURCE_INVALID"): + await case.prepare(revisions=(revision,)) + assert case.writer.open_count == case.writer.write_effects == 0 + + +async def test_rotation_revocation_is_checked_again_before_readback(): + case = await materializer_case() + locator = await case.io.write(case.prepared, context=io_context()) + case.resolver.revoke(case.binding.principal) + with pytest.raises(ReportingWriterError, match="AUTHORIZATION_DENIED"): + await case.io.verify(case.prepared, locator, context=io_context()) + assert case.writer.open_count == case.writer.close_count == 2 + + +async def test_binding_and_tenant_collisions_do_not_share_effect_identity_or_resolution(): + first, second = await materializer_case(account="acct_a"), await materializer_case( + account="acct_b" + ) + assert first.prepared.request.external_id != second.prepared.request.external_id + wrong = ReportingDestinationIO( + first.registry, ReferenceReportingResolver(first.writer, first.registry, (second.binding,)) + ) + with pytest.raises(ReportingWriterError, match="AUTHORIZATION_DENIED"): + await wrong.write(first.prepared, context=io_context()) + assert first.writer.write_effects == 0 + revised = replace(first.prepared.request, reporting_revision_id="different") + assert revised.external_id != first.prepared.request.external_id # Equal attempt 1 is isolated. + + +@pytest.mark.parametrize("coordinate", ["account", "consumer"]) +async def test_colliding_public_ids_in_one_destination_never_cross_principals(coordinate): + first = await materializer_case() + second = await materializer_case( + **{ + coordinate: ( + "acct_b" if coordinate == "account" else "https://another.example.test/agent" + ) + } + ) + resolver = ReferenceReportingResolver( + first.writer, first.registry, (first.binding, second.binding) + ) + io = ReportingDestinationIO(first.registry, resolver) + left = await io.write(first.prepared, context=io_context()) + right = await io.write(second.prepared, context=io_context()) + assert ( + left.external_id != right.external_id and left.resource.location != right.resource.location + ) + assert first.writer.write_effects == 2 + with pytest.raises(ReportingWriterError, match="BINDING_MISMATCH"): + await io.verify(first.prepared, right, context=io_context()) + assert first.writer.open_count == first.writer.close_count == 2 + for case, locator in ((first, left), (second, right)): + result = await io.verify(case.prepared, locator, context=io_context()) + assert result.request.principal == case.binding.principal + + +async def test_registry_and_prepared_bytes_are_immutable_and_target_reselection_fails_closed(): + case = await materializer_case() + with pytest.raises(FrozenInstanceError): + case.registry.verifiers = () + assert all(type(r) is bytes for r in case.prepared.rows) + official = replace( + case.revision, + reporting_revision_id="official", + finality="official", + finality_basis="source_final", + finality_policy_id="reference-final", + finalized_at=case.revision.created_at, + ) + with pytest.raises(ReportingWriterError, match="CURRENT_REVISION_CHANGED"): + validate_materialization_target( + case.prepared, binding=case.binding, revisions=(case.revision, official) + ) + with pytest.raises(ReportingWriterError, match="HISTORY_CORRUPT"): + validate_materialization_target( + case.prepared, binding=case.binding, revisions=(case.revision, case.revision) + ) + assert case.writer.write_effects == 0 + + +@pytest.mark.parametrize( + "path", + [ + "../private", + "/private", + "a//private", + "a/./private", + "a/%2e%2e/private", + "a\\private", + "C:/private", + "C:private", + ], +) +async def test_object_locator_traversal_fails_before_readback_authorization(path): + case = await materializer_case() + locator = await case.io.write(case.prepared, context=io_context()) + with pytest.raises((ReportingWriterError, ValueError)): + locator = replace(locator, resource=replace(locator.resource, object_refs=(path,))) + await case.io.verify(case.prepared, locator, context=io_context()) + assert case.writer.open_count == case.writer.close_count == 1 + + +@pytest.mark.parametrize("count", [0, 501]) +async def test_snapshot_to_official_retains_both_histories_and_separate_attempt_one_identity(count): + case = await materializer_case(count) + snapshot_locator = await case.io.write(case.prepared, context=io_context()) + official = replace( + case.revision, + reporting_revision_id="revision-official", + finality="official", + finality_basis="source_final", + finality_policy_id="reference-final", + finalized_at=case.revision.created_at, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="revision-official", + row_count=count, + control_totals=case.revision.control_totals, + reporting_rows=case.rows, + control_total_evidence=case.revision.managed_control_totals, + ), + ) + await case.store.commit_revision(official, case.rows) + history = await case.store.list_revisions( + account_id=case.obligation.account_id, + reporting_obligation_id=case.obligation.reporting_obligation_id, + ) + assert len(history) == 2 and official.supersedes_reporting_revision_id is None + with pytest.raises(ReportingWriterError, match="CURRENT_REVISION_CHANGED"): + validate_materialization_target(case.prepared, binding=case.binding, revisions=history) + attempt = replace( + case.attempt, + reporting_revision_id=official.reporting_revision_id, + reporting_materialization_id="materialization-official", + created_at=case.attempt.created_at + timedelta(seconds=1), + ) + await case.store.commit_materialization_attempt(attempt) + prepared = await case.prepare(revisions=history, attempt=attempt) + locator = await case.io.write(prepared, context=io_context()) + verified = await case.io.verify(prepared, locator, context=io_context()) + assert verified.request.reporting_revision_id == official.reporting_revision_id + assert attempt.attempt == case.attempt.attempt == 1 + assert locator.external_id != snapshot_locator.external_id + assert locator.resource.location != snapshot_locator.resource.location + assert case.writer.write_effects == 2 + assert (await case.store.get_materialization(case.attempt.key)).outcome is None + assert (await case.store.get_materialization(attempt.key)).outcome is None + + +async def test_official_required_preparation_never_falls_back_to_a_snapshot(): + case = await materializer_case(finality="official") + snapshot = replace( + case.revision, + reporting_revision_id="snapshot-only", + finality="snapshot", + finality_basis=None, + finality_policy_id=None, + finalized_at=None, + ) + with pytest.raises(ReportingWriterError, match="REVISION_NOT_READY"): + await case.prepare(revisions=(snapshot,)) + assert case.writer.open_count == 0 + locator = await case.io.write(case.prepared, context=io_context()) + result = await case.io.verify(case.prepared, locator, context=io_context()) + assert result.request.reporting_revision_id == case.revision.reporting_revision_id + + +def test_reference_writer_has_no_production_config_override(): + writer = ReferenceReportingDestinationWriter(()) + with pytest.raises(AttributeError): + writer.production_eligible = True + with pytest.raises(TypeError): + ReferenceReportingDestinationWriter((), production_eligible=True) + with pytest.raises(ReportingWriterError): + ReportingRevisionVerifierRegistry((reference_verifier(), reference_verifier())) diff --git a/tests/conformance/reporting/test_reporting_notification_packaging.py b/tests/conformance/reporting/test_reporting_notification_packaging.py index ff56c5260..e902fb342 100644 --- a/tests/conformance/reporting/test_reporting_notification_packaging.py +++ b/tests/conformance/reporting/test_reporting_notification_packaging.py @@ -228,6 +228,8 @@ def installed_distribution(built_distribution, request): assert files("adcp.reporting.outbox").joinpath("required_schema.json").is_file() assert files("adcp.reporting.outbox").joinpath("required_status_schema.json").is_file() assert files("adcp.reporting.ledger").joinpath("reporting_status_notifications.sql").is_file() +assert files("adcp.reporting.ledger").joinpath("reporting_status_selector_version.sql").is_file() +assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() assert get_named_validator("core/reporting-status-changed-webhook.json", version="3.2.0-rc.3") is not None assert get_named_validator("core/webhook-activity-record.json", version="3.2.0-rc.3") is not None assert ( @@ -251,6 +253,7 @@ def test_wheel_and_sdist_contain_exact_complete_sql_chain(built_distribution): *CHAIN, "reporting_webhook_activity.sql", "reporting_status_notifications.sql", + "reporting_status_selector_version.sql", ): expected = (ROOT / "src" / "adcp" / "reporting" / "ledger" / name).read_bytes() assert archive.read(f"adcp/reporting/ledger/{name}") == expected diff --git a/tests/conformance/reporting/test_reporting_notification_readiness.py b/tests/conformance/reporting/test_reporting_notification_readiness.py index ce88a85cd..e51bbe772 100644 --- a/tests/conformance/reporting/test_reporting_notification_readiness.py +++ b/tests/conformance/reporting/test_reporting_notification_readiness.py @@ -39,7 +39,7 @@ async def test_capability_fragment_exposes_only_complete_notifications(notificat ) -async def test_managed_capability_requires_retained_configuration_and_frozen_scope( +async def test_frozen_binding_alone_never_advertises_managed_readiness( notification_harness, ): h = notification_harness @@ -49,7 +49,6 @@ async def test_managed_capability_requires_retained_configuration_and_frozen_sco ) assert fields == { "ledger_notification": "reporting.ledger_changed", - "readiness_notification": "reporting.delivery_ready", "supports_webhook_activity": False, } with pytest.raises(ReportingNotificationError): diff --git a/tests/conformance/reporting/test_reporting_revision_selection.py b/tests/conformance/reporting/test_reporting_revision_selection.py new file mode 100644 index 000000000..36bfc057e --- /dev/null +++ b/tests/conformance/reporting/test_reporting_revision_selection.py @@ -0,0 +1,236 @@ +"""Whole-history selection is identical for Core, C, ingest and B1 preparation.""" + +from dataclasses import replace +from itertools import permutations + +import pytest + +from adcp.reporting.ledger import ( + InMemoryReportingLedgerStore, + LedgerConflictError, + ProducerOfferings, + ReportingProducer, + current_required_revision, + project_obligation_health, + select_reporting_revision, +) +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.reporting.ledger.status_projection import ( + ReportingStatusSnapshot, + StatusProjectionInput, + lifecycle_intents, + project_status_scope, +) +from adcp.reporting.ledger.status_snapshot import validate_status_evidence +from adcp.reporting.outbox import ReportingStatusScope +from adcp.reporting.revision_selection import RevisionHistoryEntry + +from ._generation_support import NOW, UncalledSource, configuration, obligation_for, revision_for +from .test_reporting_notification_outbox import statement + + +def entry(name, predecessor=None, finality="snapshot", account="acct_a", obligation="rpo_acct_a"): + return RevisionHistoryEntry(account, obligation, name, finality, predecessor) + + +VECTORS = ( + ((), "snapshot", "not_ready", "empty_history"), + ((), "official", "not_ready", "empty_history"), + ((entry("a"),), "official", "not_ready", "official_required"), + ((entry("a"),), "snapshot", "selected", "a"), + ((entry("a"), entry("b", "a")), "snapshot", "selected", "b"), + ( + (entry("a"), entry("b", "a"), entry("official", finality="official")), + "snapshot", + "selected", + "official", + ), + ((entry("official", finality="official"),), "official", "selected", "official"), + ((entry("a"), entry("a")), "snapshot", "corrupt", "duplicate_revision_id"), + ((entry("a", account="acct_b"),), "snapshot", "corrupt", "ownership_mismatch"), + ((entry("a", obligation="rpo_other"),), "snapshot", "corrupt", "ownership_mismatch"), + ((entry(""),), "snapshot", "corrupt", "invalid_revision_identity"), + ((entry("a", finality="draft"),), "snapshot", "corrupt", "invalid_finality"), + ((entry("a", "absent"),), "official", "corrupt", "missing_predecessor"), + ( + (entry("a", "o"), entry("o", finality="official")), + "official", + "corrupt", + "cross_finality_edge", + ), + ((entry("a"), entry("o", "a", "official")), "official", "corrupt", "cross_finality_edge"), + ( + (entry("o", finality="official"), entry("p", "o", "official")), + "official", + "corrupt", + "official_predecessor", + ), + ( + (entry("a"), entry("b", "a"), entry("c", "a")), + "official", + "corrupt", + "forked_snapshot_history", + ), + ((entry("a"), entry("b")), "official", "corrupt", "disconnected_snapshot_history"), + ((entry("a", "a"),), "snapshot", "corrupt", "revision_cycle"), + ((entry("a", "b"), entry("b", "a")), "snapshot", "corrupt", "revision_cycle"), + ( + (entry("unique-leaf"), entry("a", "b"), entry("b", "a")), + "snapshot", + "corrupt", + "revision_cycle", + ), + ( + (entry("a", finality="official"), entry("b", finality="official")), + "snapshot", + "corrupt", + "multiple_officials", + ), +) + + +@pytest.mark.parametrize("history,required,kind,detail", VECTORS) +def test_complete_history_in_any_input_order(history, required, kind, detail): + for rows in permutations(history): + result = select_reporting_revision( + rows, + account_id="acct_a", + reporting_obligation_id="rpo_acct_a", + required_finality=required, + ) + assert result.kind == kind + assert ( + result.revision.reporting_revision_id if kind == "selected" else result.reason + ) == detail + if kind == "corrupt" and not any(r.finality == "official" for r in history): + # A unique official never excuses a damaged retained snapshot history. + result = select_reporting_revision( + (*history, entry("official", finality="official")), + account_id="acct_a", + reporting_obligation_id="rpo_acct_a", + required_finality="official", + ) + assert result.kind == "corrupt" + + +@pytest.mark.parametrize( + "field,value", + [ + ("account_id", []), + ("reporting_obligation_id", {}), + ("reporting_revision_id", []), + ("supersedes_reporting_revision_id", []), + ("finality", {}), + ], +) +def test_damaged_identity_types_return_corrupt_without_hashing_them(field, value): + result = select_reporting_revision( + (replace(entry("a"), **{field: value}),), + account_id="acct_a", + reporting_obligation_id="rpo_acct_a", + required_finality="official", + ) + assert result.kind == "corrupt" + + +def records(history, obligation): + revision, _ = revision_for(obligation) + result = [] + for r in history: + record = replace( + revision, + account_id=r.account_id, + reporting_obligation_id=r.reporting_obligation_id, + reporting_revision_id=r.reporting_revision_id, + finality=r.finality, + supersedes_reporting_revision_id=None, + finality_basis="source_final" if r.finality == "official" else None, + finality_policy_id="closed" if r.finality == "official" else None, + finalized_at=NOW if r.finality == "official" else None, + ) + # Model a damaged custom-store image, including edges normal dataclass + # construction already rejects. The pure wire adapter covers these too. + object.__setattr__( + record, "supersedes_reporting_revision_id", r.supersedes_reporting_revision_id + ) + result.append(record) + return tuple(result) + + +@pytest.mark.parametrize("history,required,kind,detail", VECTORS) +def test_compatibility_health_handler_projection_and_ingest_agree(history, required, kind, detail): + if any(not r.reporting_revision_id or r.finality == "draft" for r in history): + return # Persisted records have their own closed field validation. + config = replace(configuration(), required_finality=required) + obligation = obligation_for(config) + revisions = records(history, obligation) + current = current_required_revision(obligation, revisions) + assert (current.reporting_revision_id if current else None) == ( + detail if kind == "selected" else None + ) + if any(r.reporting_obligation_id != obligation.reporting_obligation_id for r in revisions): + return # Snapshot readers partition the complete account set by obligation. + projection = project_obligation_health( + obligation, revisions, ledger_as_of=NOW, scope_closed=True + ) + snapshot = ReportingStatusSnapshot("acct_a", NOW, (config,), (obligation,), revisions) + scoped = project_status_scope( + StatusProjectionInput(snapshot, ReportingStatusScope.for_obligation(obligation)) + ) + response = ReportingStatusHandler(InMemoryReportingLedgerStore()).render_snapshot( + {"view": "summary"}, caller=ReportingStatusCaller("acct_a", "buyer"), snapshot=snapshot + ) + assert projection.health == scoped.health == response["health"] + if kind == "corrupt": + assert projection.current_revision is None + assert [i.code for i in scoped.issues] == ["HISTORY_UNAVAILABLE"] + disputed = replace(statement(obligation), consumer_status="missing") + with pytest.raises(LedgerConflictError, match="history requires repair"): + validate_status_evidence(disputed, snapshot) + with pytest.raises(LedgerConflictError, match="history requires repair"): + validate_status_evidence(replace(disputed, reporting_obligation_id=None), snapshot) + assert lifecycle_intents(replace(snapshot, statuses=(disputed,))) == () + + +@pytest.mark.parametrize( + "history", + [ + v[0] + for v in VECTORS + if v[2] == "corrupt" + and all(r.reporting_revision_id and r.finality != "draft" for r in v[0]) + ], +) +async def test_producer_rejects_corruption_before_source_or_adapter_io(history): + class DamagedStore(InMemoryReportingLedgerStore): + async def list_revisions(self, **kwargs): + return records(history, obligation) + + config = configuration() + store = DamagedStore() + await store.put_configuration(config) + obligation = await store.commit_obligation(obligation_for(config)) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=store + ) + with pytest.raises(LedgerConflictError, match="history requires repair"): + await producer.acquire_obligation(config, obligation, restate=True) + + +def test_unreadable_official_never_falls_back_to_materialized_snapshot(): + obligation = obligation_for(configuration()) + snapshot, _ = revision_for(obligation) + official = replace( + snapshot, + reporting_revision_id="official", + finality="official", + readable=False, + finality_basis="source_final", + finality_policy_id="closed", + finalized_at=NOW, + ) + assert current_required_revision(obligation, (snapshot, official)) is official + result = project_obligation_health( + obligation, (snapshot, official), ledger_as_of=NOW, scope_closed=True + ) + assert result.health == "action_required" and result.current_revision is official diff --git a/tests/conformance/reporting/test_reporting_status_migration.py b/tests/conformance/reporting/test_reporting_status_migration.py index f5f85c9e1..eb5a97426 100644 --- a/tests/conformance/reporting/test_reporting_status_migration.py +++ b/tests/conformance/reporting/test_reporting_status_migration.py @@ -24,7 +24,11 @@ ReportingNotificationWorker, ) from adcp.reporting.outbox._schema import REQUIRED_OBJECTS, schema_objects, validate_schema -from adcp.reporting.outbox.status_schema import REQUIRED_STATUS_OBJECTS, validate_status_schema +from adcp.reporting.outbox.status_schema import ( + REQUIRED_STATUS_OBJECTS, + REQUIRED_STATUS_SELECTOR_OBJECTS, + validate_status_schema, +) from . import test_reporting_notification_process_matrix as _process from ._generation_support import ( @@ -206,9 +210,10 @@ async def test_populated_repeated_c_manifest_preserves_every_a_b_object_and_row( async with pool.connection() as conn: objects = await schema_objects(conn) assert {k: objects[k] for k in REQUIRED_OBJECTS} == REQUIRED_OBJECTS - assert { - k: v for k, v in objects.items() if k not in REQUIRED_OBJECTS - } == REQUIRED_STATUS_OBJECTS + assert {k: v for k, v in objects.items() if k not in REQUIRED_OBJECTS} == { + **REQUIRED_STATUS_OBJECTS, + **REQUIRED_STATUS_SELECTOR_OBJECTS, + } assert ( json.dumps(REQUIRED_STATUS_OBJECTS, sort_keys=True, indent=2) + "\n" == files("adcp.reporting.outbox") diff --git a/tests/conformance/reporting/test_reporting_status_process_matrix.py b/tests/conformance/reporting/test_reporting_status_process_matrix.py index 8db641a67..99550d01f 100644 --- a/tests/conformance/reporting/test_reporting_status_process_matrix.py +++ b/tests/conformance/reporting/test_reporting_status_process_matrix.py @@ -14,6 +14,7 @@ ) from adcp.reporting.ledger.status_snapshot import read_snapshot_on from adcp.reporting.outbox import PgStatusNotificationStore, ReportingStatusSweeper +from adcp.reporting.outbox.status import escalation_identity from adcp.reporting.outbox.status_pg import _replay_storage from ._generation_support import ( @@ -139,13 +140,13 @@ async def database_seed(pool, case="expected", *, baseline=True): await c.execute( "INSERT INTO reporting_status_accounts (account_id, policy)" " VALUES ('acct_a', %s::jsonb)", - (json.dumps(policy.to_wire() if policy else {}),), + (json.dumps(escalation_identity(policy)),), ) await status._apply_on(c, snapshot, through=through, baseline=True) await c.execute( "UPDATE reporting_status_accounts SET baseline_complete=TRUE," " baseline_highwater=%s, dirty_sequence=%s, baseline_at=%s," - " replay_lifecycles=%s::jsonb WHERE account_id='acct_a'", + " replay_lifecycles=%s::jsonb, selector_target_version=2, selector_transition='complete' WHERE account_id='acct_a'", (through, through, snapshot.as_of, _replay_storage(snapshot)), ) return ledger, status, at, escalation @@ -236,7 +237,7 @@ async def test_real_clock_checkpoint_event_crash_restart_converges_once(crash): await child.kill() await settle_status(pool) # Capture expiry into durable state; reclaim uses production DB time. - async with pool.connection() as c: + async with status._transaction("acct_a") as c: await c.execute( "UPDATE reporting_status_scope_checkpoints SET lease_expires_at=clock_timestamp()" " WHERE lease_token IS NOT NULL" diff --git a/tests/conformance/reporting/test_reporting_status_selector_epoch.py b/tests/conformance/reporting/test_reporting_status_selector_epoch.py new file mode 100644 index 000000000..71ffd7d5b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_status_selector_epoch.py @@ -0,0 +1,417 @@ +"""Shared memory/PostgreSQL cutover, directional scope and replay contract.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import InMemoryReportingLedgerStore +from adcp.reporting.ledger.status_projection import StatusProjectionInput, project_status_scope +from adcp.reporting.outbox import ReportingStatusScope, ReportingStatusSweeper + +from . import test_reporting_status_projection_contract as contract +from ._generation_support import configuration, obligation_for, revision_for +from ._reliable_support import SimulatedCrash + +status_harness = contract.status_harness + + +def old_selected(scope, target): + """Exact v1 scope predicate, frozen here for shared-state vector setup.""" + return ( + scope.account_id == target.account_id + and (scope.consumer_id is None or scope.consumer_id == target.consumer_id) + and ( + scope.generation_key is None + or target.generation_key is None + or scope.generation_key == target.generation_key + ) + and ( + scope.reporting_obligation_id is None + or target.reporting_obligation_id is None + or scope.reporting_obligation_id == target.reporting_obligation_id + ) + ) + + +async def mark_v1(h): + """Restore old metadata only in a private test schema/state image. + + Actual unmodified C binaries establish their own baselines in the rolling + tests. This helper makes the same vectors usable by the memory reference. + """ + if isinstance(h.ledger, InMemoryReportingLedgerStore): + state = h.status._state + state.selector_accounts.clear() + state.accounts = { + a: (seq, {k: v for k, v in policy.items() if k != "selector_semantics_version"}) + for a, (seq, policy) in state.accounts.items() + } + state.checkpoints = { + key: replace(c, selector_semantics_version=1, selector_writer_floor=1) + for key, c in state.checkpoints.items() + } + else: + async with h.ledger._pool.connection() as c, c.transaction(): + await c.execute( + "ALTER TABLE reporting_status_scope_checkpoints" + " DISABLE TRIGGER reporting_status_selector_writer_v2" + ) + await c.execute( + "UPDATE reporting_status_scope_checkpoints" + " SET selector_semantics_version=1, selector_writer_floor=1" + ) + await c.execute( + "UPDATE reporting_status_accounts SET policy=policy-'selector_semantics_version'," + " selector_target_version=1, selector_transition='pending'" + ) + await c.execute( + "ALTER TABLE reporting_status_scope_checkpoints" + " ENABLE TRIGGER reporting_status_selector_writer_v2" + ) + + +async def restart(h): + cls = type(h.status) + await h.reliable.restart() + h.status = cls(h.ledger) + + +async def two_feeds(h, *, issue=True): + owners = {} + for feed in ("analytics", "billing"): + config = replace(configuration(), delivery_config_id=feed, feed_purpose=feed) + await h.ledger.put_configuration(config) + obligation = replace(obligation_for(config), reporting_obligation_id=f"rpo_{feed}") + await h.ledger.commit_obligation(obligation) + revision, rows = revision_for(obligation, suffix=feed) + await h.ledger.commit_revision(revision, rows) + owners[feed] = obligation + issues = {} + if issue: + for key, scope in ( + ( + "billing-partial", + ReportingStatusScope("acct_a", consumer_id="buyer", feed_purpose="billing"), + ), + ("analytics-public", ReportingStatusScope.for_obligation(owners["analytics"])), + ("billing-private", ReportingStatusScope.for_obligation(owners["billing"], "auditor")), + ): + issues[key] = await h.ledger.ensure_issue_opened( + issue_key=key, + account_id="acct_a", + consumer_id=scope.consumer_id, + observed_at=h.clock(), + status_scope=scope, + ) + return owners, issues + + +async def test_directional_feed_public_private_configuration_obligation_scopes(status_harness): + h = status_harness + owners, issues = await two_feeds(h) + snapshot = await h.ledger.read_status_snapshot(account_id="acct_a") + for consumer in (None, "buyer", "auditor"): + for feed in (None, "analytics", "billing"): + for level in ("account", "configuration", "obligation"): + if level != "account" and feed is None: + continue + owner = owners[feed] if feed else None + scope = ReportingStatusScope( + "acct_a", + owner.generation_key if level != "account" else None, + owner.reporting_obligation_id if level == "obligation" else None, + consumer, + feed, + ) + result = project_status_scope(StatusProjectionInput(snapshot, scope)) + expected = set() + if feed in (None, "analytics"): + expected.add(issues["analytics-public"].issue_id) + if feed in (None, "billing") and consumer == "buyer": + expected.add(issues["billing-partial"].issue_id) + if feed in (None, "billing") and consumer == "auditor": + expected.add(issues["billing-private"].issue_id) + assert {i.issue_id for i in result.issues} == expected, (consumer, feed, level) + assert all( + feed is None or o.obligation.feed_purpose == feed for o in result.obligations + ) + + +async def test_contaminated_epoch_corrects_changed_issue_set_once_and_preserves_every_identity( + status_harness, monkeypatch +): + h = status_harness + await two_feeds(h) + import adcp.reporting.ledger.status_projection as projection + + with monkeypatch.context() as patch: + patch.setattr(projection, "_selected", old_selected) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + before = {c.scope.checkpoint_key: c for c in await h.status.checkpoints(account_id="acct_a")} + assert not await h.status.baseline_ready(account_id="acct_a") + assert (await h.status.rebuild_one()).events == 0 # Durable fence only. + fenced = await h.status.checkpoints(account_id="acct_a") + assert all(c.selector_writer_floor == 2 and c.selector_semantics_version == 1 for c in fenced) + assert not await h.status.claim_due(account_id="acct_a") + await restart(h) + await h.drain() + events = await h.status.outbox.list_events(account_id="acct_a") + after = {c.scope.checkpoint_key: c for c in await h.status.checkpoints(account_id="acct_a")} + changed = {k for k, c in after.items() if c.fingerprint != before[k].fingerprint} + assert changed and len(changed) == len(events) == 2 + assert all(e.cause.health == e.cause.previous_health == "action_required" for e in events) + for key, current in after.items(): + old = before[key] + assert current.scope == old.scope and current.baseline == old.baseline + assert current.source_sequence == old.source_sequence + assert ( + current.lease_token == old.lease_token + and current.lease_expires_at == old.lease_expires_at + ) + assert current.generation == old.generation + (key in changed) + assert current.selector_semantics_version == current.selector_writer_floor == 2 + assert await h.status.baseline_ready(account_id="acct_a") + assert not (await h.status.rebuild_one()).did_work + assert not (await h.status.project_one(account_id="acct_a")).did_work + assert await h.status.checkpoints(account_id="acct_a") == tuple(after.values()) + assert await h.status.outbox.list_events(account_id="acct_a") == events + + +async def test_captured_reversal_boundaries_precede_chronological_overdue_deadlines(status_harness): + h = status_harness + first, revision, _ = await h.seed(readable=True) + config = replace(configuration(), delivery_config_id="second") + await h.ledger.put_configuration(config) + waiting = replace(obligation_for(config), reporting_obligation_id="rpo_waiting") + await h.ledger.commit_obligation(waiting) + h.clock.now = waiting.period.expected_at - timedelta(seconds=5) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + for readable in (False, True, False, True): + h.clock.advance() + await h.ledger.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revision.reporting_revision_id, + readable=readable, + ) + h.clock.now = waiting.automated_recovery_deadline_at + timedelta(hours=2) + for _ in range(30): + turn = await h.status.rebuild_one() + if not turn.did_work: + break + await restart(h) # Every committed fence/boundary/deadline/final mark survives restart. + else: + pytest.fail("selector epoch did not converge") + events = await h.events() + ready_events = sorted( + ( + e + for e in events + if e.cause.scope.reporting_obligation_id == first.reporting_obligation_id + ), + key=lambda e: e.cause.checkpoint_generation, + ) + assert [e.cause.health for e in ready_events] == [ + "action_required", + "complete", + "action_required", + "complete", + ] + assert [e.cause.previous_health for e in ready_events] == [ + "complete", + "action_required", + "complete", + "action_required", + ] + late = sorted( + ( + e + for e in events + if e.cause.scope.reporting_obligation_id == waiting.reporting_obligation_id + ), + key=lambda e: e.cause.checkpoint_generation, + ) + assert [e.cause.health for e in late] == ["delayed", "action_required"] + assert [e.cause.previous_health for e in late] == ["waiting", "delayed"] + assert len({e.notification_id for e in events}) == len(events) + assert not (await ReportingStatusSweeper(h.status).run_once(account_id="acct_a")).did_work + + +@pytest.mark.parametrize("fail_at", [1, 2, 4]) +async def test_checkpoint_event_failure_rolls_back_epoch_turn_and_restart_is_once_only( + status_harness, monkeypatch, fail_at +): + h = status_harness + await two_feeds(h, issue=False) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + assert (await h.status.rebuild_one()).did_work + await h.ledger.ensure_issue_opened( + issue_key="late-public", + account_id="acct_a", + consumer_id=None, + observed_at=h.clock(), + status_scope=ReportingStatusScope("acct_a"), + ) + before = await h.status.checkpoints(account_id="acct_a") + from adcp.reporting.outbox import status_memory, status_pg + + original = status_pg.advance_checkpoint + called = 0 + + def crash(*args, **kwargs): + nonlocal called + called += 1 + if called == fail_at: + raise SimulatedCrash("selector-checkpoint-event-pre-commit") + return original(*args, **kwargs) + + with monkeypatch.context() as patch: + patch.setattr(status_memory, "advance_checkpoint", crash) + patch.setattr(status_pg, "advance_checkpoint", crash) + with pytest.raises(SimulatedCrash): + await h.status.rebuild_one() + assert await h.status.checkpoints(account_id="acct_a") == before + assert not await h.status.outbox.list_events(account_id="acct_a") + await restart(h) + await h.drain() + events = await h.status.outbox.list_events(account_id="acct_a") + assert len(events) == 4 and {e.cause.checkpoint_generation for e in events} == {1} + assert not (await h.status.rebuild_one()).did_work + + +@pytest.mark.parametrize("phase", ["fence", "final_mark"]) +async def test_crash_before_fence_or_final_mark_commit_is_restartable( + status_harness, monkeypatch, phase +): + h = status_harness + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + if phase == "final_mark": + assert (await h.status.rebuild_one()).did_work + before = await h.status.checkpoints(account_id="acct_a") + name = "_rebuild" if isinstance(h.ledger, InMemoryReportingLedgerStore) else "_rebuild_on" + original = getattr(h.status, name) + + def memory_crash(*args): + original(*args) + raise SimulatedCrash("selector-epoch-pre-commit") + + async def pg_crash(*args): + await original(*args) + raise SimulatedCrash("selector-epoch-pre-commit") + + with monkeypatch.context() as patch: + patch.setattr(h.status, name, memory_crash if name == "_rebuild" else pg_crash) + with pytest.raises(SimulatedCrash): + await h.status.rebuild_one() + assert await h.status.checkpoints(account_id="acct_a") == before + assert not await h.status.baseline_ready(account_id="acct_a") + await restart(h) + await h.drain() + assert await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.outbox.list_events(account_id="acct_a") + + +async def test_retained_scopes_outside_current_discovery_are_reprojected_without_deletion( + status_harness, monkeypatch +): + h = status_harness + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + before = await h.status.checkpoints(account_id="acct_a") + expired = h.clock() - timedelta(seconds=1) + if isinstance(h.ledger, InMemoryReportingLedgerStore): + h.status._state.checkpoints = { + key: replace(c, next_due_at=expired) for key, c in h.status._state.checkpoints.items() + } + else: + async with h.ledger._pool.connection() as c: + await c.execute( + "UPDATE reporting_status_scope_checkpoints SET next_due_at=%s", (expired,) + ) + from adcp.reporting.outbox import status_memory, status_pg + + with monkeypatch.context() as patch: + patch.setattr(status_memory, "projection_scopes", lambda snapshot: ()) + patch.setattr(status_pg, "projection_scopes", lambda snapshot: ()) + await h.drain() + after = await h.status.checkpoints(account_id="acct_a") + assert len(after) == len(before) + assert [(c.scope, c.fingerprint, c.generation, c.baseline) for c in after] == [ + (c.scope, c.fingerprint, c.generation, c.baseline) for c in before + ] + assert all(c.selector_semantics_version == 2 and c.next_due_at is None for c in after) + assert not (await h.status.rebuild_one()).did_work + assert not await h.status.outbox.list_events(account_id="acct_a") + + +async def test_account_epoch_readiness_and_new_baseline_are_isolated(status_harness): + h = status_harness + for account in ("acct_a", "acct_b"): + await h.ledger.put_configuration(configuration(account)) + await h.status.baseline(account_id=account) + await mark_v1(h) + assert not await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.baseline_ready(account_id="acct_b") + await h.drain() + assert await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.baseline_ready(account_id="acct_b") + await restart(h) + assert await h.status.baseline_ready(account_id="acct_a") + assert not await h.status.baseline_ready(account_id="acct_b") + assert await h.status.baseline(account_id="new-account") + assert await h.status.baseline_ready(account_id="new-account") + assert not await h.status.outbox.list_events(account_id="new-account") + + +async def test_old_memory_state_image_without_any_epoch_fields_is_imported_as_v1(): + from types import SimpleNamespace + + from adcp.reporting.outbox import InMemoryStatusNotificationStore + + from ._reliable_support import reliable_factory + + async with reliable_factory("memory", notifications=True) as reliable: + h = contract.StatusHarness(reliable, InMemoryStatusNotificationStore(reliable.store)) + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + before = await h.status.checkpoints(account_id="acct_a") + state = h.status._state + del state.selector_accounts + state.accounts = {a: (seq, {}) for a, (seq, _) in state.accounts.items()} + state.checkpoints = { + key: SimpleNamespace( + **{k: v for k, v in vars(c).items() if not k.startswith("selector_")} + ) + for key, c in state.checkpoints.items() + } + h.status = InMemoryStatusNotificationStore(h.ledger) + assert all( + c.selector_semantics_version == 1 + for c in await h.status.checkpoints(account_id="acct_a") + ) + assert not await h.status.baseline_ready(account_id="acct_a") + await h.drain() + after = await h.status.checkpoints(account_id="acct_a") + assert after == before + assert not await h.status.outbox.list_events(account_id="acct_a") + + +async def test_existing_c_service_discovers_old_accounts_without_an_account_list(status_harness): + from types import SimpleNamespace + + from adcp.reporting.outbox import ReportingStatusService + + h = status_harness + await h.seed(readable=True) + await h.status.baseline(account_id="acct_a") + await mark_v1(h) + service = ReportingStatusService(SimpleNamespace(store=h.status, account_ids=())) + assert await service.drain() >= 2 + assert await h.status.baseline_ready(account_id="acct_a") + assert await service.drain() == 0 diff --git a/tests/conformance/reporting/test_reporting_status_selector_pg.py b/tests/conformance/reporting/test_reporting_status_selector_pg.py new file mode 100644 index 000000000..96f6db711 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_status_selector_pg.py @@ -0,0 +1,416 @@ +"""Populated real-C cutover, immutable retention and old row-only lease races.""" + +import asyncio +import json +import subprocess +import sys +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from importlib.resources import files +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import PgReportingReconciliationStore +from adcp.reporting.outbox import ( + ActivityRequest, + PgStatusNotificationStore, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ReportingStatusSweeper, +) +from adcp.reporting.outbox.status_schema import validate_status_schema + +from ._generation_support import ( + NOW, + assert_c_collated_rolling_database, + configuration, + isolated_reporting_pool, + obligation_for, + revision_for, +) +from ._reliable_support import ( + FailurePlan, + ScriptedSigning, + ScriptedSubscriptions, + notification_subscription, +) +from .test_reporting_notification_migration import retained_physical_rows +from .test_reporting_status_migration import C_QUEUES, physical_rows + +ROOT = Path(__file__).resolve().parents[3] +C_SHA = "ea150fabd5ad90e3abf93f89729d2919f1c61798" + + +@pytest.fixture(scope="module") +def frozen_c(tmp_path_factory): + assert_c_collated_rolling_database() + root = tmp_path_factory.mktemp("frozen-c-selector") / "source" + subprocess.run( + ["git", "worktree", "add", "--detach", str(root), C_SHA], + cwd=ROOT, + check=True, + capture_output=True, + timeout=60, + ) + try: + yield root + finally: + subprocess.run( + ["git", "worktree", "remove", "--force", str(root)], + cwd=ROOT, + check=True, + capture_output=True, + timeout=60, + ) + + +class OldC: + def __init__(self, process): + self.process = process + + async def send(self, **command): + self.process.stdin.write( + (json.dumps(command, default=lambda v: v.isoformat()) + "\n").encode() + ) + await self.process.stdin.drain() + + async def receive(self): + line = await asyncio.wait_for(self.process.stdout.readline(), 30) + assert line, "frozen C process exited before replying" + return json.loads(line) + + async def call(self, **command): + await self.send(**command) + return await self.receive() + + +@asynccontextmanager +async def old_c(pool, source, *, now=NOW): + process = await asyncio.create_subprocess_exec( + sys.executable, + "-I", + str(Path(__file__).with_name("_frozen_status_c.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + child = OldC(process) + try: + await child.send(source=str(source), conninfo=pool.conninfo, kwargs=pool.kwargs, now=now) + assert Path((await child.receive())["origin"]).is_relative_to(source) + yield child + assert await child.call(action="stop") == {"stopped": True} + await asyncio.wait_for(process.wait(), 10) + assert process.returncode == 0 + finally: + if process.returncode is None: + process.kill() + await process.wait() + error = await process.stderr.read() + assert not error, error.decode() + + +async def seed(ledger, *, account="acct_a", readable=False, name="daily"): + config = replace(configuration(account), delivery_config_id=name) + await ledger.put_configuration(config) + obligation = replace(obligation_for(config), reporting_obligation_id=f"rpo_{account}_{name}") + await ledger.commit_obligation(obligation) + revision, rows = revision_for(obligation, suffix=name) + if readable: + await ledger.commit_revision(revision, rows) + return obligation, revision + + +async def drain(status, account="acct_a"): + for _ in range(40): + if not (await status.project_one(account_id=account)).did_work: + return + pytest.fail("selector migration failed to converge") + + +async def populate_status_activity(status): + failures = FailurePlan() + subscriptions = ScriptedSubscriptions(failures) + subscriptions.put(notification_subscription(events=("reporting.status_changed",))) + worker = ReportingNotificationWorker( + outbox=status.outbox, + subscriptions=subscriptions, + signing=ScriptedSigning(failures), + cipher=ReportingEnvelopeCipher(b"e" * 32), + activity=status.outbox, + ) + while await worker.expand_one(account_id="acct_a"): + pass + now = datetime.now(timezone.utc) + lease = await status.outbox.claim_delivery(account_id="acct_a", now=now, lease_seconds=60) + assert lease is not None + assert await status.outbox.reserve_attempt( + lease, request=ActivityRequest("https://receiver.example.test/reporting", 1), now=now + ) + assert await status.outbox.finish_delivery(lease, state="pending", retry_at=now, now=now) + + +async def test_actual_c_claim_fence_pool_marker_two_projectors_two_sweepers_and_source(frozen_c): + from psycopg_pool import AsyncConnectionPool + + async with isolated_reporting_pool(autocommit=True) as owner, old_c(owner, frozen_c) as old: + assert await old.call(action="schema") == {"result": True} + clock = [NOW] + ledger = PgReportingReconciliationStore( + pool=owner, notifications=True, clock=lambda: clock[0] + ) + waiting, _ = await seed(ledger) + _, ready = await seed(ledger, readable=True, name="ready") + await seed(ledger, account="acct_b") + baseline_at = waiting.period.expected_at - timedelta(seconds=1) + for account in ("acct_a", "acct_b"): + assert await old.call(action="baseline", account=account, now=baseline_at) == { + "result": True + } + assert await old.call(action="ready", account=account) == {"result": True} + # A populated C event and every C activity table exist before cutover. + await old.call( + action="source", revision=ready.reporting_revision_id, readable=False, now=baseline_at + ) + assert (await old.call(action="project"))["result"]["events"] == 2 + status = PgStatusNotificationStore(ledger) + await populate_status_activity(status) + immutable = await retained_physical_rows(owner) + queue_rows = await physical_rows(owner) + assert all(queue_rows.values()) + await status.create_schema() + assert await retained_physical_rows(owner) == immutable + assert await physical_rows(owner) == queue_rows + assert not await status.baseline_ready(account_id="acct_a") + assert await old.call(action="ready") == {"result": True} # Schema alone isn't cutover. + # Old claim owns only a checkpoint row. The new fence owns the account + # then waits for that row; the old trigger must never acquire account. + await old.send(action="claim", now=NOW, hold="claim") + assert await old.receive() == {"held": True} + fencing = asyncio.create_task(status.project_one(account_id="acct_a")) + await asyncio.sleep(0.02) + assert not fencing.done() + await old.send(action="release_hold") + old_lease = (await old.receive())["result"] + assert old_lease is not None + assert (await asyncio.wait_for(fencing, 10)).events == 0 + # The fence preserves existing lease identity; old claims/completion + # fail before a write, while v2 migration doesn't wait for lease expiry. + assert any( + c.lease_token == old_lease["token"] + for c in await status.checkpoints(account_id="acct_a") + ) + fenced_rows = await physical_rows( + owner, + tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), + ) + assert (await old.call(action="claim"))["error"] == "database_fence" + assert (await old.call(action="complete", lease=0))["error"] == "status_policy_conflict" + assert (await old.call(action="release", lease=0))["error"] == "database_fence" + assert (await old.call(action="ready"))["error"] == "status_policy_conflict" + assert (await old.call(action="project"))["error"] == "status_policy_conflict" + assert await physical_rows(owner, tables=tuple(fenced_rows)) == fenced_rows + assert await old.call(action="schema") == {"result": True} + assert await physical_rows(owner, tables=tuple(fenced_rows)) == fenced_rows + async with owner.connection() as c: + await validate_status_schema(c, activity=True) + assert ( + await ( + await c.execute( + "SELECT current_setting('adcp.reporting.selector_semantics_version',true)" + ) + ).fetchone() + )[0] in (None, "") + # Four distinct size-one pools exercise the account lock across every + # new projector/sweeper path; source dirties arrive concurrently. + pools = [ + AsyncConnectionPool( + owner.conninfo, kwargs=owner.kwargs, min_size=1, max_size=1, open=False + ) + for _ in range(4) + ] + try: + for pool in pools: + await pool.open(wait=True) + stores = [ + PgStatusNotificationStore( + PgReportingReconciliationStore( + pool=p, notifications=True, clock=lambda: clock[0] + ) + ) + for p in pools + ] + + async def sweep(s): + for _ in range(12): + await ReportingStatusSweeper(s).run_once(account_id="acct_a") + + await asyncio.wait_for( + asyncio.gather( + drain(stores[0]), + drain(stores[1]), + sweep(stores[2]), + sweep(stores[3]), + ledger.set_revision_readable( + account_id="acct_a", + reporting_revision_id=ready.reporting_revision_id, + readable=True, + ), + ), + 25, + ) + await drain(stores[0]) + for pool in pools: + async with pool.connection() as c: + assert ( + await ( + await c.execute( + "SELECT current_setting(" + "'adcp.reporting.selector_semantics_version',true)" + ) + ).fetchone() + )[0] in (None, "") + finally: + for pool in pools: + await pool.close() + assert await status.baseline_ready(account_id="acct_a") + assert not await status.baseline_ready(account_id="acct_b") + assert (await old.call(action="ready"))["error"] == "status_policy_conflict" + assert await old.call(action="ready", account="acct_b") == {"result": True} + clock[0] = waiting.automated_recovery_deadline_at + assert (await ReportingStatusSweeper(status).run_once(account_id="acct_a")).did_work + events = await status.outbox.list_events(account_id="acct_a") + late = sorted( + ( + e + for e in events + if e.cause.scope.reporting_obligation_id == waiting.reporting_obligation_id + ), + key=lambda e: e.cause.checkpoint_generation, + ) + assert [(e.cause.previous_health, e.cause.health) for e in late] == [ + ("waiting", "delayed"), + ("delayed", "action_required"), + ] + assert len( + {(e.cause.scope.checkpoint_key, e.cause.checkpoint_generation) for e in events} + ) == len(events) + after = await physical_rows(owner) + assert all(all(row in after[table] for row in rows) for table, rows in queue_rows.items()) + assert await status.baseline(account_id="brand-new") + assert await status.baseline_ready(account_id="brand-new") + assert not await status.outbox.list_events(account_id="brand-new") + + +async def test_actual_c_unique_looking_snapshot_corruption_corrects_without_baseline_reset( + frozen_c, +): + async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: + assert await old.call(action="schema") == {"result": True} + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) + obligation, _ = await seed(ledger, readable=True) + second, rows = revision_for(obligation, suffix="disconnected") + await ledger.commit_revision(second, rows) # v1 persistence accepts another snapshot root. + assert await old.call(action="baseline") == {"result": True} + status = PgStatusNotificationStore(ledger) + immutable = await retained_physical_rows(pool) + await status.create_schema() + before = await status.checkpoints(account_id="acct_a") + assert all( + c.snapshot["health"] == "complete" and c.selector_semantics_version == 1 for c in before + ) + await drain(status) + after = await status.checkpoints(account_id="acct_a") + events = await status.outbox.list_events(account_id="acct_a") + assert len(events) == 2 + assert all( + e.cause.previous_health == "complete" and e.cause.health == "action_required" + for e in events + ) + assert all(c.snapshot["issues"][0]["code"] == "HISTORY_UNAVAILABLE" for c in after) + assert [(c.scope, c.baseline, c.source_sequence) for c in after] == [ + (c.scope, c.baseline, c.source_sequence) for c in before + ] + assert await retained_physical_rows(pool) == immutable + physical = await physical_rows( + pool, + tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), + ) + assert not (await status.rebuild_one()).did_work + assert not (await status.project_one(account_id="acct_a")).did_work + assert await physical_rows(pool, tables=tuple(physical)) == physical + + +async def test_actual_c_inflight_projector_commits_before_v2_fence_and_source_serializes(frozen_c): + async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: + assert await old.call(action="schema") == {"result": True} + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) + _, revision = await seed(ledger, readable=True) + assert await old.call(action="baseline") == {"result": True} + await old.call(action="source", revision=revision.reporting_revision_id, readable=False) + status = PgStatusNotificationStore(ledger) + await status.create_schema() + await old.send(action="project", hold="project") + assert await old.receive() == {"held": True} + fence = asyncio.create_task(status.project_one(account_id="acct_a")) + publication = asyncio.create_task( + ledger.set_revision_readable( + account_id="acct_a", + reporting_revision_id=revision.reporting_revision_id, + readable=True, + ) + ) + await asyncio.sleep(0.02) + assert not fence.done() and not publication.done() + await old.send(action="release_hold") + assert (await old.receive())["result"]["events"] == 2 + assert (await asyncio.wait_for(fence, 10)).events == 0 + await asyncio.wait_for(publication, 10) + old_events = await physical_rows(pool) + await drain(status) + assert await status.baseline_ready(account_id="acct_a") + events = await status.outbox.list_events(account_id="acct_a") + assert len(events) == 4 + assert all( + c.snapshot["health"] == "complete" + for c in await status.checkpoints(account_id="acct_a") + ) + after = await physical_rows(pool) + assert all(all(row in after[table] for row in rows) for table, rows in old_events.items()) + assert (await old.call(action="project"))["error"] == "status_policy_conflict" + + +async def test_populated_selector_sql_rollback_and_repeated_concurrent_install(frozen_c): + async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: + assert await old.call(action="schema") == {"result": True} + ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) + await seed(ledger, readable=True) + assert await old.call(action="baseline") == {"result": True} + rows = await physical_rows( + pool, + tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), + ) + immutable = await retained_physical_rows(pool) + migration = ( + files("adcp.reporting.ledger") + .joinpath("reporting_status_selector_version.sql") + .read_text() + ) + with pytest.raises(RuntimeError, match="interrupt migration"): + async with pool.connection() as c, c.transaction(): + await c.execute(migration) + raise RuntimeError("interrupt migration") + assert await physical_rows(pool, tables=tuple(rows)) == rows + assert await retained_physical_rows(pool) == immutable + assert await old.call(action="ready") == {"result": True} + status = PgStatusNotificationStore(ledger) + await asyncio.wait_for(asyncio.gather(status.create_schema(), status.create_schema()), 30) + await status.create_schema() + assert await retained_physical_rows(pool) == immutable + assert not await status.baseline_ready(account_id="acct_a") + assert await old.call(action="ready") == {"result": True} + await drain(status) + assert await status.baseline_ready(account_id="acct_a") + assert not await status.outbox.list_events(account_id="acct_a") diff --git a/tests/type_checks/reporting_destination_writer.py b/tests/type_checks/reporting_destination_writer.py new file mode 100644 index 000000000..c9d42a325 --- /dev/null +++ b/tests/type_checks/reporting_destination_writer.py @@ -0,0 +1,134 @@ +"""Public B1 adoption, without private model imports, Any, or type suppressions.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator, Sequence +from dataclasses import dataclass +from datetime import datetime +from typing import Literal + +from adcp.reporting.ledger import ( + ReportingObligationRecord, + ReportingRevisionRecord, + select_reporting_revision, +) +from adcp.reporting.materializer import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, + ReportingDestinationBinding, + ReportingDestinationIO, + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationRequest, + ReportingDestinationResolver, + ReportingDestinationSession, + ReportingDestinationWriter, + ReportingIOContext, + ReportingIOPhase, + ReportingMaterializationAttempt, + ReportingNativeObservation, + ReportingObligationDeliveryRecord, + ReportingPreparedRevision, + ReportingRevisionRowReader, + ReportingRevisionVerifierRegistry, + ReportingVerifiedDestination, + reference_verifier, +) + + +@dataclass(frozen=True) +class TrustedResolver: + """Real adopters acquire credentials in session._open and release in _close.""" + + delegate: ReferenceReportingResolver + + def resolve( + self, + request: ReportingDestinationRequest, + *, + phase: ReportingIOPhase, + context: ReportingIOContext, + ) -> ReportingDestinationSession: + return self.delegate.resolve(request, phase=phase, context=context) + + +class AdopterSession(ReportingDestinationSession): + """Typed provider hooks; the SDK owns redaction and the async context lifecycle.""" + + async def _open(self) -> None: + pass + + async def _close(self) -> None: + pass + + async def write(self, content: ReportingPreparedRevision) -> ReportingDestinationLocator: + raise NotImplementedError + + async def read_rows( + self, locator: ReportingDestinationLocator, *, cursor: str | None, limit: int + ) -> ReportingDestinationPage: + raise NotImplementedError + + async def read_manifest(self, locator: ReportingDestinationLocator) -> bytes: + raise NotImplementedError + + async def list_objects(self, locator: ReportingDestinationLocator) -> tuple[str, ...]: + raise NotImplementedError + + async def read_object( + self, locator: ReportingDestinationLocator, *, object_ref: str + ) -> AsyncIterator[bytes]: + yield b"" + + async def observe_native_version( + self, locator: ReportingDestinationLocator + ) -> ReportingNativeObservation: + raise NotImplementedError + + +async def adopter( + reader: ReportingRevisionRowReader, + binding: ReportingDestinationBinding, + delivery: ReportingObligationDeliveryRecord, + obligation: ReportingObligationRecord, + history: Sequence[ReportingRevisionRecord], + attempt: ReportingMaterializationAttempt, + deadline: datetime, +) -> ReportingVerifiedDestination | None: + selection = select_reporting_revision( + history, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt" or selection.kind == "not_ready": + reason: str = selection.reason + assert reason + return None + chosen: ReportingRevisionRecord = selection.revision + assert chosen.reporting_revision_id == attempt.reporting_revision_id + verifier = reference_verifier() + registry = ReportingRevisionVerifierRegistry((verifier,)) + reference = ReferenceReportingDestinationWriter((verifier.key.capability,)) + production: Literal[False] = reference.production_eligible + assert production is False + writer: ReportingDestinationWriter = reference + assert writer.capabilities + resolver: ReportingDestinationResolver = TrustedResolver( + ReferenceReportingResolver(reference, registry, (binding,)) + ) + cancel = asyncio.Event() + prepared = await registry.prepare( + key=verifier.key, + binding=binding, + delivery=delivery, + obligation=obligation, + revisions=history, + attempt=attempt, + reader=reader, + context=ReportingIOContext(deadline, cancel), + ) + io = ReportingDestinationIO(registry, resolver) + locator = await io.write(prepared, context=ReportingIOContext(deadline, cancel)) + return await io.verify(prepared, locator, context=ReportingIOContext(deadline, cancel)) From cc5c097c979034de9824ea6b32dce07cf22b2e5b Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 00:44:36 +0000 Subject: [PATCH 2/5] fix(reporting): chain snapshot restatements past an official close `commit_revision_from_manifest` chose `supersedes` from a whole-history selection, which returns the *official* revision outright once one exists. A snapshot restatement published after an official close therefore rooted itself at `None`, leaving the obligation with two snapshot roots -- `disconnected_snapshot_history` over immutable rows, which health projection, producer acquisition, consumer-status ingest and status evidence validation all park for a repair the SDK cannot perform. Take the leaf from the snapshot subset instead, and fail closed when either pass reports corruption. The replaced `_current_snapshot_leaf` chained this correctly. Two further closed-contract gaps found in the same review: - A retained Core `ReportingDefinitionBinding` is only loosely validated, so an obligation can carry a definition -- a versioned query URI, an uppercase or short digest, a legacy dialect or schema version -- that the strict `ReportingVerificationKey` cannot express. Rebinding it raised a raw `ValueError` out of `ReportingRevisionVerifierRegistry.prepare()`, `ReportingDestinationIO.write()` and `.verify()` rather than the closed `BINDING_MISMATCH` every caller of that boundary catches. - `plan_consumer_statuses`' new fail-closed history checks reported "no reading was supplied" when the missing input was `obligation_revisions`, pointing adopters at the wrong argument, and the existing revision-without-reading regression test silently stopped covering its own path. Name the obligation and the argument, document the complete-partition contract on the public function, and restore that coverage. Regression tests cover snapshot-after-official chaining on memory and PostgreSQL, all four definition coordinates through prepare/write/verify, and the absent, short, long and damaged partition shapes. Documents the new 1..500 `read_revision_rows` page-size bound. Co-Authored-By: Claude Opus 5 (1M context) --- docs/reporting-destination-writer.md | 15 ++++- src/adcp/reporting/_consumer.py | 21 ++++++- src/adcp/reporting/ledger/producer.py | 18 ++++-- .../reporting/materializer/verification.py | 31 ++++++++-- .../test_reporting_materializer_contracts.py | 33 +++++++++++ .../test_reporting_revision_selection.py | 57 +++++++++++++++++++ tests/test_reporting_consumer_loop.py | 20 +++++++ 7 files changed, 180 insertions(+), 15 deletions(-) diff --git a/docs/reporting-destination-writer.md b/docs/reporting-destination-writer.md index f2c12112a..895aa9ed2 100644 --- a/docs/reporting-destination-writer.md +++ b/docs/reporting-destination-writer.md @@ -98,9 +98,18 @@ Preparation reads **every** frozen source page, including zero rows and 501+ rows, rederives the Core digest and the canonical digest, and recomputes typed totals before destination authorization. SDK source cursors bind revision and offset; custom row readers return the same `ReportingRowPage` identity/cursor -contract. Stable totals, cursor progress, cycles, `has_more` pairing and final -count are enforced. Source/destination walks bound bytes, recursive items, -nesting, rows, pages, objects and chunks. +contract. Both bundled ledger stores now reject a cursor issued for another +revision and any `read_revision_rows` page size outside 1..500 with +`INVALID_CURSOR` / `INVALID_PAGE_SIZE`; a caller that paged in larger windows +must split its walk. Stable totals, cursor progress, cycles, `has_more` pairing +and final count are enforced. Source/destination walks bound bytes, recursive +items, nesting, rows, pages, objects and chunks. + +A retained Core `ReportingDefinitionBinding` is only loosely constrained, so an +obligation may carry a definition the strict verification key cannot express -- +a versioned query URI, an uppercase or short digest, a legacy dialect or schema +version. Preparation, write and readback answer that with the closed +`BINDING_MISMATCH` failure rather than a raw `ValueError`. Readback independently verifies every logical destination row in order. File verification also reads the exact manifest bytes and its closed schema, diff --git a/src/adcp/reporting/_consumer.py b/src/adcp/reporting/_consumer.py index ba7b98ec8..7705da28a 100644 --- a/src/adcp/reporting/_consumer.py +++ b/src/adcp/reporting/_consumer.py @@ -482,6 +482,14 @@ def plan_consumer_statuses( publishing nothing it demonstrably published, and ``received`` asserts bytes the buyer never looked at. + ``obligation_revisions`` must carry each due obligation's *complete* + retained history -- the same partition its ``revision_count`` declares. + Finality is chosen by whole-history selection + (:func:`~adcp.reporting.revision_selection.select_reporting_revision`), not + by a count, so a partition that is absent, short, long or structurally + damaged raises :class:`ConsumerStatusPlanError` instead of selecting from + it. An obligation with ``revision_count == 0`` needs no entry. + ``current_statuses`` is this caller's own status history from the same ledger read. An intent whose content matches the current leaf is skipped entirely -- re-filing an unchanged claim under a fresh id churns the chain @@ -820,13 +828,20 @@ def _has_required_revision( ) if result.kind == "corrupt" or len(revisions) != obligation.revision_count: raise ConsumerStatusPlanError( - "the complete revision history requires repair before planning a consumer status" + f"obligation {obligation.reporting_obligation_id!r} was given " + f"{len(revisions)} revisions against a declared revision_count of " + f"{obligation.revision_count}, or a damaged chain; obligation_revisions must " + "be that obligation's complete retained history. Re-read the seller's " + "snapshot rather than filing a status against a history neither party can " + "reconcile" ) return result.kind == "selected" if obligation.revision_count: raise ConsumerStatusPlanError( - "the obligation advertises revisions but no reading was supplied; " - "a complete revision history is required to select finality" + f"obligation {obligation.reporting_obligation_id!r} advertises " + f"{obligation.revision_count} revisions but obligation_revisions carries no entry " + "for it; pass its complete retained history. required_finality cannot be decided " + "from a count, and guessing would file a status nobody validated" ) return False diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 3fce9fb71..273912457 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -684,12 +684,22 @@ async def commit_revision_from_manifest( reporting_obligation_id=obligation.reporting_obligation_id, required_finality="snapshot", ) - if selection.kind == "corrupt": + # A retained official close coexists with the snapshot chain and wins + # whole-history selection outright, so it is never the snapshot leaf. + # Reading it as one would root a restatement at ``None`` and split the + # obligation into two snapshot roots -- a permanently corrupt history + # over immutable rows, with no repair path. + leaf = select_reporting_revision( + tuple(item for item in existing if item.finality == "snapshot"), + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality="snapshot", + ) + if selection.kind == "corrupt" or leaf.kind == "corrupt": raise LedgerConflictError("HISTORY_UNAVAILABLE", "the revision history requires repair") - current = selection.revision if selection.kind == "selected" else None supersedes = ( - current.reporting_revision_id - if finality == "snapshot" and current is not None and current.finality == "snapshot" + leaf.revision.reporting_revision_id + if finality == "snapshot" and leaf.kind == "selected" else None ) diff --git a/src/adcp/reporting/materializer/verification.py b/src/adcp/reporting/materializer/verification.py index 8cf7c9f9b..308b7880d 100644 --- a/src/adcp/reporting/materializer/verification.py +++ b/src/adcp/reporting/materializer/verification.py @@ -31,7 +31,11 @@ _ClosedValue, _freeze_fields, ) -from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.ledger.models import ( + ReportingDefinitionBinding, + ReportingObligationRecord, + ReportingRevisionRecord, +) from adcp.reporting.ledger.producer import revision_content_sha256 from adcp.reporting.ledger.store import LedgerConflictError, ReportingRowPage, revision_row_offset from adcp.reporting.materializer._json import ( @@ -330,8 +334,7 @@ async def prepare( raise failure("REVISION_NOT_READY") request = ReportingDestinationRequest.from_binding(binding, attempt, key) if ( - obligation.definition is None - or replace(key, definition=obligation.definition) != key + not _same_definition(key, obligation.definition) or (obligation.report_definition_id, obligation.reporting_profile) != (key.report_definition_id, key.reporting_profile) or request.generation != obligation.generation_key @@ -513,8 +516,7 @@ def _validate_prepared(self, prepared: ReportingPreparedRevision) -> ReportingRe or prepared.delivery.scope.generation_key != request.generation or prepared.delivery.scope.reporting_obligation_id != request.reporting_obligation_id or prepared.delivery.currency != obligation.currency - or obligation.definition is None - or replace(verifier.key, definition=obligation.definition) != verifier.key + or not _same_definition(verifier.key, obligation.definition) or (obligation.report_definition_id, obligation.reporting_profile) != (verifier.key.report_definition_id, verifier.key.reporting_profile) or (binding.method, binding.transport, binding.format, binding.verification_profile) @@ -557,6 +559,25 @@ async def verify( raise ReportingWriterError(problem) +def _same_definition( + key: ReportingVerificationKey, definition: ReportingDefinitionBinding | None +) -> bool: + """Total, closed comparison against the frozen verification key. + + A retained Core definition is only loosely constrained, so one this strict + key contract cannot even express -- a versioned query URI, an uppercase or + short digest, a legacy dialect -- is a binding mismatch. Rebinding it must + never surface a raw ``ValueError`` through the closed failure boundary. + """ + rebound: ReportingVerificationKey | None = None + if definition is not None: + try: + rebound = replace(key, definition=definition) + except Exception: + rebound = None + return rebound == key + + def _session_binding( session: ReportingDestinationSession, request: ReportingDestinationRequest, diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py index 7a2f8bbc7..e0f56586c 100644 --- a/tests/conformance/reporting/test_reporting_materializer_contracts.py +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -173,6 +173,39 @@ def test_public_verification_key_rejects_credentials_in_every_definition_coordin assert "do-not-expose" not in str(caught.value) + repr(caught.value) +@pytest.mark.parametrize( + "field", ["report_definition_uri", "schema_uri", "schema_dialect", "schema_version"] +) +async def test_a_retained_definition_the_key_cannot_express_is_a_closed_binding_mismatch(field): + # ``ReportingDefinitionBinding`` is Core's loosely constrained retained + # record: it applies none of the verification key's HTTPS/digest/identifier + # screens, so real obligations carry values this key cannot express. Every + # public entry point must answer that with the closed BINDING_MISMATCH its + # callers already catch, never a raw ValueError through the failure contract. + case = await materializer_case() + loose = replace( + case.obligation.definition, + **{field: "https://provider.example.test/data?token=do-not-expose"}, + ) + obligation = replace(case.obligation, definition=loose) + with pytest.raises(ReportingWriterError) as caught: + await case.prepare(obligation=obligation) + assert caught.value.failure.code == "BINDING_MISMATCH" + assert case.writer.open_count == 0 + + locator = await case.io.write(case.prepared, context=io_context()) + prepared = replace(case.prepared, obligation=obligation) + for operation in ( + case.io.write(prepared, context=io_context()), + case.io.verify(prepared, locator, context=io_context()), + ): + with pytest.raises(ReportingWriterError) as caught: + await operation + assert caught.value.failure.code == "BINDING_MISMATCH" + assert "do-not-expose" not in str(caught.value) + repr(caught.value) + assert case.writer.open_count == case.writer.close_count == 1 + + async def test_service_heartbeat_is_a_checkpoint_only_and_fences_before_source_read(): case = await materializer_case() calls = 0 diff --git a/tests/conformance/reporting/test_reporting_revision_selection.py b/tests/conformance/reporting/test_reporting_revision_selection.py index 36bfc057e..028908981 100644 --- a/tests/conformance/reporting/test_reporting_revision_selection.py +++ b/tests/conformance/reporting/test_reporting_revision_selection.py @@ -1,10 +1,12 @@ """Whole-history selection is identical for Core, C, ingest and B1 preparation.""" +import asyncio from dataclasses import replace from itertools import permutations import pytest +from adcp.reporting.fixtures import SNAPSHOT_OFFERING_ID from adcp.reporting.ledger import ( InMemoryReportingLedgerStore, LedgerConflictError, @@ -26,6 +28,8 @@ from adcp.reporting.revision_selection import RevisionHistoryEntry from ._generation_support import NOW, UncalledSource, configuration, obligation_for, revision_for +from ._reliable_support import complete_fetch, reliable_factory +from ._reliable_support import configuration as reliable_configuration from .test_reporting_notification_outbox import statement @@ -234,3 +238,56 @@ def test_unreadable_official_never_falls_back_to_materialized_snapshot(): obligation, (snapshot, official), ledger_as_of=NOW, scope_closed=True ) assert result.health == "action_required" and result.current_revision is official + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_snapshot_restatement_after_an_official_close_keeps_one_chain(backend): + """A retained official close must never be mistaken for the snapshot leaf. + + Whole-history selection answers "what does this obligation currently + publish", and a unique official wins that outright. Reusing that answer to + choose ``supersedes`` roots the next restatement at ``None``, so the + obligation ends up with two snapshot roots -- ``disconnected_snapshot_history`` + over immutable rows, which every selector caller then parks for a repair + nothing in the SDK can perform. + """ + async with reliable_factory(backend) as h: + config = reliable_configuration("eur") + await h.store.put_configuration(config) + producer = h.producer(h.source(complete_fetch), managed=False) + (obligation,) = await producer.close_elapsed_periods(config) + + async def commit(observation, finality): + request = producer._build_slice( # noqa: SLF001 - the suite's slice fixture + config, obligation, SNAPSHOT_OFFERING_ID, now=h.clock(), observation=observation + ) + result = await producer._source.execute(request, cancel=asyncio.Event()) # noqa: SLF001 + return await h.commit_slice(producer, obligation, request, result, finality=finality) + + first = await commit(0, "snapshot") + second = await commit(1, "snapshot") + official = await commit(2, "official") + assert first.supersedes_reporting_revision_id is None + assert second.supersedes_reporting_revision_id == first.reporting_revision_id + assert official.supersedes_reporting_revision_id is None + + restated = await commit(3, "snapshot") + assert restated.supersedes_reporting_revision_id == second.reporting_revision_id + + history = await h.store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + selection = select_reporting_revision( + history, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + assert selection.kind == "selected" + assert selection.revision.reporting_revision_id == official.reporting_revision_id + projection = project_obligation_health( + obligation, history, ledger_as_of=h.clock(), scope_closed=True + ) + assert [i.code for i in projection.issues] == [] + assert projection.current_revision is not None diff --git a/tests/test_reporting_consumer_loop.py b/tests/test_reporting_consumer_loop.py index 9dd4f2d68..69b9a878c 100644 --- a/tests/test_reporting_consumer_loop.py +++ b/tests/test_reporting_consumer_loop.py @@ -774,9 +774,29 @@ def test_an_obligation_with_a_revision_and_no_reading_refuses_to_guess() -> None [_obligation(revision_count=1)], now=EXPECTED_AT + RECOVERY, automated_recovery_window=RECOVERY, + obligation_revisions={"rpo_1": [_revision()]}, ) +@pytest.mark.parametrize( + "history,count", + [(None, 1), ([], 1), ([_revision()], 2), ([_revision(), _revision()], 2)], +) +def test_an_incomplete_obligation_partition_names_obligation_revisions(history, count) -> None: + # The missing input is the obligation's retained history, not a reading: + # a message that says "reading" sends the adopter to fix the wrong + # argument and fail again on the next turn. + with pytest.raises(ConsumerStatusPlanError, match="obligation_revisions") as caught: + plan_consumer_statuses( + [_obligation(revision_count=count)], + now=EXPECTED_AT + RECOVERY, + automated_recovery_window=RECOVERY, + obligation_revisions=None if history is None else {"rpo_1": history}, + ) + assert "rpo_1" in str(caught.value) + assert "no reading was supplied" not in str(caught.value) + + def test_required_finality_decides_whether_a_revision_counts() -> None: # An obligation needing `official` is not satisfied by snapshots, so a # snapshot-only ledger really has no required revision. From 1c91311ec28d25506d5db43f59d0c34936ecb8f7 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 17 Sep 2026 00:57:35 +0000 Subject: [PATCH 3/5] ci: bound the Python matrix suite step and widen its job budget MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `Test Python 3.12` on head a89f9900 (run 35163238220, job 105018502311) was cancelled at the 30-minute job ceiling, not by a failing test: step timestamps put checkout, Python setup, the editable `[dev]` install, ruff, mypy, `mypy --strict` and the type-ignore contract at 1m27s, leaving the coverage suite 28m47s before the runner killed it — the log shows 99% of ~9.4k tests at 00:09:23 with no failure and no summary. The other three legs finish in 17-19 minutes because only 3.12 pays for coverage tracing on top of the same suite. A ceiling that cancels a passing job reports a red matrix and hides whatever the run was meant to tell us. Keep the complete suite and the coverage gate. Give the job 60 minutes so the step bound plus setup and cleanup still fit with margin for runner variance, and bound each pytest step at 45 minutes so a genuinely stuck suite fails that named step with its own timeout instead of silently consuming the job. Validated with actionlint 1.7.7 -- the version the workflow-security job pins -- over ci.yml and every other workflow. Workflow-only: `src/` and `tests/` are byte-identical to the previous commit, so that commit's local gate results carry over unchanged. Co-Authored-By: Claude Opus 5 (1M context) --- .github/workflows/ci.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ad1ebff25..958ecf34c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -58,7 +58,18 @@ jobs: test: name: Test Python ${{ matrix.python-version }} runs-on: ubuntu-latest - timeout-minutes: 30 + # The job ceiling is not the suite's budget: it also has to absorb + # checkout, Python setup, the editable [dev] install and, on 3.12 only, + # ruff/mypy/mypy --strict/the type-ignore contract before pytest starts, + # then post-job cleanup after it ends. On an ubuntu-latest runner that + # pre-test work is ~1m30s and the plain suite is ~16m, but 3.12 adds + # coverage tracing on ~9.4k tests: at 30 minutes that leg was cancelled + # mid-run at 99% with no failing test, which reports as a red matrix and + # hides real signal. Bound the suite itself below (so a hang fails one + # named step with its own message instead of silently taking the job), and + # leave the job enough room that step bound plus setup and cleanup still + # fit with margin for runner variance. + timeout-minutes: 60 strategy: matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] @@ -94,12 +105,18 @@ jobs: if: matrix.python-version == '3.12' run: python scripts/check_type_ignore_contract.py + # Bounded well above the observed ~16m (plain) and ~30m (coverage) runs + # so ordinary variance never trips it, and well below the job ceiling so + # a genuinely stuck suite still fails *this* step with a timeout rather + # than being cancelled as a whole job. - name: Run tests if: matrix.python-version != '3.12' + timeout-minutes: 45 run: pytest tests/ -v - name: Run tests with coverage if: matrix.python-version == '3.12' + timeout-minutes: 45 run: pytest tests/ -v --cov=src/adcp --cov-report=term-missing pg-conformance: From 157809d66e61103fbbdb236158e60fbabcf3c8ad Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 12:58:41 +0000 Subject: [PATCH 4/5] test(reporting): qualify destination controls against integrated rc6 --- .github/workflows/ci.yml | 2 +- docs/reporting-destination-writer.md | 9 +- .../conformance/reporting/_frozen_status_c.py | 3 +- ...est_reporting_materializer_verification.py | 3 +- .../test_reporting_status_selector_epoch.py | 36 ++++-- .../test_reporting_status_selector_pg.py | 119 ++++++++++++------ 6 files changed, 115 insertions(+), 57 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7d2bd263a..11a78e79e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -246,7 +246,7 @@ jobs: git fetch --no-tags --depth=1 origin \ 17ee407ae3978c8a2bb54437287afbf9dafb8130 \ 0f34c666ac1961e9832fce43ef0ef6937b3c1dde \ - ea150fabd5ad90e3abf93f89729d2919f1c61798 + 967b6e286301d7e5d089aea6fdbb90bea8ee5a16 - name: Set up Python 3.12 uses: actions/setup-python@v6 diff --git a/docs/reporting-destination-writer.md b/docs/reporting-destination-writer.md index 895aa9ed2..01a66bfc2 100644 --- a/docs/reporting-destination-writer.md +++ b/docs/reporting-destination-writer.md @@ -1,7 +1,7 @@ # Verified destination I/O — #1167B1 -**B1 of B1/B2**, stacked on reviewed #1168C at -`ea150fabd5ad90e3abf93f89729d2919f1c61798`. Refs #1167. +**B1 of B1/B2**, composed with integrated #1168C at +`967b6e286301d7e5d089aea6fdbb90bea8ee5a16`. Refs #1167. B1 supplies immutable public contracts, whole-history revision selection, SDK-owned source/destination verification, and a deterministic development @@ -170,3 +170,8 @@ old schema recreation, competing v2 projectors/sweepers, pool-local marker cleanup and retained physical rows. Shared memory/PG vectors cover interrupted fence, checkpoint/event and final-mark commits, ordered boundaries, late clocks, retained scopes, unchanged fingerprints and once-only restart convergence. + +The C control is the integrated rc.6 artifact above, including exact waiver +bindings and locale-independent catalog validation. These controls do not +qualify pre-`967b6e28` C binaries, including the earlier `ea150fab` snapshot. +The existing pre-`17ee407a` A and pre-`0f34c666` B rolling limits remain. diff --git a/tests/conformance/reporting/_frozen_status_c.py b/tests/conformance/reporting/_frozen_status_c.py index d7c1dea63..20e709f6e 100644 --- a/tests/conformance/reporting/_frozen_status_c.py +++ b/tests/conformance/reporting/_frozen_status_c.py @@ -48,7 +48,8 @@ async def execute(self, query, params=None, **kwargs): if gate == "project" or "SET lease_token=" in query: gate = None emit({"held": True}) - assert (await read())["action"] == "release_hold" + destination_operation_1 = await read() + assert (destination_operation_1)["action"] == "release_hold" return result async with AsyncConnectionPool( diff --git a/tests/conformance/reporting/test_reporting_materializer_verification.py b/tests/conformance/reporting/test_reporting_materializer_verification.py index e82a885d1..f14abb6ce 100644 --- a/tests/conformance/reporting/test_reporting_materializer_verification.py +++ b/tests/conformance/reporting/test_reporting_materializer_verification.py @@ -70,7 +70,8 @@ async def test_entire_source_and_destination_walk_in_every_supported_profile(cou "acct_a", "https://buyer.example.test/agents/reporting" ) assert result.verification.verification_path == capability.verification_path - assert await case.io.write(case.prepared, context=io_context()) == locator + destination_operation_1 = await case.io.write(case.prepared, context=io_context()) + assert destination_operation_1 == locator assert case.writer.write_effects == 1 and case.writer.open_count == case.writer.close_count == 3 assert case.writer.production_eligible is False assert (await case.store.get_materialization(case.attempt.key)).outcome is None diff --git a/tests/conformance/reporting/test_reporting_status_selector_epoch.py b/tests/conformance/reporting/test_reporting_status_selector_epoch.py index 71ffd7d5b..2780ae268 100644 --- a/tests/conformance/reporting/test_reporting_status_selector_epoch.py +++ b/tests/conformance/reporting/test_reporting_status_selector_epoch.py @@ -151,10 +151,12 @@ async def test_contaminated_epoch_corrects_changed_issue_set_once_and_preserves_ await mark_v1(h) before = {c.scope.checkpoint_key: c for c in await h.status.checkpoints(account_id="acct_a")} assert not await h.status.baseline_ready(account_id="acct_a") - assert (await h.status.rebuild_one()).events == 0 # Durable fence only. + destination_operation_1 = await h.status.rebuild_one() + assert (destination_operation_1).events == 0 # Durable fence only. fenced = await h.status.checkpoints(account_id="acct_a") assert all(c.selector_writer_floor == 2 and c.selector_semantics_version == 1 for c in fenced) - assert not await h.status.claim_due(account_id="acct_a") + destination_operation_2 = await h.status.claim_due(account_id="acct_a") + assert not destination_operation_2 await restart(h) await h.drain() events = await h.status.outbox.list_events(account_id="acct_a") @@ -173,8 +175,10 @@ async def test_contaminated_epoch_corrects_changed_issue_set_once_and_preserves_ assert current.generation == old.generation + (key in changed) assert current.selector_semantics_version == current.selector_writer_floor == 2 assert await h.status.baseline_ready(account_id="acct_a") - assert not (await h.status.rebuild_one()).did_work - assert not (await h.status.project_one(account_id="acct_a")).did_work + destination_operation_3 = await h.status.rebuild_one() + assert not (destination_operation_3).did_work + destination_operation_4 = await h.status.project_one(account_id="acct_a") + assert not (destination_operation_4).did_work assert await h.status.checkpoints(account_id="acct_a") == tuple(after.values()) assert await h.status.outbox.list_events(account_id="acct_a") == events @@ -236,7 +240,8 @@ async def test_captured_reversal_boundaries_precede_chronological_overdue_deadli assert [e.cause.health for e in late] == ["delayed", "action_required"] assert [e.cause.previous_health for e in late] == ["waiting", "delayed"] assert len({e.notification_id for e in events}) == len(events) - assert not (await ReportingStatusSweeper(h.status).run_once(account_id="acct_a")).did_work + destination_operation_5 = await ReportingStatusSweeper(h.status).run_once(account_id="acct_a") + assert not (destination_operation_5).did_work @pytest.mark.parametrize("fail_at", [1, 2, 4]) @@ -247,7 +252,8 @@ async def test_checkpoint_event_failure_rolls_back_epoch_turn_and_restart_is_onc await two_feeds(h, issue=False) await h.status.baseline(account_id="acct_a") await mark_v1(h) - assert (await h.status.rebuild_one()).did_work + destination_operation_6 = await h.status.rebuild_one() + assert (destination_operation_6).did_work await h.ledger.ensure_issue_opened( issue_key="late-public", account_id="acct_a", @@ -279,7 +285,8 @@ def crash(*args, **kwargs): await h.drain() events = await h.status.outbox.list_events(account_id="acct_a") assert len(events) == 4 and {e.cause.checkpoint_generation for e in events} == {1} - assert not (await h.status.rebuild_one()).did_work + destination_operation_7 = await h.status.rebuild_one() + assert not (destination_operation_7).did_work @pytest.mark.parametrize("phase", ["fence", "final_mark"]) @@ -291,7 +298,8 @@ async def test_crash_before_fence_or_final_mark_commit_is_restartable( await h.status.baseline(account_id="acct_a") await mark_v1(h) if phase == "final_mark": - assert (await h.status.rebuild_one()).did_work + destination_operation_12 = await h.status.rebuild_one() + assert (destination_operation_12).did_work before = await h.status.checkpoints(account_id="acct_a") name = "_rebuild" if isinstance(h.ledger, InMemoryReportingLedgerStore) else "_rebuild_on" original = getattr(h.status, name) @@ -346,7 +354,8 @@ async def test_retained_scopes_outside_current_discovery_are_reprojected_without (c.scope, c.fingerprint, c.generation, c.baseline) for c in before ] assert all(c.selector_semantics_version == 2 and c.next_due_at is None for c in after) - assert not (await h.status.rebuild_one()).did_work + destination_operation_8 = await h.status.rebuild_one() + assert not (destination_operation_8).did_work assert not await h.status.outbox.list_events(account_id="acct_a") @@ -364,7 +373,8 @@ async def test_account_epoch_readiness_and_new_baseline_are_isolated(status_harn await restart(h) assert await h.status.baseline_ready(account_id="acct_a") assert not await h.status.baseline_ready(account_id="acct_b") - assert await h.status.baseline(account_id="new-account") + destination_operation_9 = await h.status.baseline(account_id="new-account") + assert destination_operation_9 assert await h.status.baseline_ready(account_id="new-account") assert not await h.status.outbox.list_events(account_id="new-account") @@ -412,6 +422,8 @@ async def test_existing_c_service_discovers_old_accounts_without_an_account_list await h.status.baseline(account_id="acct_a") await mark_v1(h) service = ReportingStatusService(SimpleNamespace(store=h.status, account_ids=())) - assert await service.drain() >= 2 + destination_operation_10 = await service.drain() + assert destination_operation_10 >= 2 assert await h.status.baseline_ready(account_id="acct_a") - assert await service.drain() == 0 + destination_operation_11 = await service.drain() + assert destination_operation_11 == 0 diff --git a/tests/conformance/reporting/test_reporting_status_selector_pg.py b/tests/conformance/reporting/test_reporting_status_selector_pg.py index 96f6db711..7ac7fc318 100644 --- a/tests/conformance/reporting/test_reporting_status_selector_pg.py +++ b/tests/conformance/reporting/test_reporting_status_selector_pg.py @@ -24,10 +24,10 @@ from ._generation_support import ( NOW, - assert_c_collated_rolling_database, configuration, isolated_reporting_pool, obligation_for, + require_rolling_database, revision_for, ) from ._reliable_support import ( @@ -40,12 +40,12 @@ from .test_reporting_status_migration import C_QUEUES, physical_rows ROOT = Path(__file__).resolve().parents[3] -C_SHA = "ea150fabd5ad90e3abf93f89729d2919f1c61798" +C_SHA = "967b6e286301d7e5d089aea6fdbb90bea8ee5a16" @pytest.fixture(scope="module") def frozen_c(tmp_path_factory): - assert_c_collated_rolling_database() + require_rolling_database() root = tmp_path_factory.mktemp("frozen-c-selector") / "source" subprocess.run( ["git", "worktree", "add", "--detach", str(root), C_SHA], @@ -99,9 +99,11 @@ async def old_c(pool, source, *, now=NOW): child = OldC(process) try: await child.send(source=str(source), conninfo=pool.conninfo, kwargs=pool.kwargs, now=now) - assert Path((await child.receive())["origin"]).is_relative_to(source) + destination_operation_3 = await child.receive() + assert Path((destination_operation_3)["origin"]).is_relative_to(source) yield child - assert await child.call(action="stop") == {"stopped": True} + destination_operation_4 = await child.call(action="stop") + assert destination_operation_4 == {"stopped": True} await asyncio.wait_for(process.wait(), 10) assert process.returncode == 0 finally: @@ -146,17 +148,22 @@ async def populate_status_activity(status): now = datetime.now(timezone.utc) lease = await status.outbox.claim_delivery(account_id="acct_a", now=now, lease_seconds=60) assert lease is not None - assert await status.outbox.reserve_attempt( + destination_operation_1 = await status.outbox.reserve_attempt( lease, request=ActivityRequest("https://receiver.example.test/reporting", 1), now=now ) - assert await status.outbox.finish_delivery(lease, state="pending", retry_at=now, now=now) + assert destination_operation_1 + destination_operation_2 = await status.outbox.finish_delivery( + lease, state="pending", retry_at=now, now=now + ) + assert destination_operation_2 async def test_actual_c_claim_fence_pool_marker_two_projectors_two_sweepers_and_source(frozen_c): from psycopg_pool import AsyncConnectionPool async with isolated_reporting_pool(autocommit=True) as owner, old_c(owner, frozen_c) as old: - assert await old.call(action="schema") == {"result": True} + destination_operation_5 = await old.call(action="schema") + assert destination_operation_5 == {"result": True} clock = [NOW] ledger = PgReportingReconciliationStore( pool=owner, notifications=True, clock=lambda: clock[0] @@ -166,15 +173,18 @@ async def test_actual_c_claim_fence_pool_marker_two_projectors_two_sweepers_and_ await seed(ledger, account="acct_b") baseline_at = waiting.period.expected_at - timedelta(seconds=1) for account in ("acct_a", "acct_b"): - assert await old.call(action="baseline", account=account, now=baseline_at) == { - "result": True - } - assert await old.call(action="ready", account=account) == {"result": True} + destination_operation_34 = await old.call( + action="baseline", account=account, now=baseline_at + ) + assert destination_operation_34 == {"result": True} + destination_operation_35 = await old.call(action="ready", account=account) + assert destination_operation_35 == {"result": True} # A populated C event and every C activity table exist before cutover. await old.call( action="source", revision=ready.reporting_revision_id, readable=False, now=baseline_at ) - assert (await old.call(action="project"))["result"]["events"] == 2 + destination_operation_6 = await old.call(action="project") + assert (destination_operation_6)["result"]["events"] == 2 status = PgStatusNotificationStore(ledger) await populate_status_activity(status) immutable = await retained_physical_rows(owner) @@ -184,18 +194,21 @@ async def test_actual_c_claim_fence_pool_marker_two_projectors_two_sweepers_and_ assert await retained_physical_rows(owner) == immutable assert await physical_rows(owner) == queue_rows assert not await status.baseline_ready(account_id="acct_a") - assert await old.call(action="ready") == {"result": True} # Schema alone isn't cutover. + destination_operation_7 = await old.call(action="ready") + assert destination_operation_7 == {"result": True} # Schema alone isn't cutover. # Old claim owns only a checkpoint row. The new fence owns the account # then waits for that row; the old trigger must never acquire account. await old.send(action="claim", now=NOW, hold="claim") - assert await old.receive() == {"held": True} + destination_operation_8 = await old.receive() + assert destination_operation_8 == {"held": True} fencing = asyncio.create_task(status.project_one(account_id="acct_a")) await asyncio.sleep(0.02) assert not fencing.done() await old.send(action="release_hold") old_lease = (await old.receive())["result"] assert old_lease is not None - assert (await asyncio.wait_for(fencing, 10)).events == 0 + destination_operation_9 = await asyncio.wait_for(fencing, 10) + assert (destination_operation_9).events == 0 # The fence preserves existing lease identity; old claims/completion # fail before a write, while v2 migration doesn't wait for lease expiry. assert any( @@ -206,13 +219,19 @@ async def test_actual_c_claim_fence_pool_marker_two_projectors_two_sweepers_and_ owner, tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), ) - assert (await old.call(action="claim"))["error"] == "database_fence" - assert (await old.call(action="complete", lease=0))["error"] == "status_policy_conflict" - assert (await old.call(action="release", lease=0))["error"] == "database_fence" - assert (await old.call(action="ready"))["error"] == "status_policy_conflict" - assert (await old.call(action="project"))["error"] == "status_policy_conflict" + destination_operation_10 = await old.call(action="claim") + assert (destination_operation_10)["error"] == "database_fence" + destination_operation_11 = await old.call(action="complete", lease=0) + assert (destination_operation_11)["error"] == "status_policy_conflict" + destination_operation_12 = await old.call(action="release", lease=0) + assert (destination_operation_12)["error"] == "database_fence" + destination_operation_13 = await old.call(action="ready") + assert (destination_operation_13)["error"] == "status_policy_conflict" + destination_operation_14 = await old.call(action="project") + assert (destination_operation_14)["error"] == "status_policy_conflict" assert await physical_rows(owner, tables=tuple(fenced_rows)) == fenced_rows - assert await old.call(action="schema") == {"result": True} + destination_operation_15 = await old.call(action="schema") + assert destination_operation_15 == {"result": True} assert await physical_rows(owner, tables=tuple(fenced_rows)) == fenced_rows async with owner.connection() as c: await validate_status_schema(c, activity=True) @@ -277,10 +296,15 @@ async def sweep(s): await pool.close() assert await status.baseline_ready(account_id="acct_a") assert not await status.baseline_ready(account_id="acct_b") - assert (await old.call(action="ready"))["error"] == "status_policy_conflict" - assert await old.call(action="ready", account="acct_b") == {"result": True} + destination_operation_16 = await old.call(action="ready") + assert (destination_operation_16)["error"] == "status_policy_conflict" + destination_operation_17 = await old.call(action="ready", account="acct_b") + assert destination_operation_17 == {"result": True} clock[0] = waiting.automated_recovery_deadline_at - assert (await ReportingStatusSweeper(status).run_once(account_id="acct_a")).did_work + destination_operation_18 = await ReportingStatusSweeper(status).run_once( + account_id="acct_a" + ) + assert (destination_operation_18).did_work events = await status.outbox.list_events(account_id="acct_a") late = sorted( ( @@ -299,7 +323,8 @@ async def sweep(s): ) == len(events) after = await physical_rows(owner) assert all(all(row in after[table] for row in rows) for table, rows in queue_rows.items()) - assert await status.baseline(account_id="brand-new") + destination_operation_19 = await status.baseline(account_id="brand-new") + assert destination_operation_19 assert await status.baseline_ready(account_id="brand-new") assert not await status.outbox.list_events(account_id="brand-new") @@ -308,12 +333,14 @@ async def test_actual_c_unique_looking_snapshot_corruption_corrects_without_base frozen_c, ): async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: - assert await old.call(action="schema") == {"result": True} + destination_operation_20 = await old.call(action="schema") + assert destination_operation_20 == {"result": True} ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) obligation, _ = await seed(ledger, readable=True) second, rows = revision_for(obligation, suffix="disconnected") await ledger.commit_revision(second, rows) # v1 persistence accepts another snapshot root. - assert await old.call(action="baseline") == {"result": True} + destination_operation_21 = await old.call(action="baseline") + assert destination_operation_21 == {"result": True} status = PgStatusNotificationStore(ledger) immutable = await retained_physical_rows(pool) await status.create_schema() @@ -338,22 +365,27 @@ async def test_actual_c_unique_looking_snapshot_corruption_corrects_without_base pool, tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), ) - assert not (await status.rebuild_one()).did_work - assert not (await status.project_one(account_id="acct_a")).did_work + destination_operation_22 = await status.rebuild_one() + assert not (destination_operation_22).did_work + destination_operation_23 = await status.project_one(account_id="acct_a") + assert not (destination_operation_23).did_work assert await physical_rows(pool, tables=tuple(physical)) == physical async def test_actual_c_inflight_projector_commits_before_v2_fence_and_source_serializes(frozen_c): async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: - assert await old.call(action="schema") == {"result": True} + destination_operation_24 = await old.call(action="schema") + assert destination_operation_24 == {"result": True} ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) _, revision = await seed(ledger, readable=True) - assert await old.call(action="baseline") == {"result": True} + destination_operation_25 = await old.call(action="baseline") + assert destination_operation_25 == {"result": True} await old.call(action="source", revision=revision.reporting_revision_id, readable=False) status = PgStatusNotificationStore(ledger) await status.create_schema() await old.send(action="project", hold="project") - assert await old.receive() == {"held": True} + destination_operation_26 = await old.receive() + assert destination_operation_26 == {"held": True} fence = asyncio.create_task(status.project_one(account_id="acct_a")) publication = asyncio.create_task( ledger.set_revision_readable( @@ -365,8 +397,10 @@ async def test_actual_c_inflight_projector_commits_before_v2_fence_and_source_se await asyncio.sleep(0.02) assert not fence.done() and not publication.done() await old.send(action="release_hold") - assert (await old.receive())["result"]["events"] == 2 - assert (await asyncio.wait_for(fence, 10)).events == 0 + destination_operation_27 = await old.receive() + assert (destination_operation_27)["result"]["events"] == 2 + destination_operation_28 = await asyncio.wait_for(fence, 10) + assert (destination_operation_28).events == 0 await asyncio.wait_for(publication, 10) old_events = await physical_rows(pool) await drain(status) @@ -379,15 +413,18 @@ async def test_actual_c_inflight_projector_commits_before_v2_fence_and_source_se ) after = await physical_rows(pool) assert all(all(row in after[table] for row in rows) for table, rows in old_events.items()) - assert (await old.call(action="project"))["error"] == "status_policy_conflict" + destination_operation_29 = await old.call(action="project") + assert (destination_operation_29)["error"] == "status_policy_conflict" async def test_populated_selector_sql_rollback_and_repeated_concurrent_install(frozen_c): async with isolated_reporting_pool(autocommit=True) as pool, old_c(pool, frozen_c) as old: - assert await old.call(action="schema") == {"result": True} + destination_operation_30 = await old.call(action="schema") + assert destination_operation_30 == {"result": True} ledger = PgReportingReconciliationStore(pool=pool, notifications=True, clock=lambda: NOW) await seed(ledger, readable=True) - assert await old.call(action="baseline") == {"result": True} + destination_operation_31 = await old.call(action="baseline") + assert destination_operation_31 == {"result": True} rows = await physical_rows( pool, tables=("reporting_status_accounts", "reporting_status_scope_checkpoints", *C_QUEUES), @@ -404,13 +441,15 @@ async def test_populated_selector_sql_rollback_and_repeated_concurrent_install(f raise RuntimeError("interrupt migration") assert await physical_rows(pool, tables=tuple(rows)) == rows assert await retained_physical_rows(pool) == immutable - assert await old.call(action="ready") == {"result": True} + destination_operation_32 = await old.call(action="ready") + assert destination_operation_32 == {"result": True} status = PgStatusNotificationStore(ledger) await asyncio.wait_for(asyncio.gather(status.create_schema(), status.create_schema()), 30) await status.create_schema() assert await retained_physical_rows(pool) == immutable assert not await status.baseline_ready(account_id="acct_a") - assert await old.call(action="ready") == {"result": True} + destination_operation_33 = await old.call(action="ready") + assert destination_operation_33 == {"result": True} await drain(status) assert await status.baseline_ready(account_id="acct_a") assert not await status.outbox.list_events(account_id="acct_a") From 58c82997ef82b8b5a5996eaa070c142fa488ba87 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 13:29:16 +0000 Subject: [PATCH 5/5] fix(reporting): compose service capabilities outside producer extras --- src/adcp/reporting/service.py | 16 ++++++++-------- .../test_reporting_materializer_contracts.py | 5 +++-- tests/test_reliable_reporting_service.py | 13 ++++++++++--- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/src/adcp/reporting/service.py b/src/adcp/reporting/service.py index 50675297e..6d97c4e5f 100644 --- a/src/adcp/reporting/service.py +++ b/src/adcp/reporting/service.py @@ -781,21 +781,21 @@ def capability_block(self) -> dict[str, Any]: offerings[offering_id] = offering configurations = [binding.configuration for binding in self._bindings.values()] first = next(iter(self._bindings.values())).producer - extra: dict[str, Any] = { - "managed_delivery": self._materialization_worker is not None, - "reconciled_billing": self._reconciled_billing, - } - if self._reconciled_billing: - extra["receipt_task"] = "sync_reporting_receipts" - return first.advertised_reporting_delivery( + payload = first.advertised_reporting_delivery( consumer_status_task=self._consumer_status_enabled, offerings=[offerings[key] for key in sorted(offerings)], automated_recovery_window=max( item.automated_recovery_window for item in configurations ), status_retention_days=min(item.status_retention_days for item in configurations), - extra=extra, ) + # The service owns these installed-component declarations; they are not + # caller-provided producer extensions. + payload["managed_delivery"] = self._materialization_worker is not None + payload["reconciled_billing"] = self._reconciled_billing + if self._reconciled_billing: + payload["receipt_task"] = "sync_reporting_receipts" + return payload def inject_capabilities(self, response: Any) -> dict[str, Any]: """Merge the truthful reporting block into a base capability response.""" diff --git a/tests/conformance/reporting/test_reporting_materializer_contracts.py b/tests/conformance/reporting/test_reporting_materializer_contracts.py index e0f56586c..e164315c5 100644 --- a/tests/conformance/reporting/test_reporting_materializer_contracts.py +++ b/tests/conformance/reporting/test_reporting_materializer_contracts.py @@ -49,7 +49,8 @@ "delivery_task", ], ) -def test_producer_extra_cannot_inject_sdk_owned_readiness(key): +@pytest.mark.parametrize("value", [False, True]) +def test_producer_extra_cannot_inject_sdk_owned_readiness(key, value): producer = ReportingProducer( source=UncalledSource(), offerings=ProducerOfferings(), store=InMemoryReportingLedgerStore() ) @@ -59,7 +60,7 @@ def test_producer_extra_cannot_inject_sdk_owned_readiness(key): offerings=(), automated_recovery_window=timedelta(hours=1), status_retention_days=30, - extra={key: True}, + extra={key: value}, ) diff --git a/tests/test_reliable_reporting_service.py b/tests/test_reliable_reporting_service.py index 0f0c69097..e62500ff1 100644 --- a/tests/test_reliable_reporting_service.py +++ b/tests/test_reliable_reporting_service.py @@ -226,9 +226,12 @@ async def get_media_buy_delivery(self, params: Any, context: Any = None) -> Any: assert {"get_products", "get_reporting_status", "get_media_buy_delivery"} <= tools -async def test_capability_block_is_schema_valid_and_only_advertises_installed_tiers() -> None: +@pytest.mark.parametrize("consumer_status_enabled", [False, True]) +async def test_capability_block_is_schema_valid_and_only_advertises_installed_tiers( + consumer_status_enabled: bool, +) -> None: service = ReliableReportingService.memory( - account_context=_account_context, consumer_status_enabled=True + account_context=_account_context, consumer_status_enabled=consumer_status_enabled ) service.sources.register("gam", ScriptedReportingAdapter(redacted_capabilities(), [_rows(1)])) await service.configure(_configuration()) @@ -236,7 +239,11 @@ async def test_capability_block_is_schema_valid_and_only_advertises_installed_ti assert block["managed_delivery"] is False assert block["reconciled_billing"] is False - assert block["consumer_status_task"] == "sync_reporting_status" + assert "receipt_task" not in block + if consumer_status_enabled: + assert block["consumer_status_task"] == "sync_reporting_status" + else: + assert "consumer_status_task" not in block validator = get_named_validator("core/reporting-delivery-capabilities.json") assert validator is not None assert list(validator.iter_errors(block)) == []