From f5f94e04d6facf92ddcdc1d9bfb7a056b648b181 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 25 Sep 2026 11:18:54 +0000 Subject: [PATCH 1/3] feat(reporting): persist buyer receipt submission intents --- .../ledger/reporting_buyer_submissions.sql | 33 ++ src/adcp/reporting/submissions/__init__.py | 55 ++ src/adcp/reporting/submissions/models.py | 540 ++++++++++++++++++ src/adcp/reporting/submissions/pg.py | 255 +++++++++ src/adcp/reporting/submissions/store.py | 143 +++++ src/adcp/reporting/submissions/submit.py | 183 ++++++ .../reporting/_buyer_submission_installed.py | 168 ++++++ .../reporting/_buyer_submission_process.py | 145 +++++ .../reporting/_buyer_submission_support.py | 143 +++++ ...st_reporting_buyer_submission_migration.py | 99 ++++ ...st_reporting_buyer_submission_packaging.py | 156 +++++ ...test_reporting_buyer_submission_process.py | 168 ++++++ .../test_reporting_buyer_submission_seller.py | 109 ++++ ..._reporting_buyer_submission_uncertainty.py | 260 +++++++++ .../test_reporting_buyer_submissions.py | 346 +++++++++++ .../reporting_buyer_submission_intents.py | 86 +++ 16 files changed, 2889 insertions(+) create mode 100644 src/adcp/reporting/ledger/reporting_buyer_submissions.sql create mode 100644 src/adcp/reporting/submissions/__init__.py create mode 100644 src/adcp/reporting/submissions/models.py create mode 100644 src/adcp/reporting/submissions/pg.py create mode 100644 src/adcp/reporting/submissions/store.py create mode 100644 src/adcp/reporting/submissions/submit.py create mode 100644 tests/conformance/reporting/_buyer_submission_installed.py create mode 100644 tests/conformance/reporting/_buyer_submission_process.py create mode 100644 tests/conformance/reporting/_buyer_submission_support.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submission_migration.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submission_packaging.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submission_process.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submission_seller.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submission_uncertainty.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submissions.py create mode 100644 tests/type_checks/reporting_buyer_submission_intents.py diff --git a/src/adcp/reporting/ledger/reporting_buyer_submissions.sql b/src/adcp/reporting/ledger/reporting_buyer_submissions.sql new file mode 100644 index 000000000..c45810ac1 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_buyer_submissions.sql @@ -0,0 +1,33 @@ +-- Buyer-only additive persistence. Use a dedicated buyer schema/pool. +-- Never expire/delete a pending intent: uncertainty has no safe time limit. +CREATE TABLE IF NOT EXISTS reporting_buyer_submission_scopes ( + scope_sha256 text NOT NULL, + canonical_identity text NOT NULL, + current_submission_id text, + CONSTRAINT reporting_buyer_submission_scopes_pkey PRIMARY KEY (scope_sha256), + CONSTRAINT reporting_buyer_scope_digest CHECK (scope_sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT reporting_buyer_scope_bound CHECK (octet_length(canonical_identity) <= 32768) +); + +CREATE TABLE IF NOT EXISTS reporting_buyer_submission_intents ( + scope_sha256 text NOT NULL, + submission_id text NOT NULL, + canonical_plan text NOT NULL, + plan_sha256 text NOT NULL, + confirmed_results text NOT NULL, + confirmed_sha256 text NOT NULL, + pending boolean NOT NULL, + CONSTRAINT reporting_buyer_submission_intents_pkey PRIMARY KEY (scope_sha256, submission_id), + CONSTRAINT reporting_buyer_submission_scope_fk FOREIGN KEY (scope_sha256) + REFERENCES reporting_buyer_submission_scopes (scope_sha256), + CONSTRAINT reporting_buyer_submission_id CHECK ( + submission_id ~ '^reporting-submission:[a-f0-9]{64}$' + ), + CONSTRAINT reporting_buyer_plan_digest CHECK (plan_sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT reporting_buyer_confirmed_digest CHECK (confirmed_sha256 ~ '^[a-f0-9]{64}$'), + CONSTRAINT reporting_buyer_plan_bound CHECK (octet_length(canonical_plan) <= 16777216), + CONSTRAINT reporting_buyer_confirmed_bound CHECK (octet_length(confirmed_results) <= 16777216) +); + +CREATE UNIQUE INDEX IF NOT EXISTS reporting_buyer_one_pending_scope + ON reporting_buyer_submission_intents (scope_sha256) WHERE pending; diff --git a/src/adcp/reporting/submissions/__init__.py b/src/adcp/reporting/submissions/__init__.py new file mode 100644 index 000000000..c4cff83d4 --- /dev/null +++ b/src/adcp/reporting/submissions/__init__.py @@ -0,0 +1,55 @@ +"""Buyer-only durable receipt submission, separate from reconciliation planning. + +Public imports are additive and work without the optional PostgreSQL driver. +The application supplies a trusted authorizer, a receipt client, and an explicit +intent store. Use PgReportingSubmissionIntentStore for restart durability; the +memory implementation is a volatile reference for tests. No checkpoint, seller +service, consumer-status, or client.reporting facade behavior is changed. + +Rollout: use a dedicated buyer schema/pool; explicitly call create_schema before +enabling submissions. The migration adds only reporting_buyer_submission_*. +Retain pending intents indefinitely and resume them after any uncertain result. +Do not drop the tables on rollback or replace pending plans with new receipt IDs. +Disable new planning while recovering, then resume the same stored scope with +current authorization. Exact final-head review precedes facade integration. +""" + +from adcp.reporting.submissions.models import ( + ReportingReceiptFailureCode, + ReportingReceiptOutcome, + ReportingReceiptSubmission, + ReportingSubmissionCode, + ReportingSubmissionError, + ReportingSubmissionReceipt, + ReportingSubmissionResult, + ReportingSubmissionScope, + prepare_reporting_receipt_submission, +) +from adcp.reporting.submissions.pg import PgReportingSubmissionIntentStore +from adcp.reporting.submissions.store import ( + InMemoryReportingSubmissionIntentStore, + ReportingSubmissionIntentStore, +) +from adcp.reporting.submissions.submit import ( + ReportingReceiptSubmissionClient, + ReportingSubmissionAuthorizer, + submit_reporting_receipts, +) + +__all__ = [ + "InMemoryReportingSubmissionIntentStore", + "PgReportingSubmissionIntentStore", + "ReportingReceiptFailureCode", + "ReportingReceiptOutcome", + "ReportingReceiptSubmission", + "ReportingReceiptSubmissionClient", + "ReportingSubmissionAuthorizer", + "ReportingSubmissionCode", + "ReportingSubmissionError", + "ReportingSubmissionIntentStore", + "ReportingSubmissionReceipt", + "ReportingSubmissionResult", + "ReportingSubmissionScope", + "prepare_reporting_receipt_submission", + "submit_reporting_receipts", +] diff --git a/src/adcp/reporting/submissions/models.py b/src/adcp/reporting/submissions/models.py new file mode 100644 index 000000000..8920e7c46 --- /dev/null +++ b/src/adcp/reporting/submissions/models.py @@ -0,0 +1,540 @@ +"""Immutable buyer intent records and closed diagnostics. + +These records describe submission, not evidence selection or reconciliation. +The caller must validate its receipt plan against the complete seller history. +Received adjustment evidence must be retained separately, before model parsing. +""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Sequence +from dataclasses import dataclass, field, replace +from enum import Enum +from typing import Any, Literal, TypeAlias + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import consumer_reference, principal_reference +from adcp.reporting.outbox.identity import canonical_consumer +from adcp.reporting.receipts.wire import ReceiptBatch, validate_receipt_response +from adcp.types import ( + ReportingAdjustmentReceipt, + ReportingReceipt, + SyncReportingReceiptsRequest, + SyncReportingReceiptsResponse, +) + +ReportingSubmissionReceipt: TypeAlias = ReportingReceipt | ReportingAdjustmentReceipt +ReceiptKind: TypeAlias = Literal["receipt", "adjustment_receipt"] +ReceiptResult: TypeAlias = Literal["recorded", "unchanged", "failed"] + +MAX_SUBMISSION_RECEIPTS = 10_000 +MAX_SUBMISSION_BYTES = 16 * 1024 * 1024 +MAX_CONFIRMATION_BYTES = 16 * 1024 * 1024 +MAX_CHUNK_BYTES = 1024 * 1024 +MAX_RESPONSE_BYTES = 2 * MAX_CHUNK_BYTES +_CHUNK_SIZE = 100 +_VERSION = "3.2-rc.6" + + +class ReportingSubmissionCode(str, Enum): + INVALID_SCOPE = "INVALID_SUBMISSION_SCOPE" + INVALID_PLAN = "INVALID_SUBMISSION_PLAN" + UNAUTHORIZED = "SUBMISSION_NOT_AUTHORIZED" + NOT_FOUND = "SUBMISSION_NOT_FOUND" + HISTORY_CORRUPT = "SUBMISSION_HISTORY_CORRUPT" + STORAGE_UNAVAILABLE = "SUBMISSION_STORAGE_UNAVAILABLE" + PG_REQUIRED = "SUBMISSION_PG_REQUIRED" + INVALID_RESPONSE = "SUBMISSION_RESPONSE_INVALID" + TRANSPORT_UNCERTAIN = "SUBMISSION_TRANSPORT_UNCERTAIN" + RESPONSE_UNCONFIRMED = "SUBMISSION_RESPONSE_UNCONFIRMED" + + +class ReportingSubmissionError(RuntimeError): + """Closed code only: no provider, authentication, SQL or wire body details.""" + + def __init__(self, code: ReportingSubmissionCode) -> None: + self.code = ( + code + if isinstance(code, ReportingSubmissionCode) + else ReportingSubmissionCode.INVALID_PLAN + ) + super().__init__(self.code.value) + + +class ReportingReceiptFailureCode(str, Enum): + """Known item failures; unrecognized seller codes map to UNKNOWN. + + Seller messages/details are deliberately not persisted or exposed. Each + failed item and each error position is retained, including unknown codes. + """ + + UNKNOWN = "UNKNOWN" + INVALID_REQUEST = "INVALID_REQUEST" + UNAUTHORIZED = "UNAUTHORIZED" + RATE_LIMITED = "RATE_LIMITED" + NOT_SUPPORTED = "NOT_SUPPORTED" + IDEMPOTENCY_CONFLICT = "IDEMPOTENCY_CONFLICT" + INVALID_REPORTING_RECORD = "INVALID_REPORTING_RECORD" + REPORTING_RECORD_UNAVAILABLE = "REPORTING_RECORD_UNAVAILABLE" + REPORTING_HISTORY_CORRUPT = "REPORTING_HISTORY_CORRUPT" + REPORTING_IDENTITY_CONFLICT = "REPORTING_IDENTITY_CONFLICT" + REPORTING_TIME_INVALID = "REPORTING_TIME_INVALID" + RECEIPTS_NOT_ENABLED = "RECEIPTS_NOT_ENABLED" + RECEIVED_AT_READ_ONLY = "RECEIVED_AT_READ_ONLY" + RECEIPT_PROFILE_MISMATCH = "RECEIPT_PROFILE_MISMATCH" + RECEIPT_TOTALS_MISMATCH = "RECEIPT_TOTALS_MISMATCH" + RECEIPT_EVIDENCE_MISMATCH = "RECEIPT_EVIDENCE_MISMATCH" + MATERIALIZATION_UNREADABLE = "MATERIALIZATION_UNREADABLE" + ADJUSTMENT_REQUIRES_OFFICIAL = "ADJUSTMENT_REQUIRES_OFFICIAL" + ADJUSTMENT_ORDER_INVALID = "ADJUSTMENT_ORDER_INVALID" + ADJUSTMENT_DIGEST_MISMATCH = "ADJUSTMENT_DIGEST_MISMATCH" + ACCEPTED_RECEIPT_TERMINAL = "ACCEPTED_RECEIPT_TERMINAL" + + +@dataclass(frozen=True) +class ReportingSubmissionScope: + """Identity resolved by a trusted authorization adapter, never from a request. + + ``seller_id`` identifies the configured seller, ``account_id`` is the + seller-resolved account and ``consumer_id`` is the canonical authenticated + principal. Syntax checks cannot establish provenance: the required + authorizer must bind all three to the exact client and its current access. + No credential or natural-key account assertion belongs in these fields. + """ + + seller_id: str = field(repr=False) + account_id: str = field(repr=False) + consumer_id: str = field(repr=False) + + def __post_init__(self) -> None: + valid = False + try: + consumer_reference(self.seller_id) + principal_reference(self.account_id) + canonical_consumer(self.consumer_id) + valid = all(value == value.strip() for value in (self.seller_id, self.account_id)) + except (ValueError, TypeError, RuntimeError): + pass + if not valid: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_SCOPE) + + @property + def canonical_identity(self) -> bytes: + return canonical_json_utf8_v1([self.seller_id, self.account_id, self.consumer_id]) + + @property + def storage_key(self) -> str: + # Index a fixed-size digest, not a possibly 2048-character principal. + # Stores also compare the full identity to fail closed on collisions. + return hashlib.sha256(self.canonical_identity).hexdigest() + + +def _receipt(kind: ReceiptKind, body: dict[str, Any]) -> ReportingSubmissionReceipt: + if kind == "receipt": + return ReportingReceipt.model_validate(body) + return ReportingAdjustmentReceipt.model_validate(body) + + +@dataclass(frozen=True) +class ReportingReceiptOutcome: + """One confirmed item, in original input order, with fresh model views.""" + + ordinal: int + kind: ReceiptKind + result: ReceiptResult + error_codes: tuple[ReportingReceiptFailureCode, ...] = () + _submitted: bytes = field(default=b"", repr=False) + _stored: bytes | None = field(default=None, repr=False) + + @property + def submitted_receipt(self) -> ReportingSubmissionReceipt: + return _receipt(self.kind, json.loads(self._submitted)) + + @property + def receipt(self) -> ReportingSubmissionReceipt | None: + """The seller's actual stored receipt on success, including received_at.""" + return _receipt(self.kind, json.loads(self._stored)) if self._stored is not None else None + + +@dataclass(frozen=True) +class ReportingReceiptSubmission: + """Exact reserved plan and its immutable, fully confirmed chunk prefix. + + Construct with :func:`prepare_reporting_receipt_submission`. Stores validate + persisted records before returning them. Private bytes keep mutable caller + models, driver results, and representations out of the retained identity. + """ + + scope: ReportingSubmissionScope = field(repr=False) + submission_id: str = field(repr=False) + _plan: bytes = field(repr=False) + _confirmed: tuple[bytes, ...] = field(default=(), repr=False) + + @property + def chunk_count(self) -> int: + return len(json.loads(self._plan)["requests"]) + + @property + def confirmed_chunks(self) -> int: + return len(self._confirmed) + + @property + def pending(self) -> bool: + return self.confirmed_chunks < self.chunk_count + + def request(self, ordinal: int) -> SyncReportingReceiptsRequest: + """Return a fresh typed view of one exact persisted request body.""" + body = json.loads(self._plan)["requests"][ordinal] + return SyncReportingReceiptsRequest.model_validate(body) + + @property + def outcomes(self) -> tuple[ReportingReceiptOutcome, ...]: + plan = json.loads(self._plan) + confirmed = { + item["reporting_receipt_id"]: item + for chunk in self._confirmed + for item in json.loads(chunk) + } + outcomes = [] + for ordinal, item in enumerate(plan["items"]): + result = confirmed.get(item["body"]["reporting_receipt_id"]) + if result is not None: + stored = result.get("receipt") + outcomes.append( + ReportingReceiptOutcome( + ordinal, + item["kind"], + result["result"], + tuple(ReportingReceiptFailureCode(code) for code in result["errors"]), + canonical_json_utf8_v1(item["body"]), + canonical_json_utf8_v1(stored) if stored is not None else None, + ) + ) + return tuple(outcomes) + + +@dataclass(frozen=True) +class ReportingSubmissionResult: + """Submission outcomes only; completion does not establish reconciliation. + + ``proposal_deferred`` means an earlier pending scope reservation was resumed + instead. Inspect its outcomes before planning any subsequent submission. + """ + + submission: ReportingReceiptSubmission + proposal_deferred: bool = False + diagnostic: ReportingSubmissionCode | None = None + + @property + def pending(self) -> bool: + return self.submission.pending + + @property + def outcomes(self) -> tuple[ReportingReceiptOutcome, ...]: + return self.submission.outcomes + + @property + def submitted_receipts(self) -> tuple[ReportingSubmissionReceipt, ...]: + return tuple( + receipt for outcome in self.outcomes if (receipt := outcome.receipt) is not None + ) + + +def prepare_reporting_receipt_submission( + scope: ReportingSubmissionScope, + receipts: Sequence[ReportingSubmissionReceipt], +) -> ReportingReceiptSubmission: + """Freeze an already validated receipt plan, with at most 100 items per call. + + Outbound typed values are normalized once, then retained byte-for-byte. + This is not a capture or proof of raw inbound adjustment evidence. Requests + contain the authorized resolved account only, with no caller-provided + account, idempotency key, context, extensions or credentials. + """ + result = None + try: + if ( + type(scope) is not ReportingSubmissionScope + or not 1 <= len(receipts) <= MAX_SUBMISSION_RECEIPTS + ): + raise ValueError + items: list[dict[str, Any]] = [] + identifiers: set[str] = set() + size = 0 + for receipt in receipts: + if isinstance(receipt, ReportingReceipt): + kind = "receipt" + elif isinstance(receipt, ReportingAdjustmentReceipt): + kind = "adjustment_receipt" + else: + raise ValueError + body = receipt.model_dump(mode="json", exclude_none=True) + identifier = body["reporting_receipt_id"] + if identifier in identifiers or "received_at" in body: + raise ValueError + identifiers.add(identifier) + item = {"kind": kind, "body": body} + size += len(canonical_json_utf8_v1(item)) + if size > MAX_SUBMISSION_BYTES // 2: + raise ValueError + items.append(item) + fingerprint = hashlib.sha256( + scope.canonical_identity + b"\n" + canonical_json_utf8_v1(items) + ).hexdigest() + requests = [] + for offset in range(0, len(items), _CHUNK_SIZE): + request: dict[str, Any] = { + "adcp_version": _VERSION, + "account": {"account_id": scope.account_id}, + "idempotency_key": f"reporting-buyer:{fingerprint}:{offset // _CHUNK_SIZE}", + } + for kind, name in ( + ("receipt", "receipts"), + ("adjustment_receipt", "adjustment_receipts"), + ): + values = [ + item["body"] + for item in items[offset : offset + _CHUNK_SIZE] + if item["kind"] == kind + ] + if values: + request[name] = values + encoded = canonical_json_utf8_v1(request) + if len(encoded) > MAX_CHUNK_BYTES: + raise ValueError + ReceiptBatch.parse(request) + requests.append(request) + plan = canonical_json_utf8_v1({"version": 1, "items": items, "requests": requests}) + if len(plan) > MAX_SUBMISSION_BYTES: + raise ValueError + result = ReportingReceiptSubmission(scope, f"reporting-submission:{fingerprint}", plan) + except Exception: + result = None + if result is None: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_PLAN) + return result + + +def validate_submission(submission: ReportingReceiptSubmission) -> None: + """Check every supplied/persisted byte and confirmed result before use.""" + valid = False + try: + if type(submission._plan) is not bytes or len(submission._plan) > MAX_SUBMISSION_BYTES: + raise ValueError + plan = json.loads(submission._plan) + if ( + type(plan) is not dict + or set(plan) != {"version", "items", "requests"} + or plan["version"] != 1 + ): + raise ValueError + if not 1 <= len(plan["items"]) <= MAX_SUBMISSION_RECEIPTS: + raise ValueError + rebuilt = prepare_reporting_receipt_submission( + submission.scope, + [_receipt(item["kind"], item["body"]) for item in plan["items"]], + ) + if rebuilt._plan != submission._plan or rebuilt.submission_id != submission.submission_id: + raise ValueError + if ( + not isinstance(submission._confirmed, tuple) + or len(submission._confirmed) > len(plan["requests"]) + or _confirmation_bytes(submission._confirmed) > MAX_CONFIRMATION_BYTES + ): + raise ValueError + for ordinal, chunk in enumerate(submission._confirmed): + if type(chunk) is not bytes or len(chunk) > MAX_RESPONSE_BYTES: + raise ValueError + if _restore_confirmation(submission, ordinal, chunk) != chunk: + raise ValueError + valid = True + except Exception: + valid = False + if not valid: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + + +def _confirmation( + submission: ReportingReceiptSubmission, + ordinal: int, + response: SyncReportingReceiptsResponse, +) -> bytes: + """Validate complete unique coverage and exact immutable success bodies.""" + result = None + try: + body = response.model_dump(mode="json", exclude_none=True) + if len(canonical_json_utf8_v1(body)) > MAX_RESPONSE_BYTES: + raise ValueError + batch = ReceiptBatch.parse(json.loads(submission._plan)["requests"][ordinal]) + results = body["results"] + by_id = {} + for entry in results: + identifier = ( + entry["reporting_receipt_id"] + if entry["result"] == "failed" + else entry.get("receipt", entry.get("adjustment_receipt"))["reporting_receipt_id"] + ) + if identifier in by_id: + raise ValueError + by_id[identifier] = entry + expected = [item["reporting_receipt_id"] for _, item in batch.items] + if set(by_id) != set(expected): + raise ValueError + # Sellers may reorder results. Validate their actual schema after matching + # exact IDs, then restore the original *mixed* input order in outcomes. + body["results"] = [by_id[identifier] for identifier in expected] + validate_receipt_response(body, batch) + normalized = [] + known = {code.value for code in ReportingReceiptFailureCode} + for kind, submitted in batch.items: + identifier = submitted["reporting_receipt_id"] + entry = by_id[identifier] + errors = [] + stored = None + if entry["result"] == "failed": + errors = [ + code if (code := error["code"]) in known else "UNKNOWN" + for error in entry["errors"] + ] + else: + key = "receipt" if kind == "revision_receipt" else "adjustment_receipt" + stored = entry[key] + immutable = {key: value for key, value in stored.items() if key != "received_at"} + if canonical_json_utf8_v1(immutable) != canonical_json_utf8_v1(submitted): + raise ValueError + normalized.append( + { + "reporting_receipt_id": identifier, + "result": entry["result"], + "errors": errors, + "receipt": stored, + } + ) + result = canonical_json_utf8_v1(normalized) + if len(result) > MAX_RESPONSE_BYTES: + raise ValueError + except Exception: + result = None + if result is None: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_RESPONSE) + return result + + +def _restore_confirmation( + submission: ReportingReceiptSubmission, ordinal: int, encoded: bytes +) -> bytes: + """Revalidate retained sanitized evidence, including exact result coverage.""" + kinds = { + item["body"]["reporting_receipt_id"]: item["kind"] + for item in json.loads(submission._plan)["items"] + } + results = [] + for item in json.loads(encoded): + if item["result"] == "failed": + if item["receipt"] is not None: + raise ValueError + results.append( + { + "result": "failed", + "reporting_receipt_id": item["reporting_receipt_id"], + "errors": [ + {"code": code, "message": "receipt submission failed"} + for code in item["errors"] + ], + } + ) + else: + if item["errors"]: + raise ValueError + results.append( + {"result": item["result"], kinds[item["reporting_receipt_id"]]: item["receipt"]} + ) + response = SyncReportingReceiptsResponse.model_validate({"results": results}) + return _confirmation(submission, ordinal, response) + + +def _confirmation_bytes(chunks: tuple[bytes, ...]) -> int: + # Each chunk is canonical JSON. Account for the enclosing array/commas as + # well, so memory and PostgreSQL enforce the same cumulative storage bound. + return 2 + sum(map(len, chunks)) + max(0, len(chunks) - 1) + + +def confirm_submission( + submission: ReportingReceiptSubmission, + ordinal: int, + response: SyncReportingReceiptsResponse, +) -> ReportingReceiptSubmission: + if ( + type(ordinal) is not int + or not 0 <= ordinal < submission.chunk_count + or ordinal > submission.confirmed_chunks + ): + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + confirmed = _confirmation(submission, ordinal, response) + if ordinal < submission.confirmed_chunks: + # First confirmation is immutable. Concurrent exact replays may use + # recorded/unchanged differently, but must agree on the actual evidence + # and failures. A contradictory reply cannot overwrite retained results. + previous = json.loads(submission._confirmed[ordinal]) + repeated = json.loads(confirmed) + for results in (previous, repeated): + for item in results: + if item["result"] in {"recorded", "unchanged"}: + item["result"] = "recorded" + if previous != repeated: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_RESPONSE) + return submission + chunks = (*submission._confirmed, confirmed) + if _confirmation_bytes(chunks) > MAX_CONFIRMATION_BYTES: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_RESPONSE) + return replace(submission, _confirmed=chunks) + + +def encode_submission(submission: ReportingReceiptSubmission) -> tuple[str, str, str, str]: + """Exact text/hashes for PostgreSQL; JSONB is not the identity store.""" + confirmed = canonical_json_utf8_v1([json.loads(chunk) for chunk in submission._confirmed]) + return ( + submission._plan.decode(), + hashlib.sha256(submission._plan).hexdigest(), + confirmed.decode(), + hashlib.sha256(confirmed).hexdigest(), + ) + + +def decode_submission( + scope: ReportingSubmissionScope, row: Sequence[Any] +) -> ReportingReceiptSubmission: + result = None + try: + identifier, plan, plan_digest, confirmed, confirmed_digest, pending = row + if ( + len(plan.encode()) > MAX_SUBMISSION_BYTES + or len(confirmed.encode()) > MAX_CONFIRMATION_BYTES + ): + raise ValueError + if ( + hashlib.sha256(plan.encode()).hexdigest() != plan_digest + or hashlib.sha256(confirmed.encode()).hexdigest() != confirmed_digest + ): + raise ValueError + candidate = ReportingReceiptSubmission( + scope, + identifier, + plan.encode(), + tuple(canonical_json_utf8_v1(chunk) for chunk in json.loads(confirmed)), + ) + validate_submission(candidate) + if ( + type(pending) is not bool + or pending != candidate.pending + or encode_submission(candidate) != (plan, plan_digest, confirmed, confirmed_digest) + ): + raise ValueError + result = candidate + except Exception: + result = None + if result is None: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return result diff --git a/src/adcp/reporting/submissions/pg.py b/src/adcp/reporting/submissions/pg.py new file mode 100644 index 000000000..50d1ff19e --- /dev/null +++ b/src/adcp/reporting/submissions/pg.py @@ -0,0 +1,255 @@ +"""PostgreSQL buyer intent storage with short transactions and no network locks. + +The application owns the pool and its lifecycle. ``create_schema`` installs only +the isolated buyer tables, without modifying seller schemas or existing APIs. +Importing this module does not require psycopg; actual use requires adcp[pg]. +""" + +from __future__ import annotations + +from collections.abc import Callable, Coroutine +from functools import wraps +from importlib.resources import files +from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar + +from adcp.reporting.submissions.models import ( + ReportingReceiptSubmission, + ReportingSubmissionCode, + ReportingSubmissionError, + ReportingSubmissionScope, + confirm_submission, + decode_submission, + encode_submission, + validate_submission, +) +from adcp.types import SyncReportingReceiptsResponse + +if TYPE_CHECKING: + from psycopg_pool import AsyncConnectionPool + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def _closed_storage_errors( + fn: Callable[_P, Coroutine[Any, Any, _R]], +) -> Callable[_P, Coroutine[Any, Any, _R]]: + @wraps(fn) + async def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _R: + try: + return await fn(*args, **kwargs) + except ReportingSubmissionError as error: + code = error.code + except Exception: + code = ReportingSubmissionCode.STORAGE_UNAVAILABLE + # Outside both handlers: even __context__ cannot contain driver details. + raise ReportingSubmissionError(code) + + return wrapped + + +class PgReportingSubmissionIntentStore: + """Production persistence for optional buyer receipt submission intents. + + Pass a trusted application-owned psycopg AsyncConnectionPool, preferably + with a dedicated schema/search_path and bounded connection/statement waits. + There is no credential or connection string in this object's representation. + Run create_schema explicitly during rollout, then validate custom backends + with the same memory/PostgreSQL state-machine vectors. + """ + + def __init__(self, *, pool: AsyncConnectionPool) -> None: + self._pool = pool + + @_closed_storage_errors + async def create_schema(self) -> None: + available = False + try: + import psycopg # noqa: F401 + + available = True + except ImportError: + pass + if not available: + raise ReportingSubmissionError(ReportingSubmissionCode.PG_REQUIRED) + async with self._pool.connection() as connection, connection.transaction(): + await connection.execute("SELECT pg_advisory_xact_lock(%s)", (712071172,)) + sql = ( + files("adcp.reporting.ledger") + .joinpath("reporting_buyer_submissions.sql") + .read_text() + ) + await connection.execute(sql) + await self._check_schema_on(connection) + + async def _check_schema_on(self, connection: Any) -> None: + # These are the concurrency invariants, not an IF NOT EXISTS assumption. + rows = await ( + await connection.execute( + "SELECT c.conname,c.contype,c.convalidated,c.condeferrable," + " pg_get_constraintdef(c.oid) FROM pg_constraint c" + " WHERE c.conrelid IN ('reporting_buyer_submission_scopes'::regclass," + " 'reporting_buyer_submission_intents'::regclass)" + ) + ).fetchall() + constraints = {row[0]: tuple(row[1:]) for row in rows} + required = { + "reporting_buyer_submission_scopes_pkey": ( + "p", + True, + False, + "PRIMARY KEY (scope_sha256)", + ), + "reporting_buyer_submission_intents_pkey": ( + "p", + True, + False, + "PRIMARY KEY (scope_sha256, submission_id)", + ), + "reporting_buyer_submission_scope_fk": ( + "f", + True, + False, + "FOREIGN KEY (scope_sha256) REFERENCES " + "reporting_buyer_submission_scopes(scope_sha256)", + ), + } + if any(constraints.get(name) != expected for name, expected in required.items()): + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + index = await ( + await connection.execute( + "SELECT i.indisunique,i.indisvalid,i.indisready,i.indnkeyatts," + " pg_get_indexdef(i.indexrelid,1,true),pg_get_expr(i.indpred,i.indrelid)," + " i.indrelid='reporting_buyer_submission_intents'::regclass" + " FROM pg_index i WHERE i.indexrelid='reporting_buyer_one_pending_scope'::regclass" + ) + ).fetchone() + if index is None or tuple(index) != (True, True, True, 1, "scope_sha256", "pending", True): + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + + async def _scope_on( + self, connection: Any, scope: ReportingSubmissionScope, *, create: bool = False + ) -> tuple[bool, str | None]: + if create: + await connection.execute( + "INSERT INTO reporting_buyer_submission_scopes" + " (scope_sha256,canonical_identity) VALUES (%s,%s)" + " ON CONFLICT (scope_sha256) DO NOTHING", + (scope.storage_key, scope.canonical_identity.decode()), + ) + row = await ( + await connection.execute( + "SELECT canonical_identity,current_submission_id" + " FROM reporting_buyer_submission_scopes WHERE scope_sha256=%s FOR UPDATE", + (scope.storage_key,), + ) + ).fetchone() + if row is None: + return False, None + if row[0] != scope.canonical_identity.decode(): + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + if row[1] is None: + existing = await ( + await connection.execute( + "SELECT 1 FROM reporting_buyer_submission_intents" + " WHERE scope_sha256=%s LIMIT 1", + (scope.storage_key,), + ) + ).fetchone() + if existing is not None: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return True, row[1] + + async def _read_on( + self, connection: Any, scope: ReportingSubmissionScope, submission_id: str + ) -> ReportingReceiptSubmission | None: + row = await ( + await connection.execute( + "SELECT submission_id,canonical_plan,plan_sha256,confirmed_results," + " confirmed_sha256,pending FROM reporting_buyer_submission_intents" + " WHERE scope_sha256=%s AND submission_id=%s", + (scope.storage_key, submission_id), + ) + ).fetchone() + return decode_submission(scope, row) if row is not None else None + + @_closed_storage_errors + async def reserve(self, proposed: ReportingReceiptSubmission) -> ReportingReceiptSubmission: + validate_submission(proposed) + if proposed.confirmed_chunks: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_PLAN) + scope = proposed.scope + async with self._pool.connection() as connection, connection.transaction(): + _, current_id = await self._scope_on(connection, scope, create=True) + if current_id is not None: + current = await self._read_on(connection, scope, current_id) + if current is None: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + if current.pending: + return current + previous = await self._read_on(connection, scope, proposed.submission_id) + if previous is not None: + if previous._plan != proposed._plan: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return previous + plan, plan_digest, confirmed, confirmed_digest = encode_submission(proposed) + await connection.execute( + "INSERT INTO reporting_buyer_submission_intents" + " (scope_sha256,submission_id,canonical_plan,plan_sha256," + " confirmed_results,confirmed_sha256,pending) VALUES (%s,%s,%s,%s,%s,%s,true)", + ( + scope.storage_key, + proposed.submission_id, + plan, + plan_digest, + confirmed, + confirmed_digest, + ), + ) + await connection.execute( + "UPDATE reporting_buyer_submission_scopes SET current_submission_id=%s" + " WHERE scope_sha256=%s", + (proposed.submission_id, scope.storage_key), + ) + return proposed + + @_closed_storage_errors + async def get( + self, scope: ReportingSubmissionScope, submission_id: str | None = None + ) -> ReportingReceiptSubmission | None: + async with self._pool.connection() as connection, connection.transaction(): + exists, current_id = await self._scope_on(connection, scope) + if not exists or current_id is None: + return None + current = await self._read_on(connection, scope, current_id) + if current is None: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + if submission_id is None or submission_id == current_id: + return current + return await self._read_on(connection, scope, submission_id) + + @_closed_storage_errors + async def confirm( + self, + scope: ReportingSubmissionScope, + submission_id: str, + chunk: int, + response: SyncReportingReceiptsResponse, + ) -> ReportingReceiptSubmission: + async with self._pool.connection() as connection, connection.transaction(): + exists, current_id = await self._scope_on(connection, scope) + state = await self._read_on(connection, scope, submission_id) if exists else None + if state is None: + raise ReportingSubmissionError(ReportingSubmissionCode.NOT_FOUND) + if state.pending and current_id != submission_id: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + updated = confirm_submission(state, chunk, response) + if updated is not state: + _, _, confirmed, digest = encode_submission(updated) + await connection.execute( + "UPDATE reporting_buyer_submission_intents" + " SET confirmed_results=%s,confirmed_sha256=%s,pending=%s" + " WHERE scope_sha256=%s AND submission_id=%s", + (confirmed, digest, updated.pending, scope.storage_key, submission_id), + ) + return updated diff --git a/src/adcp/reporting/submissions/store.py b/src/adcp/reporting/submissions/store.py new file mode 100644 index 000000000..893224dda --- /dev/null +++ b/src/adcp/reporting/submissions/store.py @@ -0,0 +1,143 @@ +"""Optional buyer intent protocol, independent of ReportingCheckpointStore.""" + +from __future__ import annotations + +import asyncio +from typing import Protocol + +from adcp.reporting.submissions.models import ( + ReportingReceiptSubmission, + ReportingSubmissionCode, + ReportingSubmissionError, + ReportingSubmissionScope, + confirm_submission, + validate_submission, +) +from adcp.types import SyncReportingReceiptsResponse + + +class ReportingSubmissionIntentStore(Protocol): + """Atomic exact-request reservation and monotonic confirmed chunk storage. + + Adopters implementing this protocol must run the shared store vectors against + their durable backend. A scope has one unresolved reservation, never a TTL, + lease expiry, or automatic abandonment. Each mutation commits before return. + No transport call may run while holding a storage transaction or lock. + + The supplied scope is already resolved/authorized by a trusted adapter. The + store is an internal persistence boundary, not a request authentication API. + Old ReportingCheckpointStore implementations need no new methods. + """ + + async def reserve(self, proposed: ReportingReceiptSubmission) -> ReportingReceiptSubmission: + """Atomically reserve or return the scope's earlier unresolved intent. + + A different proposal must not replace an uncertain request. An exact + already completed proposal returns its retained outcomes. Preserve all + request bytes, chunk keys/order and confirmed outcomes across restart. + """ + ... + + async def get( + self, scope: ReportingSubmissionScope, submission_id: str | None = None + ) -> ReportingReceiptSubmission | None: + """Read a validated reservation, including completed work after a lost return. + + With no ID return the latest reserved intent; do not erase it when it + completes. Never resolve an ID outside the exact trusted scope. + """ + ... + + async def confirm( + self, + scope: ReportingSubmissionScope, + submission_id: str, + chunk: int, + response: SyncReportingReceiptsResponse, + ) -> ReportingReceiptSubmission: + """Atomically acknowledge full unique ID/body coverage for the next chunk. + + Retain item successes AND failures in the original plan order. Confirmed + chunks never change. A concurrent duplicate acknowledgement returns the + retained state; it cannot replace an earlier confirmed outcome. + """ + ... + + +class InMemoryReportingSubmissionIntentStore: + """Volatile reference implementation for tests; it cannot survive restart. + + Production callers must explicitly supply a durable implementation such as + PgReportingSubmissionIntentStore. There is no implicit in-memory fallback. + """ + + def __init__(self) -> None: + self._lock = asyncio.Lock() + self._scopes: dict[str, tuple[bytes, str]] = {} + self._submissions: dict[tuple[str, str], ReportingReceiptSubmission] = {} + + def _current(self, scope: ReportingSubmissionScope) -> ReportingReceiptSubmission | None: + entry = self._scopes.get(scope.storage_key) + if entry is None: + return None + if entry[0] != scope.canonical_identity: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + current = self._submissions.get((scope.storage_key, entry[1])) + if current is None or current.scope != scope: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + validate_submission(current) + return current + + async def reserve(self, proposed: ReportingReceiptSubmission) -> ReportingReceiptSubmission: + validate_submission(proposed) + if proposed.confirmed_chunks: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_PLAN) + async with self._lock: + current = self._current(proposed.scope) + if current is not None and current.pending: + return current + key = proposed.scope.storage_key, proposed.submission_id + prior = self._submissions.get(key) + if prior is not None: + validate_submission(prior) + if prior._plan != proposed._plan or prior.scope != proposed.scope: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return prior + self._submissions[key] = proposed + self._scopes[key[0]] = proposed.scope.canonical_identity, proposed.submission_id + return proposed + + async def get( + self, scope: ReportingSubmissionScope, submission_id: str | None = None + ) -> ReportingReceiptSubmission | None: + async with self._lock: + current = self._current(scope) + if current is None or submission_id is None: + return current + result = self._submissions.get((scope.storage_key, submission_id)) + if result is not None: + validate_submission(result) + if result.scope != scope: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return result + + async def confirm( + self, + scope: ReportingSubmissionScope, + submission_id: str, + chunk: int, + response: SyncReportingReceiptsResponse, + ) -> ReportingReceiptSubmission: + async with self._lock: + current = self._current(scope) + state = self._submissions.get((scope.storage_key, submission_id)) + if current is None or state is None: + raise ReportingSubmissionError(ReportingSubmissionCode.NOT_FOUND) + validate_submission(state) + if state.scope != scope or ( + state.pending and state.submission_id != current.submission_id + ): + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + updated = confirm_submission(state, chunk, response) + self._submissions[(scope.storage_key, submission_id)] = updated + return updated diff --git a/src/adcp/reporting/submissions/submit.py b/src/adcp/reporting/submissions/submit.py new file mode 100644 index 000000000..841cbbe44 --- /dev/null +++ b/src/adcp/reporting/submissions/submit.py @@ -0,0 +1,183 @@ +"""Explicit low-level receipt submission with durable uncertainty semantics.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Awaitable, Sequence +from math import isfinite +from typing import Protocol, TypeVar + +from adcp.reporting.submissions.models import ( + ReportingReceiptSubmission, + ReportingSubmissionCode, + ReportingSubmissionError, + ReportingSubmissionReceipt, + ReportingSubmissionResult, + ReportingSubmissionScope, + prepare_reporting_receipt_submission, + validate_submission, +) +from adcp.reporting.submissions.store import ReportingSubmissionIntentStore +from adcp.types import SyncReportingReceiptsRequest, SyncReportingReceiptsResponse +from adcp.types.core import TaskResult + + +class ReportingReceiptSubmissionClient(Protocol): + async def sync_reporting_receipts( + self, request: SyncReportingReceiptsRequest + ) -> TaskResult[SyncReportingReceiptsResponse]: ... + + +class ReportingSubmissionAuthorizer(Protocol): + """Trusted application adapter resolving the exact client's current access. + + Resolve seller identity from trusted client/registry configuration, account + from an authorized seller account lookup, and consumer from authenticated + credentials/registry. Never source these identities from receipt payloads, + an asserted account reference, or a debug/raw response. Aliases must already + be resolved. Raise on revoked access or an identity disagreement. + + Called before any store read/reservation, before each send, and before + returning completed cached outcomes. It must authorize replay afresh too. + """ + + async def __call__( + self, client: ReportingReceiptSubmissionClient + ) -> ReportingSubmissionScope: ... + + +async def _authorize( + client: ReportingReceiptSubmissionClient, + authorizer: ReportingSubmissionAuthorizer, + expected: ReportingSubmissionScope | None = None, +) -> ReportingSubmissionScope: + resolved = None + try: + scope = await authorizer(client) + if type(scope) is ReportingSubmissionScope: + scope.__post_init__() + if expected is None or scope == expected: + resolved = scope + except Exception: + resolved = None + if resolved is None: + raise ReportingSubmissionError(ReportingSubmissionCode.UNAUTHORIZED) + return resolved + + +_T = TypeVar("_T") + + +async def _storage(operation: Awaitable[_T]) -> _T: + try: + return await operation + except ReportingSubmissionError as error: + code = error.code + except Exception: + code = ReportingSubmissionCode.STORAGE_UNAVAILABLE + raise ReportingSubmissionError(code) + + +def _check_state( + state: ReportingReceiptSubmission, + scope: ReportingSubmissionScope, + previous: ReportingReceiptSubmission | None = None, +) -> None: + validate_submission(state) + if state.scope != scope or ( + previous is not None + and ( + previous.submission_id != state.submission_id + or previous._plan != state._plan + or state._confirmed[: previous.confirmed_chunks] != previous._confirmed + ) + ): + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + + +async def submit_reporting_receipts( + client: ReportingReceiptSubmissionClient, + *, + authorizer: ReportingSubmissionAuthorizer, + store: ReportingSubmissionIntentStore, + receipts: Sequence[ReportingSubmissionReceipt] | None = None, + timeout_seconds: float = 30.0, +) -> ReportingSubmissionResult: + """Reserve or resume a receipt plan without replacing uncertain work. + + Omit ``receipts`` to resume the latest intent for the freshly authorized + scope, including its completed outcomes after a lost return. Otherwise the + supplied plan is frozen and atomically reserved before ANY network write. + An earlier pending scope intent takes priority, with proposal_deferred=True. + The caller must reconsider the deferred plan against fresh seller history. + + A transport failure, timeout, failed task or malformed response leaves the + chunk uncertain and returns pending=True. Retry uses the exact durable body + and idempotency key. Cancellation propagates with the intent still retained. + Storage failure raises a closed error; resume the stored scope because the + last commit may have succeeded. There is no expiry/abandon/replacement path. + + Every item failure is retained alongside successes. Completion only means + every submission outcome is confirmed; it does not mean every receipt was + recorded, accepted, readable, or sufficient for definitive reconciliation. + This API does not select history, build adjustment evidence, post consumer + statuses, or modify the original reconciliation/checkpoint interfaces. + """ + if ( + isinstance(timeout_seconds, bool) + or not isinstance(timeout_seconds, (int, float)) + or not isfinite(timeout_seconds) + or timeout_seconds <= 0 + ): + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_PLAN) + scope = await _authorize(client, authorizer) + proposed = ( + prepare_reporting_receipt_submission(scope, receipts) if receipts is not None else None + ) + state = await _storage(store.reserve(proposed) if proposed is not None else store.get(scope)) + if state is None: + raise ReportingSubmissionError(ReportingSubmissionCode.NOT_FOUND) + _check_state(state, scope) + deferred = proposed is not None and proposed.submission_id != state.submission_id + while state.pending: + await _authorize(client, authorizer, scope) + chunk = state.confirmed_chunks + request = state.request(chunk) + response = None + try: + response = await asyncio.wait_for( + client.sync_reporting_receipts(request), timeout=timeout_seconds + ) + except Exception: + response = None + if response is None: + return ReportingSubmissionResult( + state, deferred, ReportingSubmissionCode.TRANSPORT_UNCERTAIN + ) + if ( + response.success is not True + or response.status != "completed" + or not isinstance(response.data, SyncReportingReceiptsResponse) + ): + return ReportingSubmissionResult( + state, deferred, ReportingSubmissionCode.RESPONSE_UNCONFIRMED + ) + invalid = False + try: + updated = await _storage( + store.confirm(scope, state.submission_id, chunk, response.data) + ) + except ReportingSubmissionError as error: + if error.code != ReportingSubmissionCode.INVALID_RESPONSE: + raise + invalid = True + if invalid: + return ReportingSubmissionResult( + state, deferred, ReportingSubmissionCode.INVALID_RESPONSE + ) + _check_state(updated, scope, state) + if updated.confirmed_chunks <= chunk: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + state = updated + await _authorize(client, authorizer, scope) + return ReportingSubmissionResult(state, deferred) diff --git a/tests/conformance/reporting/_buyer_submission_installed.py b/tests/conformance/reporting/_buyer_submission_installed.py new file mode 100644 index 000000000..dd6ebbb5a --- /dev/null +++ b/tests/conformance/reporting/_buyer_submission_installed.py @@ -0,0 +1,168 @@ +"""Public Python 3.10 wheel checks, executed with -I outside the source tree.""" + +import asyncio +import hashlib +import importlib +import importlib.util +import json +import sys +from importlib.resources import files +from pathlib import Path + + +async def main(): + settings = json.loads(sys.stdin.read()) + assert sys.version_info[:2] == (3, 10) + workspace = Path(settings["workspace"]).resolve() + origins = {} + for name, expected in settings["modules"].items(): + module = importlib.import_module(name) + origin = Path(module.__file__).resolve() + assert "site-packages" in str(origin) and not origin.is_relative_to(workspace) + assert hashlib.sha256(origin.read_bytes()).hexdigest() == expected + origins[name] = str(origin) + assert not any(Path(entry).resolve().is_relative_to(workspace) for entry in sys.path) + + from adcp.reporting.submissions import ( + InMemoryReportingSubmissionIntentStore, + PgReportingSubmissionIntentStore, + ReportingSubmissionCode, + ReportingSubmissionError, + ReportingSubmissionScope, + submit_reporting_receipts, + ) + from adcp.types import ( + ReportingAdjustment, + ReportingAdjustmentReceipt, + ReportingReceipt, + SyncReportingReceiptsResponse, + ) + from adcp.types.core import TaskResult, TaskStatus + + assert ReportingAdjustment.__name__ == "ReportingAdjustment" + sql = files("adcp.reporting.ledger").joinpath("reporting_buyer_submissions.sql").read_bytes() + assert hashlib.sha256(sql).hexdigest() == settings["sql_sha256"] + if not settings["drivers"]: + assert importlib.util.find_spec("psycopg") is None + assert importlib.util.find_spec("psycopg_pool") is None + try: + await PgReportingSubmissionIntentStore(pool=None).create_schema() + except ReportingSubmissionError as error: + assert error.code == ReportingSubmissionCode.PG_REQUIRED + assert error.__context__ is None + else: + raise AssertionError("missing PG driver was accepted") + + inputs = [] + for index in range(201): + common = { + "reporting_receipt_id": f"installed-receipt-{index:06d}", + "status": "accepted", + "observed_at": "2026-09-01T01:00:00Z", + } + if index % 2 == 0: + inputs.append( + ReportingAdjustmentReceipt.model_validate( + { + **common, + "reporting_adjustment_id": f"adjustment-{index}", + "adjusts_reporting_revision_id": f"revision-{index}", + "observed_adjustment_sha256": "b" * 64, + } + ) + ) + else: + inputs.append( + ReportingReceipt.model_validate( + { + **common, + "reporting_obligation_id": f"obligation-{index}", + "reporting_revision_id": f"revision-{index}", + "reporting_materialization_id": f"materialization-{index}", + "verification_profile": "manifest_checksums", + "observed_row_count": 0, + "observed_control_totals": [], + "observed_manifest_sha256": "a" * 64, + } + ) + ) + requests = [] + + class Client: + async def sync_reporting_receipts(self, request): + body = request.model_dump(mode="json", exclude_none=True) + requests.append(body) + results = [] + for kind, key in ( + ("receipt", "receipts"), + ("adjustment_receipt", "adjustment_receipts"), + ): + for item in body.get(key, []): + if item["reporting_receipt_id"] == inputs[0].reporting_receipt_id: + results.append( + { + "result": "failed", + "reporting_receipt_id": item["reporting_receipt_id"], + "errors": [ + { + "code": "UNKNOWN_PRIVATE_SENTINEL", + "message": "password=PRIVATE_SENTINEL", + } + ], + } + ) + else: + results.append( + { + "result": "recorded", + kind: {**item, "received_at": "2026-09-01T02:00:00Z"}, + } + ) + # Lose the second response after the exact body has been sent. + if len(requests) == 2: + raise TimeoutError("password=PRIVATE_SENTINEL") + results.reverse() + return TaskResult( + status=TaskStatus.COMPLETED, + data=SyncReportingReceiptsResponse.model_validate({"results": results}), + ) + + client = Client() + scope = ReportingSubmissionScope("seller", "account", "https://buyer.example.test/agent") + + async def authorize(candidate): + assert candidate is client + return scope + + store = InMemoryReportingSubmissionIntentStore() + first = await submit_reporting_receipts( + client, authorizer=authorize, store=store, receipts=inputs + ) + assert first.pending and len(first.outcomes) == 100 + result = await submit_reporting_receipts(client, authorizer=authorize, store=store) + assert not result.pending and len(result.outcomes) == 201 + assert len(result.submitted_receipts) == 200 and result.outcomes[0].result == "failed" + assert [outcome.submitted_receipt for outcome in result.outcomes] == inputs + assert requests[1] == requests[2] and len(requests) == 4 + assert "PRIVATE_SENTINEL" not in repr(result) + replay = await submit_reporting_receipts(client, authorizer=authorize, store=store) + assert replay == result and len(requests) == 4 + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + print( + json.dumps( + { + "python": "3.10", + "outcomes": 201, + "submitted": 200, + "origins": origins, + "drivers": settings["drivers"], + } + ) + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_buyer_submission_process.py b/tests/conformance/reporting/_buyer_submission_process.py new file mode 100644 index 000000000..b625f9075 --- /dev/null +++ b/tests/conformance/reporting/_buyer_submission_process.py @@ -0,0 +1,145 @@ +"""Independent buyer + actual seller PG worker, paused for external SIGKILL.""" + +import asyncio +import hashlib +import importlib +import json +import sys +from pathlib import Path + + +async def main(): + settings = json.loads(await asyncio.to_thread(sys.stdin.readline)) + from psycopg import AsyncConnection + from psycopg_pool import AsyncConnectionPool + + from adcp.reporting.ledger import ReportingDeliveryPrincipal + from adcp.reporting.receipts import PgReportingReceiptStore + from adcp.reporting.submissions import ( + PgReportingSubmissionIntentStore, + ReportingSubmissionScope, + submit_reporting_receipts, + ) + from adcp.types import ( + ReportingAdjustmentReceipt, + ReportingReceipt, + SyncReportingReceiptsResponse, + ) + from adcp.types.core import TaskResult, TaskStatus + + origins = {} + if settings.get("installed"): + installed = settings["installed"] + assert list(sys.version_info[:2]) == [3, 10] + workspace = Path(installed["workspace"]).resolve() + for name, digest in installed["modules"].items(): + module = importlib.import_module(name) + origin = Path(module.__file__).resolve() + assert hashlib.sha256(origin.read_bytes()).hexdigest() == digest + assert "site-packages" in str(origin) and not origin.is_relative_to(workspace) + origins[name] = str(origin) + assert not any(Path(entry).resolve().is_relative_to(workspace) for entry in sys.path) + + async def pause(point): + if settings.get("pause") == point: + print(json.dumps({"point": point}), flush=True) + command = json.loads(await asyncio.to_thread(sys.stdin.readline)) + assert command["continue"] + + class Connection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + result = await super().execute(query, params, **kwargs) + if isinstance(query, str): + for prefix, point in ( + ("INSERT INTO reporting_buyer_submission_scopes", "scope_row"), + ("INSERT INTO reporting_buyer_submission_intents", "intent_row"), + ("UPDATE reporting_buyer_submission_intents", "confirmation_row"), + ): + if query.startswith(prefix): + await pause(point) + return result + + class BuyerStore(PgReportingSubmissionIntentStore): + async def reserve(self, proposed): + await pause("before_intent") + state = await super().reserve(proposed) + await pause("intent_committed") + return state + + async def confirm(self, scope, submission_id, chunk, response): + await pause("response_delivered") + state = await super().confirm(scope, submission_id, chunk, response) + await pause("confirmation_committed") + return state + + # This is trusted worker configuration from the parent test's authenticated + # fixture registry. Neither account nor principal is taken from a request. + scope = ReportingSubmissionScope(**settings["trusted_scope"]) + async with AsyncConnectionPool( + settings["conninfo"], + kwargs=settings["kwargs"], + min_size=1, + max_size=1, + connection_class=Connection, + open=False, + ) as pool: + buyer = BuyerStore(pool=pool) + seller = PgReportingReceiptStore(pool=pool) + + class Client: + def __init__(self): + self.calls = 0 + + async def sync_reporting_receipts(self, request): + self.calls += 1 + await pause("before_seller") + response = await seller.ingest_receipt_batch( + request.model_dump(mode="json", exclude_none=True), + caller=ReportingDeliveryPrincipal(scope.account_id, scope.consumer_id), + ) + await pause("seller_committed") + return TaskResult( + status=TaskStatus.COMPLETED, + data=SyncReportingReceiptsResponse.model_validate(response), + ) + + client = Client() + + async def authorize(candidate): + assert candidate is client and settings.get("authorized", True) + return scope + + inputs = ( + None + if settings.get("resume") + else [ + ( + ReportingReceipt + if "reporting_materialization_id" in item + else ReportingAdjustmentReceipt + ).model_validate(item) + for item in settings["receipts"] + ] + ) + result = await submit_reporting_receipts( + client, authorizer=authorize, store=buyer, receipts=inputs + ) + await pause("returned") + print( + json.dumps( + { + "point": "done", + "pending": result.pending, + "proposal_deferred": result.proposal_deferred, + "outcomes": [outcome.result for outcome in result.outcomes], + "submitted": len(result.submitted_receipts), + "calls": client.calls, + "origins": origins, + } + ), + flush=True, + ) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_buyer_submission_support.py b/tests/conformance/reporting/_buyer_submission_support.py new file mode 100644 index 000000000..53fc3161a --- /dev/null +++ b/tests/conformance/reporting/_buyer_submission_support.py @@ -0,0 +1,143 @@ +"""Shared buyer intent vectors; PostgreSQL uses isolated real schemas.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from dataclasses import dataclass +from typing import Any + +import pytest + +from adcp.reporting.submissions import ( + InMemoryReportingSubmissionIntentStore, + PgReportingSubmissionIntentStore, + ReportingReceiptSubmissionClient, + ReportingSubmissionIntentStore, + ReportingSubmissionReceipt, + ReportingSubmissionScope, +) +from adcp.types import ( + ReportingAdjustmentReceipt, + ReportingReceipt, + SyncReportingReceiptsRequest, + SyncReportingReceiptsResponse, +) +from adcp.types.core import TaskResult, TaskStatus + +from ._generation_support import isolated_reporting_pool + +SCOPE = ReportingSubmissionScope("https://seller.example.test", "account-a", "buyer-a") +OBSERVED = "2026-09-01T01:00:00Z" +RECEIVED = "2026-09-01T02:00:00Z" +SECRET = "private provider diagnostic: PRIVATE_SENTINEL" + + +def receipt(index: int = 0, *, adjustment: bool = False) -> ReportingSubmissionReceipt: + if adjustment: + return ReportingAdjustmentReceipt.model_validate( + { + "reporting_receipt_id": f"adjustment-receipt-{index:06d}", + "reporting_adjustment_id": f"adjustment-{index}", + "adjusts_reporting_revision_id": f"revision-{index}", + "status": "accepted", + "observed_adjustment_sha256": "b" * 64, + "observed_at": OBSERVED, + } + ) + return ReportingReceipt.model_validate( + { + "reporting_receipt_id": f"revision-receipt-{index:06d}", + "reporting_obligation_id": f"obligation-{index}", + "reporting_revision_id": f"revision-{index}", + "reporting_materialization_id": f"materialization-{index}", + "status": "accepted", + "verification_profile": "manifest_checksums", + "observed_row_count": 0, + "observed_control_totals": [], + "observed_manifest_sha256": "a" * 64, + "observed_at": OBSERVED, + } + ) + + +def mixed(count: int = 3) -> list[ReportingSubmissionReceipt]: + return [receipt(i, adjustment=i % 2 == 0) for i in range(count)] + + +def response_for( + request: SyncReportingReceiptsRequest, + *, + fail: set[str] | None = None, + reverse: bool = False, + result: str = "recorded", +) -> SyncReportingReceiptsResponse: + wire = request.model_dump(mode="json", exclude_none=True) + results = [] + for kind, name in (("receipt", "receipts"), ("adjustment_receipt", "adjustment_receipts")): + for item in wire.get(name, []): + if item["reporting_receipt_id"] in (fail or set()): + results.append( + { + "result": "failed", + "reporting_receipt_id": item["reporting_receipt_id"], + "errors": [ + {"code": "REPORTING_RECORD_UNAVAILABLE", "message": SECRET}, + {"code": "UNRECOGNIZED_PRIVATE_SENTINEL", "message": SECRET}, + ], + } + ) + else: + results.append({"result": result, kind: {**item, "received_at": RECEIVED}}) + if reverse: + results.reverse() + return SyncReportingReceiptsResponse.model_validate({"results": results}) + + +class ReceiptClient: + def __init__(self, *, fail: set[str] | None = None) -> None: + self.requests: list[dict[str, Any]] = [] + self.fail = fail + + async def sync_reporting_receipts( + self, request: SyncReportingReceiptsRequest + ) -> TaskResult[SyncReportingReceiptsResponse]: + self.requests.append(request.model_dump(mode="json", exclude_none=True)) + return TaskResult( + status=TaskStatus.COMPLETED, + data=response_for(request, fail=self.fail, reverse=True), + ) + + +class TrustedAuthorizer: + """A fixture registry entry bound to a client, not to request parameters.""" + + def __init__( + self, client: ReportingReceiptSubmissionClient, scope: ReportingSubmissionScope = SCOPE + ) -> None: + self.client = client + self.scope = scope + self.allowed = True + self.calls = 0 + + async def __call__(self, client: ReportingReceiptSubmissionClient) -> ReportingSubmissionScope: + self.calls += 1 + if client is not self.client or not self.allowed: + raise RuntimeError(SECRET) + return self.scope + + +@dataclass +class IntentHarness: + store: ReportingSubmissionIntentStore + pool: Any = None + + +@pytest.fixture(params=["memory", "postgres"]) +async def intent_store(request: pytest.FixtureRequest) -> AsyncIterator[IntentHarness]: + if request.param == "memory": + yield IntentHarness(InMemoryReportingSubmissionIntentStore()) + else: + async with isolated_reporting_pool() as pool: + store = PgReportingSubmissionIntentStore(pool=pool) + await store.create_schema() + yield IntentHarness(store, pool) diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_migration.py b/tests/conformance/reporting/test_reporting_buyer_submission_migration.py new file mode 100644 index 000000000..175249b69 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submission_migration.py @@ -0,0 +1,99 @@ +"""Additive isolated SQL, schema preconditions, corruption and closed PG errors.""" + +from __future__ import annotations + +import traceback + +import pytest + +from adcp.reporting.submissions import ( + PgReportingSubmissionIntentStore, + ReportingSubmissionCode, + ReportingSubmissionError, + prepare_reporting_receipt_submission, +) + +from ._buyer_submission_support import SCOPE, SECRET, mixed, response_for +from ._generation_support import isolated_reporting_pool + + +async def test_buyer_migration_is_isolated_additive_and_preserves_existing_intents(): + async with isolated_reporting_pool() as pool: + store = PgReportingSubmissionIntentStore(pool=pool) + await store.create_schema() + state = await store.reserve(prepare_reporting_receipt_submission(SCOPE, mixed())) + state = await store.confirm(SCOPE, state.submission_id, 0, response_for(state.request(0))) + await store.create_schema() + assert await store.get(SCOPE) == state + async with pool.connection() as connection: + names = await ( + await connection.execute( + "SELECT tablename FROM pg_tables WHERE schemaname=current_schema()" + " ORDER BY tablename" + ) + ).fetchall() + assert names == [ + ("reporting_buyer_submission_intents",), + ("reporting_buyer_submission_scopes",), + ] + + +async def test_existing_weakened_reservation_index_fails_rollout_closed(): + async with isolated_reporting_pool() as pool: + store = PgReportingSubmissionIntentStore(pool=pool) + await store.create_schema() + async with pool.connection() as connection: + await connection.execute("DROP INDEX reporting_buyer_one_pending_scope") + await connection.execute( + "CREATE INDEX reporting_buyer_one_pending_scope" + " ON reporting_buyer_submission_intents(scope_sha256)" + ) + with pytest.raises(ReportingSubmissionError) as error: + await store.create_schema() + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + assert error.value.__context__ is None + + +@pytest.mark.parametrize( + "mutation", ["plan", "confirmed", "identity", "pointer", "pending", "driver"] +) +async def test_pg_corruption_and_driver_diagnostics_never_escape_or_permit_replacement(mutation): + async with isolated_reporting_pool() as pool: + store = PgReportingSubmissionIntentStore(pool=pool) + await store.create_schema() + state = await store.reserve(prepare_reporting_receipt_submission(SCOPE, mixed())) + async with pool.connection() as connection: + if mutation == "plan": + await connection.execute( + "UPDATE reporting_buyer_submission_intents SET canonical_plan=%s", (SECRET,) + ) + elif mutation == "confirmed": + await connection.execute( + "UPDATE reporting_buyer_submission_intents SET confirmed_results=%s", (SECRET,) + ) + elif mutation == "identity": + await connection.execute( + "UPDATE reporting_buyer_submission_scopes SET canonical_identity=%s", (SECRET,) + ) + elif mutation == "pointer": + await connection.execute( + "UPDATE reporting_buyer_submission_scopes SET current_submission_id=NULL" + ) + elif mutation == "pending": + await connection.execute( + "UPDATE reporting_buyer_submission_intents SET pending=false" + ) + elif mutation == "driver": + await connection.execute( + "ALTER TABLE reporting_buyer_submission_intents DROP COLUMN confirmed_results" + ) + for operation in (store.get(SCOPE), store.reserve(state)): + with pytest.raises(ReportingSubmissionError) as error: + await operation + assert error.value.code == ( + ReportingSubmissionCode.STORAGE_UNAVAILABLE + if mutation == "driver" + else ReportingSubmissionCode.HISTORY_CORRUPT + ) + assert error.value.__context__ is None and error.value.__cause__ is None + assert "PRIVATE_SENTINEL" not in "".join(traceback.format_exception(error.value)) diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_packaging.py b/tests/conformance/reporting/test_reporting_buyer_submission_packaging.py new file mode 100644 index 000000000..0ef664db4 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submission_packaging.py @@ -0,0 +1,156 @@ +"""Actual Python 3.10 VCS/sdist wheels, no-driver imports, typing and PG restart.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import os +import shutil +import tarfile +import zipfile +from pathlib import Path + +import pytest + +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.submissions import PgReportingSubmissionIntentStore, ReportingSubmissionScope + +from ._generation_support import require_rolling_database +from ._receipt_support import adjustment_for, batch_state, receipt_case, receipt_harness +from .test_reporting_buyer_submission_process import buyer_worker +from .test_reporting_materializer_packaging import b1_wheels, built_distribution +from .test_reporting_notification_packaging import ROOT, run_step + +__all__ = ["b1_wheels", "built_distribution"] + + +@pytest.mark.parametrize("kind", ["vcs", "sdist"]) +@pytest.mark.parametrize("drivers", [False, True], ids=["base", "pg"]) +async def test_python310_installed_buyer_submission_contract(request, kind, drivers): + interpreter = os.environ.get("ADCP_PYTHON310") + if interpreter is None: + pytest.skip("ADCP_PYTHON310 supplies the actual installed floor runtime") + if drivers: + require_rolling_database() + root, wheels, _ = request.getfixturevalue("b1_wheels") + _, _, source = request.getfixturevalue("built_distribution") + label = "buyer-" + kind + ("-pg" if drivers else "-base") + environment = root / label + await asyncio.to_thread( + run_step, + [interpreter, "-m", "venv", str(environment)], + label=label + "-environment", + cwd=root, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + await asyncio.to_thread( + run_step, + [*installer, str(wheels[kind]) + ("[pg]" if drivers else ""), "mypy==1.20.2"], + label=label + "-install", + cwd=root, + timeout=180, + ) + modules = { + "adcp.reporting.submissions" + + ("." + path.stem if path.stem != "__init__" else ""): hashlib.sha256( + path.read_bytes() + ).hexdigest() + for path in (ROOT / "src/adcp/reporting/submissions").glob("*.py") + } + sql_path = ROOT / "src/adcp/reporting/ledger/reporting_buyer_submissions.sql" + with zipfile.ZipFile(wheels[kind]) as wheel, tarfile.open(source) as sdist: + prefix = sdist.getnames()[0].split("/")[0] + member = "adcp/reporting/ledger/reporting_buyer_submissions.sql" + assert ( + wheel.read(member) + == sdist.extractfile(f"{prefix}/src/{member}").read() + == sql_path.read_bytes() + ) + settings = { + "workspace": str(ROOT), + "modules": modules, + "sql_sha256": hashlib.sha256(sql_path.read_bytes()).hexdigest(), + "drivers": drivers, + } + smoke = root / f"{label}_smoke.py" + worker_script = root / f"{label}_worker.py" + adopter = root / f"{label.replace('-', '_')}_adopter.py" + shutil.copy2(Path(__file__).with_name("_buyer_submission_installed.py"), smoke) + shutil.copy2(Path(__file__).with_name("_buyer_submission_process.py"), worker_script) + shutil.copy2(ROOT / "tests/type_checks/reporting_buyer_submission_intents.py", adopter) + result = json.loads( + await asyncio.to_thread( + run_step, + [str(python), "-I", str(smoke)], + label=label + "-isolated-contract", + cwd=root, + value=settings, + timeout=120, + ) + ) + assert result["python"] == "3.10" and result["outcomes"] == 201 + assert result["submitted"] == 200 and result["drivers"] == drivers + config = root / f"{label}-mypy.ini" + config.write_text( + "[mypy]\npython_version = 3.10\nstrict = True\n" + "plugins = adcp.types.mypy_plugin\nfollow_imports = silent\n" + ) + await asyncio.to_thread( + run_step, + [ + str(python), + "-I", + "-m", + "mypy", + "--config-file", + str(config), + "--strict", + "--no-incremental", + str(adopter), + ], + label=label + "-strict-adopter", + cwd=root, + timeout=120, + ) + if drivers: + for point in ("seller_committed", "confirmation_committed"): + async with receipt_harness("postgres") as h: + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + adjustment = await adjustment_for(h, s) + scope = ReportingSubmissionScope( + "seller", s.obligation.account_id, s.binding.consumer_id + ) + await PgReportingSubmissionIntentStore(pool=h.pool).create_schema() + inputs = [adjustment, receipt_to_wire(s.receipt)] + async with buyer_worker( + h, + scope, + inputs, + pause=point, + python=python, + script=worker_script, + installed=settings, + ) as child: + await child.event(point) + await child.kill() + async with buyer_worker( + h, + scope, + inputs, + resume=True, + python=python, + script=worker_script, + installed=settings, + ) as restarted: + outcome = await restarted.event("done") + assert await asyncio.wait_for(restarted.process.wait(), 10) == 0 + assert outcome["outcomes"] == ["recorded", "recorded"] + assert outcome["calls"] == int(point == "seller_committed") + assert set(outcome["origins"]) == set(modules) + assert await batch_state(h) == ((2, True),) diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_process.py b/tests/conformance/reporting/test_reporting_buyer_submission_process.py new file mode 100644 index 000000000..96fb95b47 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submission_process.py @@ -0,0 +1,168 @@ +"""Actual independent-process death around buyer intent and seller receipt commits.""" + +from __future__ import annotations + +import asyncio +import sys +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest + +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.submissions import PgReportingSubmissionIntentStore, ReportingSubmissionScope + +from ._receipt_support import adjustment_for, batch_state, receipt_case, receipt_harness +from .test_reporting_materializer_process import Child + + +@asynccontextmanager +async def buyer_worker( + h, scope, inputs, *, pause=None, resume=False, python=None, script=None, installed=None +): + process = await asyncio.create_subprocess_exec( + str(python or sys.executable), + *(["-I"] if installed else []), + str(script or Path(__file__).with_name("_buyer_submission_process.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "trusted_scope": { + "seller_id": scope.seller_id, + "account_id": scope.account_id, + "consumer_id": scope.consumer_id, + }, + "receipts": inputs, + "pause": pause, + "resume": resume, + "installed": installed, + } + ) + yield child + finally: + await child.kill() + diagnostic = await process.stderr.read() + if process.returncode not in {0, -9}: + pytest.fail(diagnostic.decode()) + + +@pytest.mark.parametrize( + "point", + [ + "before_intent", + "scope_row", + "intent_row", + "intent_committed", + "before_seller", + "seller_committed", + "response_delivered", + "confirmation_row", + "confirmation_committed", + "returned", + ], +) +async def test_process_death_preserves_exact_requests_and_confirmed_outcomes(point): + async with receipt_harness("postgres") as h: + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + adjustment = await adjustment_for(h, s) + # Opposite order from wire grouping: recovery must preserve this order. + inputs = [adjustment, receipt_to_wire(s.receipt)] + scope = ReportingSubmissionScope( + "https://seller.example.test", s.obligation.account_id, s.binding.consumer_id + ) + buyer = PgReportingSubmissionIntentStore(pool=h.pool) + await buyer.create_schema() + intent_committed = point not in {"before_intent", "scope_row", "intent_row"} + seller_committed = point in { + "seller_committed", + "response_delivered", + "confirmation_row", + "confirmation_committed", + "returned", + } + confirmed = point in {"confirmation_committed", "returned"} + async with buyer_worker(h, scope, inputs, pause=point) as child: + await child.event(point) + # Ordinary MVCC reads cannot see partial headers/plans/confirmations + # even when the killed worker holds a transaction's scope row lock. + async with h.pool.connection() as connection: + row = await ( + await connection.execute( + "SELECT (SELECT count(*) FROM reporting_buyer_submission_scopes)," + " (SELECT count(*) FROM reporting_buyer_submission_intents)," + " (SELECT count(*) FROM reporting_buyer_submission_intents" + " WHERE NOT pending)" + ) + ).fetchone() + assert row == (int(intent_committed), int(intent_committed), int(confirmed)) + assert await batch_state(h) == (((2, True),) if seller_committed else ()) + await child.kill() + before = await buyer.get(scope) + if intent_committed: + assert before is not None and before.pending != confirmed + else: + assert before is None + async with buyer_worker(h, scope, inputs, resume=intent_committed) as restarted: + result = await restarted.event("done") + assert await asyncio.wait_for(restarted.process.wait(), 10) == 0 + assert result["outcomes"] == ["recorded", "recorded"] + assert not result["pending"] and result["submitted"] == 2 + assert result["calls"] == int(not confirmed) + assert await batch_state(h) == ((2, True),) + after = await PgReportingSubmissionIntentStore(pool=h.pool).get(scope) + assert not after.pending + if before is not None: + assert before._plan == after._plan and before.submission_id == after.submission_id + assert [outcome.submitted_receipt.reporting_receipt_id for outcome in after.outcomes] == [ + item["reporting_receipt_id"] for item in inputs + ] + async with h.pool.connection() as connection: + batches = await ( + await connection.execute( + "SELECT canonical_request FROM reporting_receipt_ingestion_batches" + ) + ).fetchall() + from adcp.reporting.canonical_json import canonical_json_utf8_v1 + + expected = canonical_json_utf8_v1( + after.request(0).model_dump(mode="json", exclude_none=True) + ).decode() + assert batches == [(expected,)] + + +async def test_independent_concurrent_buyers_replay_one_reserved_seller_request(): + async with receipt_harness("postgres") as h: + s = await receipt_case(h) + inputs = [await adjustment_for(h, s), receipt_to_wire(s.receipt)] + competing = [ + {**item, "reporting_receipt_id": f"competing-receipt-{index:06d}"} + for index, item in enumerate(inputs) + ] + scope = ReportingSubmissionScope("seller", s.obligation.account_id, s.binding.consumer_id) + store = PgReportingSubmissionIntentStore(pool=h.pool) + await store.create_schema() + async with buyer_worker(h, scope, inputs, pause="before_seller") as first: + await first.event("before_seller") + async with buyer_worker(h, scope, competing, pause="before_seller") as second: + await second.event("before_seller") + await asyncio.gather( + first.send({"continue": True}), second.send({"continue": True}) + ) + a, b = await asyncio.gather(first.event("done"), second.event("done")) + assert a["outcomes"] == b["outcomes"] == ["recorded", "recorded"] + assert not a["proposal_deferred"] and b["proposal_deferred"] + assert await asyncio.wait_for(first.process.wait(), 10) == 0 + assert await asyncio.wait_for(second.process.wait(), 10) == 0 + assert await batch_state(h) == ((2, True),) + final = await store.get(scope) + assert not final.pending + assert [outcome.submitted_receipt.reporting_receipt_id for outcome in final.outcomes] == [ + item["reporting_receipt_id"] for item in inputs + ] diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_seller.py b/tests/conformance/reporting/test_reporting_buyer_submission_seller.py new file mode 100644 index 000000000..49b1a79d2 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submission_seller.py @@ -0,0 +1,109 @@ +"""The buyer retains actual reviewed seller mixed-batch outcomes unchanged.""" + +from __future__ import annotations + +import pytest + +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.submissions import ( + InMemoryReportingSubmissionIntentStore, + PgReportingSubmissionIntentStore, + ReportingReceiptFailureCode, + ReportingSubmissionScope, + submit_reporting_receipts, +) +from adcp.types import ReportingAdjustmentReceipt, ReportingReceipt, SyncReportingReceiptsResponse +from adcp.types.core import TaskResult, TaskStatus + +from ._buyer_submission_support import TrustedAuthorizer +from ._receipt_support import adjustment_for, receipt_case, receipt_harness + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_seller_stale_rejected_leaf_and_accepted_terminality_remain_item_outcomes(backend): + async with receipt_harness(backend) as h: + s = await receipt_case(h) + scope = ReportingSubmissionScope("seller", s.obligation.account_id, s.binding.consumer_id) + if h.pool is None: + buyer = InMemoryReportingSubmissionIntentStore() + else: + buyer = PgReportingSubmissionIntentStore(pool=h.pool) + await buyer.create_schema() + + class Client: + async def sync_reporting_receipts(self, request): + response = await h.store.ingest_receipt_batch( + request.model_dump(mode="json", exclude_none=True), caller=s.binding.principal + ) + return TaskResult( + status=TaskStatus.COMPLETED, + data=SyncReportingReceiptsResponse.model_validate(response), + ) + + client = Client() + authorizer = TrustedAuthorizer(client, scope) + original = receipt_to_wire(s.receipt) + rejected = ReportingReceipt.model_validate( + {**original, "status": "rejected", "rejection_codes": ["LOAD_FAILED"]} + ) + first = await submit_reporting_receipts( + client, authorizer=authorizer, store=buyer, receipts=[rejected] + ) + assert not first.pending and first.outcomes[0].receipt.status == "rejected" + second_rejected = rejected.model_copy( + update={ + "reporting_receipt_id": "buyer-rejected-replacement-2", + "supersedes_reporting_receipt_id": rejected.reporting_receipt_id, + } + ) + second = await submit_reporting_receipts( + client, authorizer=authorizer, store=buyer, receipts=[second_rejected] + ) + assert ( + second.outcomes[0].receipt.reporting_receipt_id == second_rejected.reporting_receipt_id + ) + stale = ReportingReceipt.model_validate( + { + **original, + "reporting_receipt_id": "buyer-stale-replacement-3", + "supersedes_reporting_receipt_id": rejected.reporting_receipt_id, + } + ) + adjustment = ReportingAdjustmentReceipt.model_validate(await adjustment_for(h, s)) + mixed = await submit_reporting_receipts( + client, authorizer=authorizer, store=buyer, receipts=[adjustment, stale] + ) + assert not mixed.pending + assert [outcome.result for outcome in mixed.outcomes] == ["recorded", "failed"] + assert mixed.outcomes[1].error_codes == ( + ReportingReceiptFailureCode.REPORTING_RECORD_UNAVAILABLE, + ) + accepted = ReportingReceipt.model_validate( + { + **original, + "reporting_receipt_id": "buyer-current-replacement-4", + "supersedes_reporting_receipt_id": second_rejected.reporting_receipt_id, + } + ) + final = await submit_reporting_receipts( + client, authorizer=authorizer, store=buyer, receipts=[accepted] + ) + assert final.outcomes[0].receipt.status == "accepted" + terminal = accepted.model_copy( + update={ + "reporting_receipt_id": "buyer-after-terminal-5", + "supersedes_reporting_receipt_id": accepted.reporting_receipt_id, + } + ) + failure = await submit_reporting_receipts( + client, authorizer=authorizer, store=buyer, receipts=[terminal] + ) + assert not failure.pending and not failure.submitted_receipts + assert failure.outcomes[0].error_codes == ( + ReportingReceiptFailureCode.ACCEPTED_RECEIPT_TERMINAL, + ) + # Old completed intents retain rejected, failed and accepted evidence; + # the submission engine never edits a leaf or constructs a replacement. + assert await buyer.get(scope, first.submission.submission_id) == first.submission + assert await buyer.get(scope, mixed.submission.submission_id) == mixed.submission + assert await buyer.get(scope, final.submission.submission_id) == final.submission diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_uncertainty.py b/tests/conformance/reporting/test_reporting_buyer_submission_uncertainty.py new file mode 100644 index 000000000..dee75145f --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submission_uncertainty.py @@ -0,0 +1,260 @@ +"""No timeout, cancellation, malformed reply or lost commit permits replacement.""" + +from __future__ import annotations + +import asyncio +from dataclasses import replace + +import pytest + +from adcp.reporting.submissions import ( + ReportingSubmissionCode, + ReportingSubmissionError, + submit_reporting_receipts, +) +from adcp.types.core import TaskResult, TaskStatus + +from ._buyer_submission_support import ( + SCOPE, + SECRET, + ReceiptClient, + TrustedAuthorizer, + intent_store, + mixed, + response_for, +) + +__all__ = ["intent_store"] + + +class UncertainClient(ReceiptClient): + def __init__(self, failure, *, fail_on=1): + super().__init__() + self.failure = failure + self.fail_on = fail_on + self.attempts = 0 + self.started = asyncio.Event() + + async def sync_reporting_receipts(self, request): + self.attempts += 1 + self.started.set() + if self.attempts != self.fail_on: + return await super().sync_reporting_receipts(request) + self.requests.append(request.model_dump(mode="json", exclude_none=True)) + if self.failure == "exception": + raise RuntimeError(SECRET) + if self.failure in {"timeout", "cancel"}: + await asyncio.Event().wait() + if self.failure == "failed": + return TaskResult(status=TaskStatus.FAILED, success=False, error=SECRET) + if self.failure == "pending": + return TaskResult(status=TaskStatus.WORKING, data=response_for(request)) + if self.failure == "malformed": + response = response_for(request) + response.results.pop() + return TaskResult(status=TaskStatus.COMPLETED, data=response) + raise AssertionError("unknown test fault") + + +@pytest.mark.parametrize( + "failure,diagnostic", + [ + ("exception", ReportingSubmissionCode.TRANSPORT_UNCERTAIN), + ("timeout", ReportingSubmissionCode.TRANSPORT_UNCERTAIN), + ("failed", ReportingSubmissionCode.RESPONSE_UNCONFIRMED), + ("pending", ReportingSubmissionCode.RESPONSE_UNCONFIRMED), + ("malformed", ReportingSubmissionCode.INVALID_RESPONSE), + ], +) +async def test_uncertainty_replays_exact_prior_body_and_defers_new_proposal( + intent_store, failure, diagnostic +): + store = intent_store.store + client = UncertainClient(failure) + authorizer = TrustedAuthorizer(client) + original = mixed() + first = await submit_reporting_receipts( + client, authorizer=authorizer, store=store, receipts=original, timeout_seconds=0.01 + ) + assert first.pending and first.diagnostic == diagnostic + assert first.outcomes == () and "PRIVATE_SENTINEL" not in repr(first) + earlier_request = client.requests[0] + replacement = mixed(4) + recovered = await submit_reporting_receipts( + client, authorizer=authorizer, store=store, receipts=replacement + ) + assert not recovered.pending and recovered.proposal_deferred + assert recovered.submission.submission_id == first.submission.submission_id + assert client.requests == [earlier_request, earlier_request] + assert [outcome.submitted_receipt for outcome in recovered.outcomes] == original + # The same deferred proposal is not automatically submitted after recovery. + assert await store.get(SCOPE) == recovered.submission + + +async def test_second_chunk_timeout_preserves_first_chunk_outcomes_across_resume(intent_store): + store = intent_store.store + client = UncertainClient("exception", fail_on=2) + authorizer = TrustedAuthorizer(client) + first = await submit_reporting_receipts( + client, authorizer=authorizer, store=store, receipts=mixed(201) + ) + assert first.pending and len(first.outcomes) == 100 + recovered = await submit_reporting_receipts(client, authorizer=authorizer, store=store) + assert not recovered.pending and len(recovered.outcomes) == 201 + assert recovered.outcomes[:100] == first.outcomes + assert len(client.requests) == 4 + assert client.requests[1] == client.requests[2] + assert client.requests[0]["idempotency_key"] != client.requests[2]["idempotency_key"] + + +async def test_cancellation_preserves_reserved_request(intent_store): + client = UncertainClient("cancel") + authorizer = TrustedAuthorizer(client) + task = asyncio.create_task( + submit_reporting_receipts( + client, authorizer=authorizer, store=intent_store.store, receipts=mixed() + ) + ) + await asyncio.wait_for(client.started.wait(), 10) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + pending = await intent_store.store.get(SCOPE) + assert pending.pending and not pending.outcomes + recovered = await submit_reporting_receipts( + client, authorizer=authorizer, store=intent_store.store + ) + assert not recovered.pending and client.requests[0] == client.requests[1] + + +class FaultingStore: + """Inject failures on either side of actual backend commits, including PG.""" + + def __init__(self, delegate, fault): + self.delegate = delegate + self.fault = fault + + async def reserve(self, proposed): + if self.fault == "before_reserve": + raise RuntimeError(SECRET) + state = await self.delegate.reserve(proposed) + if self.fault == "after_reserve": + raise RuntimeError(SECRET) + return state + + async def get(self, scope, submission_id=None): + return await self.delegate.get(scope, submission_id) + + async def confirm(self, scope, submission_id, chunk, response): + if self.fault == "before_confirm": + raise RuntimeError(SECRET) + state = await self.delegate.confirm(scope, submission_id, chunk, response) + if self.fault == "after_confirm": + raise RuntimeError(SECRET) + return state + + +@pytest.mark.parametrize( + "fault", ["before_reserve", "after_reserve", "before_confirm", "after_confirm"] +) +async def test_storage_failure_is_closed_and_uncertainty_uses_committed_state(intent_store, fault): + client = ReceiptClient() + authorizer = TrustedAuthorizer(client) + inputs = mixed() + with pytest.raises(ReportingSubmissionError) as error: + await submit_reporting_receipts( + client, + authorizer=authorizer, + store=FaultingStore(intent_store.store, fault), + receipts=inputs, + ) + assert error.value.code == ReportingSubmissionCode.STORAGE_UNAVAILABLE + assert error.value.__context__ is None and error.value.__cause__ is None + assert "PRIVATE_SENTINEL" not in repr(error.value) + state = await intent_store.store.get(SCOPE) + if fault == "before_reserve": + assert state is None and not client.requests + elif fault == "after_reserve": + assert state.pending and not client.requests + elif fault == "before_confirm": + assert state.pending and len(client.requests) == 1 + else: + assert not state.pending and len(client.requests) == 1 + recovered = await submit_reporting_receipts( + client, + authorizer=authorizer, + store=intent_store.store, + receipts=inputs if state is None else None, + ) + assert not recovered.pending and len(recovered.outcomes) == 3 + assert len(client.requests) == (2 if fault == "before_confirm" else 1) + if len(client.requests) == 2: + assert client.requests[0] == client.requests[1] + + +@pytest.mark.parametrize("cached", [False, True]) +async def test_revoked_authorization_blocks_reservation_and_cached_replay(intent_store, cached): + client = ReceiptClient() + authorizer = TrustedAuthorizer(client) + if cached: + await submit_reporting_receipts( + client, authorizer=authorizer, store=intent_store.store, receipts=mixed() + ) + authorizer.allowed = False + before = await intent_store.store.get(SCOPE) + with pytest.raises(ReportingSubmissionError) as error: + await submit_reporting_receipts( + client, authorizer=authorizer, store=intent_store.store, receipts=mixed(4) + ) + assert error.value.code == ReportingSubmissionCode.UNAUTHORIZED + assert error.value.__context__ is None and error.value.__cause__ is None + assert "PRIVATE_SENTINEL" not in str(error.value) + assert await intent_store.store.get(SCOPE) == before + assert len(client.requests) == int(cached) + + +async def test_authorizer_must_bind_the_exact_client_before_store_use(intent_store): + client, different_client = ReceiptClient(), ReceiptClient() + with pytest.raises(ReportingSubmissionError) as error: + await submit_reporting_receipts( + client, + authorizer=TrustedAuthorizer(different_client), + store=intent_store.store, + receipts=mixed(), + ) + assert error.value.code == ReportingSubmissionCode.UNAUTHORIZED + assert await intent_store.store.get(SCOPE) is None and not client.requests + + +async def test_mid_submission_identity_change_cannot_send_under_another_scope(intent_store): + client = ReceiptClient() + + class ChangedAuthorizer(TrustedAuthorizer): + async def __call__(self, client): + scope = await super().__call__(client) + return replace(scope, consumer_id="other-buyer") if self.calls >= 3 else scope + + authorizer = ChangedAuthorizer(client) + with pytest.raises(ReportingSubmissionError) as error: + await submit_reporting_receipts( + client, authorizer=authorizer, store=intent_store.store, receipts=mixed(101) + ) + assert error.value.code == ReportingSubmissionCode.UNAUTHORIZED + assert len(client.requests) == 1 + state = await intent_store.store.get(SCOPE) + assert state.pending and state.confirmed_chunks == 1 + assert await intent_store.store.get(replace(SCOPE, consumer_id="other-buyer")) is None + recovered = await submit_reporting_receipts( + client, authorizer=TrustedAuthorizer(client), store=intent_store.store + ) + assert not recovered.pending and len(client.requests) == 2 + + +async def test_resume_before_any_intent_is_a_closed_missing_state(intent_store): + client = ReceiptClient() + with pytest.raises(ReportingSubmissionError) as error: + await submit_reporting_receipts( + client, authorizer=TrustedAuthorizer(client), store=intent_store.store + ) + assert error.value.code == ReportingSubmissionCode.NOT_FOUND + assert not client.requests diff --git a/tests/conformance/reporting/test_reporting_buyer_submissions.py b/tests/conformance/reporting/test_reporting_buyer_submissions.py new file mode 100644 index 000000000..3bc91408e --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submissions.py @@ -0,0 +1,346 @@ +"""Shared memory/PostgreSQL buyer reservation and response evidence contracts.""" + +from __future__ import annotations + +import asyncio +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.submissions import ( + ReportingReceiptFailureCode, + ReportingSubmissionCode, + ReportingSubmissionError, + prepare_reporting_receipt_submission, + submit_reporting_receipts, +) +from adcp.reporting.submissions import models as submission_models +from adcp.types import SyncReportingReceiptsResponse + +from ._buyer_submission_support import ( + RECEIVED, + SCOPE, + SECRET, + ReceiptClient, + TrustedAuthorizer, + intent_store, + mixed, + receipt, + response_for, +) + +__all__ = ["intent_store"] + + +async def test_mixed_201_inputs_have_exact_bounded_chunks_and_ordered_outcomes(intent_store): + store = intent_store.store + inputs = mixed(201) + failures = {inputs[i].reporting_receipt_id for i in (0, 99, 100, 200)} + client = ReceiptClient(fail=failures) + result = await submit_reporting_receipts( + client, authorizer=TrustedAuthorizer(client), store=store, receipts=inputs + ) + assert not result.pending and result.diagnostic is None and not result.proposal_deferred + assert len(result.outcomes) == 201 and len(result.submitted_receipts) == 197 + assert [outcome.ordinal for outcome in result.outcomes] == list(range(201)) + assert [outcome.submitted_receipt for outcome in result.outcomes] == inputs + assert [ + len(call.get("receipts", [])) + len(call.get("adjustment_receipts", [])) + for call in client.requests + ] == [100, 100, 1] + assert len({call["idempotency_key"] for call in client.requests}) == 3 + assert all(call["account"] == {"account_id": SCOPE.account_id} for call in client.requests) + for outcome in result.outcomes: + if outcome.submitted_receipt.reporting_receipt_id in failures: + assert outcome.result == "failed" and outcome.receipt is None + assert outcome.error_codes == ( + ReportingReceiptFailureCode.REPORTING_RECORD_UNAVAILABLE, + ReportingReceiptFailureCode.UNKNOWN, + ) + else: + assert outcome.result == "recorded" + assert outcome.receipt.received_at.isoformat() == "2026-09-01T02:00:00+00:00" + restored = await store.get(SCOPE) + assert restored == result.submission + replay = await submit_reporting_receipts( + client, authorizer=TrustedAuthorizer(client), store=store, receipts=inputs + ) + assert replay == result and len(client.requests) == 3 + assert SECRET not in repr(result) and "PRIVATE_SENTINEL" not in repr(result) + assert all(b"PRIVATE_SENTINEL" not in chunk for chunk in restored._confirmed) + + +async def test_concurrent_different_proposals_reserve_one_exact_scope_intent(intent_store): + store = intent_store.store + proposals = [prepare_reporting_receipt_submission(SCOPE, [receipt(i)]) for i in range(8)] + reserved = await asyncio.gather(*(store.reserve(proposal) for proposal in proposals)) + assert len({state.submission_id for state in reserved}) == 1 + winner = reserved[0] + assert winner in proposals + assert all(state == winner for state in reserved) + assert await store.get(SCOPE) == winner + confirmations = await asyncio.gather( + *( + store.confirm(SCOPE, winner.submission_id, 0, response_for(winner.request(0))) + for _ in range(8) + ) + ) + assert all(not state.pending for state in confirmations) + assert len({state._confirmed for state in confirmations}) == 1 + assert (await store.reserve(winner)) == confirmations[0] + loser = next( + proposal for proposal in proposals if proposal.submission_id != winner.submission_id + ) + assert await store.reserve(loser) == loser + # An old concurrent acknowledgement cannot clear the new pending lane. + await store.confirm( + SCOPE, winner.submission_id, 0, response_for(winner.request(0), result="unchanged") + ) + assert await store.get(SCOPE) == loser + assert await store.reserve(winner) == loser + + +@pytest.mark.parametrize("part", ["seller_id", "account_id", "consumer_id"]) +async def test_exact_scope_isolates_seller_account_and_canonical_consumer(intent_store, part): + first = prepare_reporting_receipt_submission(SCOPE, mixed()) + scope = replace(SCOPE, **{part: "other-identity"}) + second = prepare_reporting_receipt_submission(scope, mixed()) + assert first.submission_id != second.submission_id + assert first.request(0).idempotency_key != second.request(0).idempotency_key + store = intent_store.store + await asyncio.gather(store.reserve(first), store.reserve(second)) + assert await store.get(SCOPE) == first and await store.get(scope) == second + assert await store.get(scope, first.submission_id) is None + with pytest.raises(ReportingSubmissionError) as error: + await store.confirm(scope, first.submission_id, 0, response_for(first.request(0))) + assert error.value.code == ReportingSubmissionCode.NOT_FOUND + + +async def test_maximum_url_principal_survives_real_index_and_restart(intent_store): + principal = "https://buyer.example.test/" + "x" * (2048 - len("https://buyer.example.test/")) + scope = replace(SCOPE, consumer_id=principal) + state = prepare_reporting_receipt_submission(scope, mixed()) + await intent_store.store.reserve(state) + assert await intent_store.store.get(scope) == state + assert len(scope.storage_key) == 64 + assert principal not in repr(scope) and principal not in repr(state) + + +@pytest.mark.parametrize( + "mutation", + [ + "missing", + "duplicate", + "extra", + "wrong_id", + "wrong_kind", + "wrong_revision", + "wrong_materialization", + "wrong_obligation", + "wrong_adjustment", + "wrong_digest", + "wrong_status", + "wrong_time", + "optional_field_removed", + ], +) +async def test_no_chunk_ack_before_complete_unique_response_and_immutable_body_validation( + intent_store, mutation +): + inputs = [receipt(), receipt(1, adjustment=True)] + inputs[0].consumer_commit_ref = "public-load-1" + state = prepare_reporting_receipt_submission(SCOPE, inputs) + store = intent_store.store + await store.reserve(state) + body = response_for(state.request(0)).model_dump(mode="json", exclude_none=True) + results = body["results"] + if mutation == "missing": + results.pop() + elif mutation == "duplicate": + results[1] = results[0] + elif mutation == "extra": + results.append( + { + "result": "failed", + "reporting_receipt_id": "unrequested-receipt-1", + "errors": [{"code": "INVALID_REQUEST", "message": SECRET}], + } + ) + elif mutation == "wrong_id": + results[0]["receipt"]["reporting_receipt_id"] = "unrequested-receipt-1" + elif mutation == "wrong_kind": + results[0] = { + "result": "recorded", + "adjustment_receipt": { + **results[1]["adjustment_receipt"], + "reporting_receipt_id": inputs[0].reporting_receipt_id, + }, + } + elif mutation == "wrong_revision": + results[0]["receipt"]["reporting_revision_id"] = "other-revision" + elif mutation == "wrong_materialization": + results[0]["receipt"]["reporting_materialization_id"] = "other-materialization" + elif mutation == "wrong_obligation": + results[0]["receipt"]["reporting_obligation_id"] = "other-obligation" + elif mutation == "wrong_adjustment": + results[1]["adjustment_receipt"]["reporting_adjustment_id"] = "other-adjustment" + elif mutation == "wrong_digest": + results[1]["adjustment_receipt"]["observed_adjustment_sha256"] = "c" * 64 + elif mutation == "wrong_status": + results[1]["adjustment_receipt"]["status"] = "rejected" + results[1]["adjustment_receipt"]["rejection_codes"] = ["ADJUSTMENT_DIGEST_MISMATCH"] + elif mutation == "wrong_time": + results[1]["adjustment_receipt"]["observed_at"] = RECEIVED + elif mutation == "optional_field_removed": + del results[0]["receipt"]["consumer_commit_ref"] + response = SyncReportingReceiptsResponse.model_validate(body) + with pytest.raises(ReportingSubmissionError) as error: + await store.confirm(SCOPE, state.submission_id, 0, response) + assert error.value.code == ReportingSubmissionCode.INVALID_RESPONSE + assert error.value.__context__ is None and error.value.__cause__ is None + assert await store.get(SCOPE) == state + replacement = prepare_reporting_receipt_submission(SCOPE, [receipt(99)]) + assert await store.reserve(replacement) == state + + +async def test_confirmations_are_prefix_ordered_and_first_outcomes_are_immutable(intent_store): + store = intent_store.store + state = await store.reserve(prepare_reporting_receipt_submission(SCOPE, mixed(101))) + with pytest.raises(ReportingSubmissionError) as error: + await store.confirm(SCOPE, state.submission_id, 1, response_for(state.request(1))) + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + failed_id = state.request(0).receipts[0].reporting_receipt_id + first = await store.confirm( + SCOPE, state.submission_id, 0, response_for(state.request(0), fail={failed_id}) + ) + duplicate = await store.confirm( + SCOPE, + state.submission_id, + 0, + response_for(state.request(0), fail={failed_id}, result="unchanged"), + ) + assert duplicate == first + assert any(outcome.result == "failed" for outcome in duplicate.outcomes) + assert duplicate.pending + with pytest.raises(ReportingSubmissionError) as contradiction: + await store.confirm(SCOPE, state.submission_id, 0, response_for(state.request(0))) + assert contradiction.value.code == ReportingSubmissionCode.INVALID_RESPONSE + assert await store.get(SCOPE) == first + completed = await store.confirm(SCOPE, state.submission_id, 1, response_for(state.request(1))) + assert not completed.pending and completed._confirmed[:1] == first._confirmed + assert len(completed.outcomes) == 101 + + +def test_outbound_capture_is_immutable_and_each_public_model_view_is_detached(): + inputs = mixed() + first = prepare_reporting_receipt_submission(SCOPE, inputs) + same = prepare_reporting_receipt_submission(SCOPE, inputs) + assert first == same + inputs[0].observed_adjustment_sha256 = "f" * 64 + first.request(0).adjustment_receipts[0].observed_adjustment_sha256 = "e" * 64 + assert first == same + assert first.request(0).adjustment_receipts[0].observed_adjustment_sha256 == "b" * 64 + assert ( + prepare_reporting_receipt_submission(SCOPE, list(reversed(mixed()))).submission_id + != first.submission_id + ) + assert "account-a" not in repr(first) and "observed_at" not in repr(first) + + +@pytest.mark.parametrize( + "case", + [ + "empty", + "duplicate", + "received_at", + "too_many", + "oversized", + "malformed", + "accepted_rejections", + ], +) +def test_admission_rejects_invalid_whole_plans_before_reservation(case): + inputs = mixed() + if case == "empty": + inputs = [] + elif case == "duplicate": + inputs[1].reporting_receipt_id = inputs[0].reporting_receipt_id + elif case == "received_at": + inputs[0].received_at = inputs[0].observed_at + elif case == "too_many": + inputs = mixed(10001) + elif case == "oversized": + inputs[1].consumer_commit_ref = "x" * (1024 * 1024) + elif case == "malformed": + inputs[0].reporting_receipt_id = SECRET + elif case == "accepted_rejections": + inputs[0].rejection_codes = ["ADJUSTMENT_DIGEST_MISMATCH"] + with pytest.raises(ReportingSubmissionError) as error: + prepare_reporting_receipt_submission(SCOPE, inputs) + assert error.value.code == ReportingSubmissionCode.INVALID_PLAN + assert SECRET not in str(error.value) and error.value.__context__ is None + + +@pytest.mark.parametrize( + "consumer", + [ + "anonymous", + " buyer ", + "https://a:b@buyer.invalid", + "https://buyer.invalid/?token=PRIVATE_SENTINEL", + "x" * 2049, + ], +) +def test_trusted_identity_syntax_is_bounded_and_errors_are_closed(consumer): + with pytest.raises(ReportingSubmissionError) as error: + replace(SCOPE, consumer_id=consumer) + assert error.value.code == ReportingSubmissionCode.INVALID_SCOPE + assert error.value.__context__ is None and "PRIVATE_SENTINEL" not in str(error.value) + + +async def test_corrupt_proposal_is_never_an_escape_from_pending_intent(intent_store): + good = prepare_reporting_receipt_submission(SCOPE, mixed()) + await intent_store.store.reserve(good) + plan = json.loads(good._plan) + plan["requests"][0]["account"] = {"account_id": "request-asserted-other-account"} + bad = replace(good, _plan=json.dumps(plan).encode()) + with pytest.raises(ReportingSubmissionError) as error: + await intent_store.store.reserve(bad) + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + assert await intent_store.store.get(SCOPE) == good + + +async def test_retained_response_byte_limit_fails_closed_before_confirmation( + intent_store, monkeypatch +): + state = prepare_reporting_receipt_submission(SCOPE, mixed()) + await intent_store.store.reserve(state) + response = response_for(state.request(0)) + # Sanitized retention adds an explicit ID per result. Check that limit too, + # even when the actual incoming response is within its byte budget. + limit = len(canonical_json_utf8_v1(response.model_dump(mode="json", exclude_none=True))) + monkeypatch.setattr(submission_models, "MAX_RESPONSE_BYTES", limit) + with pytest.raises(ReportingSubmissionError) as error: + await intent_store.store.confirm(SCOPE, state.submission_id, 0, response) + assert error.value.code == ReportingSubmissionCode.INVALID_RESPONSE + assert await intent_store.store.get(SCOPE) == state + + +async def test_cumulative_confirmation_bound_preserves_the_confirmed_prefix( + intent_store, monkeypatch +): + store = intent_store.store + state = await store.reserve(prepare_reporting_receipt_submission(SCOPE, mixed(101))) + first = await store.confirm(SCOPE, state.submission_id, 0, response_for(state.request(0))) + # Both individual responses fit. The retained array has only enough room + # for the first one; the second must not partially commit or free the scope. + retained = canonical_json_utf8_v1([json.loads(chunk) for chunk in first._confirmed]) + monkeypatch.setattr(submission_models, "MAX_CONFIRMATION_BYTES", len(retained)) + response = response_for(state.request(1)) + with pytest.raises(ReportingSubmissionError) as error: + await store.confirm(SCOPE, state.submission_id, 1, response) + assert error.value.code == ReportingSubmissionCode.INVALID_RESPONSE + assert await store.get(SCOPE) == first + assert await store.reserve(prepare_reporting_receipt_submission(SCOPE, [receipt(500)])) == first diff --git a/tests/type_checks/reporting_buyer_submission_intents.py b/tests/type_checks/reporting_buyer_submission_intents.py new file mode 100644 index 000000000..062ac6993 --- /dev/null +++ b/tests/type_checks/reporting_buyer_submission_intents.py @@ -0,0 +1,86 @@ +"""Strict additive public buyer submission and unchanged checkpoint contracts.""" + +from collections.abc import Sequence + +from adcp import ADCPClient +from adcp.reporting import ReportingCheckpointStore +from adcp.reporting.submissions import ( + InMemoryReportingSubmissionIntentStore, + ReportingReceiptSubmissionClient, + ReportingSubmissionAuthorizer, + ReportingSubmissionIntentStore, + ReportingSubmissionReceipt, + ReportingSubmissionResult, + ReportingSubmissionScope, + submit_reporting_receipts, +) +from adcp.types import ReportingAdjustment, ReportingAdjustmentReceipt, ReportingReceipt + + +class ExistingCheckpoint: + """An existing structural implementation still requires only get and put.""" + + def __init__(self) -> None: + self.receipts: dict[str, ReportingReceipt] = {} + + async def get(self, reporting_materialization_id: str) -> ReportingReceipt | None: + return self.receipts.get(reporting_materialization_id) + + async def put(self, receipt: ReportingReceipt) -> None: + self.receipts[receipt.reporting_materialization_id] = receipt + + +checkpoint: ReportingCheckpointStore = ExistingCheckpoint() +volatile_test_store: ReportingSubmissionIntentStore = InMemoryReportingSubmissionIntentStore() + + +class AuthorizedRegistryBinding: + """Values come from the adopter's verified registry/account authorization.""" + + def __init__(self, client: ADCPClient, resolved_scope: ReportingSubmissionScope) -> None: + self.client = client + self.resolved_scope = resolved_scope + self.active = True + + async def __call__(self, client: ReportingReceiptSubmissionClient) -> ReportingSubmissionScope: + if client is not self.client or not self.active: + raise PermissionError("reporting access unavailable") + return self.resolved_scope + + +async def submit_validated_plan( + client: ADCPClient, + registry: ReportingSubmissionAuthorizer, + durable_store: ReportingSubmissionIntentStore, + selected_receipts: Sequence[ReportingSubmissionReceipt], +) -> ReportingSubmissionResult: + # Plan/history validation precedes this function. Never derive identity from + # the selected receipts, or reconstruct new receipts on an uncertain retry. + return await submit_reporting_receipts( + client, authorizer=registry, store=durable_store, receipts=selected_receipts + ) + + +async def recover_uncertain_submission( + client: ADCPClient, + registry: ReportingSubmissionAuthorizer, + durable_store: ReportingSubmissionIntentStore, +) -> ReportingSubmissionResult: + return await submit_reporting_receipts(client, authorizer=registry, store=durable_store) + + +def adjustment_target(adjustment: ReportingAdjustment, receipt: ReportingAdjustmentReceipt) -> bool: + # The supported curated imports were already present on the accepted base. + return adjustment.reporting_adjustment_id == receipt.reporting_adjustment_id + + +def confirmed_count(result: ReportingSubmissionResult) -> int: + receipts: tuple[ReportingSubmissionReceipt, ...] = result.submitted_receipts + for outcome in result.outcomes: + if outcome.result == "failed": + for error in outcome.error_codes: + assert isinstance(error.value, str) + else: + confirmed: ReportingSubmissionReceipt | None = outcome.receipt + assert confirmed is not None + return len(receipts) From 5e1789f1444a9f740da57f642a109f10d19f599c Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 25 Sep 2026 12:11:18 +0000 Subject: [PATCH 2/3] fix(reporting): bound buyer intent validation and verify checks --- .../submissions/_validation_cache.py | 65 ++++ src/adcp/reporting/submissions/models.py | 133 ++++++-- src/adcp/reporting/submissions/pg.py | 288 +++++++++++------- src/adcp/reporting/submissions/store.py | 4 +- .../reporting/_buyer_submission_benchmark.py | 229 ++++++++++++++ .../test_reporting_buyer_submission_cas.py | 276 +++++++++++++++++ ...st_reporting_buyer_submission_migration.py | 85 ++++++ ...t_reporting_buyer_submission_validation.py | 154 ++++++++++ 8 files changed, 1110 insertions(+), 124 deletions(-) create mode 100644 src/adcp/reporting/submissions/_validation_cache.py create mode 100644 tests/conformance/reporting/_buyer_submission_benchmark.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submission_cas.py create mode 100644 tests/conformance/reporting/test_reporting_buyer_submission_validation.py diff --git a/src/adcp/reporting/submissions/_validation_cache.py b/src/adcp/reporting/submissions/_validation_cache.py new file mode 100644 index 000000000..70fd1d9d1 --- /dev/null +++ b/src/adcp/reporting/submissions/_validation_cache.py @@ -0,0 +1,65 @@ +"""Bounded exact-byte validation proofs; no caller-owned objects are retained.""" + +from __future__ import annotations + +import sys +from collections import OrderedDict +from threading import RLock + + +class ValidationCache: + """An internal LRU of immutable keys and values, bounded in two dimensions. + + Count shared bytes again when sizing entries: this deliberately overcounts + retained payloads. The per-entry allowance also covers mapping/lock metadata. + A cache hit is equality of the complete bytes, never equality of a digest. + Eviction or an oversized entry merely causes full validation on the next use. + """ + + def __init__(self, *, entries: int, byte_budget: int) -> None: + if entries < 1 or byte_budget < 1: + raise ValueError("positive validation cache bounds required") + self._entries = entries + self._byte_budget = byte_budget + self._retained_bytes = 0 + self._values: OrderedDict[tuple[bytes, ...], tuple[tuple[bytes, ...], int]] = OrderedDict() + self._lock = RLock() + + def get(self, key: tuple[bytes, ...]) -> tuple[bytes, ...] | None: + with self._lock: + entry = self._values.get(key) + if entry is None: + return None + self._values.move_to_end(key) + return entry[0] + + def put(self, key: tuple[bytes, ...], value: tuple[bytes, ...]) -> None: + if ( + type(key) is not tuple + or type(value) is not tuple + or any(type(part) is not bytes for part in (*key, *value)) + ): + raise TypeError("immutable validation proof required") + size = 1024 + sum(sys.getsizeof(parts) for parts in (key, value)) + size += sum(sys.getsizeof(part) for part in (*key, *value)) + with self._lock: + previous = self._values.pop(key, None) + if previous is not None: + self._retained_bytes -= previous[1] + if size > self._byte_budget: + return + while self._values and ( + len(self._values) >= self._entries + or self._retained_bytes + size > self._byte_budget + ): + _, (_, released) = self._values.popitem(last=False) + self._retained_bytes -= released + self._values[key] = value, size + self._retained_bytes += size + + +# At most 64 MiB of conservatively counted proofs across both caches. They are +# performance aids, not durable state, and contain neither auth grants nor raw +# seller diagnostics. A cold process still validates the complete stored bytes. +PLANS = ValidationCache(entries=16, byte_budget=32 * 1024 * 1024) +CONFIRMATIONS = ValidationCache(entries=256, byte_budget=32 * 1024 * 1024) diff --git a/src/adcp/reporting/submissions/models.py b/src/adcp/reporting/submissions/models.py index 8920e7c46..a2c95a275 100644 --- a/src/adcp/reporting/submissions/models.py +++ b/src/adcp/reporting/submissions/models.py @@ -18,6 +18,7 @@ from adcp.reporting.evidence import consumer_reference, principal_reference from adcp.reporting.outbox.identity import canonical_consumer from adcp.reporting.receipts.wire import ReceiptBatch, validate_receipt_response +from adcp.reporting.submissions._validation_cache import CONFIRMATIONS, PLANS from adcp.types import ( ReportingAdjustmentReceipt, ReportingReceipt, @@ -174,7 +175,7 @@ class ReportingReceiptSubmission: @property def chunk_count(self) -> int: - return len(json.loads(self._plan)["requests"]) + return len(_validated_requests(self)) @property def confirmed_chunks(self) -> int: @@ -186,7 +187,7 @@ def pending(self) -> bool: def request(self, ordinal: int) -> SyncReportingReceiptsRequest: """Return a fresh typed view of one exact persisted request body.""" - body = json.loads(self._plan)["requests"][ordinal] + body = json.loads(_validated_requests(self)[ordinal]) return SyncReportingReceiptsRequest.model_validate(body) @property @@ -284,6 +285,7 @@ def prepare_reporting_receipt_submission( scope.canonical_identity + b"\n" + canonical_json_utf8_v1(items) ).hexdigest() requests = [] + encoded_requests = [] for offset in range(0, len(items), _CHUNK_SIZE): request: dict[str, Any] = { "adcp_version": _VERSION, @@ -306,10 +308,12 @@ def prepare_reporting_receipt_submission( raise ValueError ReceiptBatch.parse(request) requests.append(request) + encoded_requests.append(encoded) plan = canonical_json_utf8_v1({"version": 1, "items": items, "requests": requests}) if len(plan) > MAX_SUBMISSION_BYTES: raise ValueError result = ReportingReceiptSubmission(scope, f"reporting-submission:{fingerprint}", plan) + PLANS.put(_plan_key(result), tuple(encoded_requests)) except Exception: result = None if result is None: @@ -317,12 +321,28 @@ def prepare_reporting_receipt_submission( return result -def validate_submission(submission: ReportingReceiptSubmission) -> None: - """Check every supplied/persisted byte and confirmed result before use.""" - valid = False +def _plan_key(submission: ReportingReceiptSubmission) -> tuple[bytes, ...]: + if ( + type(submission) is not ReportingReceiptSubmission + or type(submission.scope) is not ReportingSubmissionScope + or type(submission._plan) is not bytes + or len(submission._plan) > MAX_SUBMISSION_BYTES + or type(submission.submission_id) is not str + or len(submission.submission_id) != len("reporting-submission:") + 64 + ): + raise ValueError + submission.scope.__post_init__() + return submission.scope.canonical_identity, submission.submission_id.encode(), submission._plan + + +def _validated_requests(submission: ReportingReceiptSubmission) -> tuple[bytes, ...]: + """Reuse only a proof for this exact immutable identity and complete plan.""" + result = None try: - if type(submission._plan) is not bytes or len(submission._plan) > MAX_SUBMISSION_BYTES: - raise ValueError + key = _plan_key(submission) + cached = PLANS.get(key) + if cached is not None: + return cached plan = json.loads(submission._plan) if ( type(plan) is not dict @@ -338,17 +358,44 @@ def validate_submission(submission: ReportingReceiptSubmission) -> None: ) if rebuilt._plan != submission._plan or rebuilt.submission_id != submission.submission_id: raise ValueError + result = tuple(canonical_json_utf8_v1(request) for request in plan["requests"]) + PLANS.put(key, result) + except Exception: + result = None + if result is None: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return result + + +def _confirmation_key( + submission: ReportingReceiptSubmission, request: bytes, chunk: bytes +) -> tuple[bytes, ...]: + return submission.scope.canonical_identity, submission.submission_id.encode(), request, chunk + + +def validate_submission(submission: ReportingReceiptSubmission) -> None: + """Check fresh bytes, bounds, identity and ordered confirmed prefix every time. + + Exact immutable-byte proofs avoid repeating plan and response schemas. No + stored digest or mutable model is sufficient to hit either bounded cache. + """ + valid = False + try: + requests = _validated_requests(submission) if ( - not isinstance(submission._confirmed, tuple) - or len(submission._confirmed) > len(plan["requests"]) + type(submission._confirmed) is not tuple + or len(submission._confirmed) > len(requests) or _confirmation_bytes(submission._confirmed) > MAX_CONFIRMATION_BYTES ): raise ValueError for ordinal, chunk in enumerate(submission._confirmed): if type(chunk) is not bytes or len(chunk) > MAX_RESPONSE_BYTES: raise ValueError - if _restore_confirmation(submission, ordinal, chunk) != chunk: - raise ValueError + key = _confirmation_key(submission, requests[ordinal], chunk) + if CONFIRMATIONS.get(key) is None: + if _restore_confirmation(submission, ordinal, chunk) != chunk: + raise ValueError + CONFIRMATIONS.put(key, ()) valid = True except Exception: valid = False @@ -359,15 +406,19 @@ def validate_submission(submission: ReportingReceiptSubmission) -> None: def _confirmation( submission: ReportingReceiptSubmission, ordinal: int, - response: SyncReportingReceiptsResponse, + response: SyncReportingReceiptsResponse | bytes, ) -> bytes: """Validate complete unique coverage and exact immutable success bodies.""" result = None try: - body = response.model_dump(mode="json", exclude_none=True) - if len(canonical_json_utf8_v1(body)) > MAX_RESPONSE_BYTES: + encoded = freeze_response(response) if not isinstance(response, bytes) else response + if len(encoded) > MAX_RESPONSE_BYTES: raise ValueError - batch = ReceiptBatch.parse(json.loads(submission._plan)["requests"][ordinal]) + body = json.loads(encoded) + request = _validated_requests(submission)[ordinal] + # This constructor retains only bytes already proven by complete plan + # validation. Response schema/coverage/body validation below stays fresh. + batch = ReceiptBatch(request) results = body["results"] by_id = {} for entry in results: @@ -415,6 +466,7 @@ def _confirmation( result = canonical_json_utf8_v1(normalized) if len(result) > MAX_RESPONSE_BYTES: raise ValueError + CONFIRMATIONS.put(_confirmation_key(submission, request, result), ()) except Exception: result = None if result is None: @@ -426,9 +478,12 @@ def _restore_confirmation( submission: ReportingReceiptSubmission, ordinal: int, encoded: bytes ) -> bytes: """Revalidate retained sanitized evidence, including exact result coverage.""" + batch = ReceiptBatch(_validated_requests(submission)[ordinal]) kinds = { - item["body"]["reporting_receipt_id"]: item["kind"] - for item in json.loads(submission._plan)["items"] + item["reporting_receipt_id"]: ( + "receipt" if kind == "revision_receipt" else "adjustment_receipt" + ) + for kind, item in batch.items } results = [] for item in json.loads(encoded): @@ -464,7 +519,7 @@ def _confirmation_bytes(chunks: tuple[bytes, ...]) -> int: def confirm_submission( submission: ReportingReceiptSubmission, ordinal: int, - response: SyncReportingReceiptsResponse, + response: SyncReportingReceiptsResponse | bytes, ) -> ReportingReceiptSubmission: if ( type(ordinal) is not int @@ -494,7 +549,7 @@ def confirm_submission( def encode_submission(submission: ReportingReceiptSubmission) -> tuple[str, str, str, str]: """Exact text/hashes for PostgreSQL; JSONB is not the identity store.""" - confirmed = canonical_json_utf8_v1([json.loads(chunk) for chunk in submission._confirmed]) + confirmed = b"[" + b",".join(submission._confirmed) + b"]" return ( submission._plan.decode(), hashlib.sha256(submission._plan).hexdigest(), @@ -503,6 +558,44 @@ def encode_submission(submission: ReportingReceiptSubmission) -> tuple[str, str, ) +def freeze_response(response: SyncReportingReceiptsResponse) -> bytes: + """Detach the caller's response once, before any await or storage lock.""" + encoded = None + try: + encoded = canonical_json_utf8_v1(response.model_dump(mode="json", exclude_none=True)) + if len(encoded) > MAX_RESPONSE_BYTES: + raise ValueError + except Exception: + encoded = None + if encoded is None: + raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_RESPONSE) + return encoded + + +def _decode_chunks(confirmed: str) -> tuple[bytes, ...]: + # Preserve received text for exact-byte proof lookup. Recanonicalizing all + # old chunks would repeat expensive work at every prefix. Each slice still + # requires its own canonical/schema proof in validate_submission below. + if confirmed == "[]": + return () + if not confirmed.startswith("[") or not confirmed.endswith("]"): + raise ValueError + decoder = json.JSONDecoder() + chunks = [] + offset = 1 + while offset < len(confirmed) - 1: + _, end = decoder.raw_decode(confirmed, offset) + chunks.append(confirmed[offset:end].encode()) + if len(chunks) > MAX_SUBMISSION_RECEIPTS // _CHUNK_SIZE: + raise ValueError + if end == len(confirmed) - 1: + return tuple(chunks) + if confirmed[end] != ",": + raise ValueError + offset = end + 1 + raise ValueError + + def decode_submission( scope: ReportingSubmissionScope, row: Sequence[Any] ) -> ReportingReceiptSubmission: @@ -523,7 +616,7 @@ def decode_submission( scope, identifier, plan.encode(), - tuple(canonical_json_utf8_v1(chunk) for chunk in json.loads(confirmed)), + _decode_chunks(confirmed), ) validate_submission(candidate) if ( diff --git a/src/adcp/reporting/submissions/pg.py b/src/adcp/reporting/submissions/pg.py index 50d1ff19e..4fb0b3b8b 100644 --- a/src/adcp/reporting/submissions/pg.py +++ b/src/adcp/reporting/submissions/pg.py @@ -7,7 +7,9 @@ from __future__ import annotations +import asyncio from collections.abc import Callable, Coroutine +from dataclasses import dataclass from functools import wraps from importlib.resources import files from typing import TYPE_CHECKING, Any, ParamSpec, TypeVar @@ -20,6 +22,7 @@ confirm_submission, decode_submission, encode_submission, + freeze_response, validate_submission, ) from adcp.types import SyncReportingReceiptsResponse @@ -29,6 +32,22 @@ _P = ParamSpec("_P") _R = TypeVar("_R") +_MAX_CAS_ATTEMPTS = 16 +_Row = tuple[Any, ...] + + +@dataclass(frozen=True, repr=False) +class _Snapshot: + """Complete raw state used by a read/validate/lock/compare transaction. + + Actual text, digests, pending flags, pointer and scope identity participate + in equality. A matching digest label alone never permits a write. + """ + + scope: _Row | None + current: _Row | None + selected: _Row | None + has_intents: bool def _closed_storage_errors( @@ -55,7 +74,11 @@ class PgReportingSubmissionIntentStore: with a dedicated schema/search_path and bounded connection/statement waits. There is no credential or connection string in this object's representation. Run create_schema explicitly during rollout, then validate custom backends - with the same memory/PostgreSQL state-machine vectors. + with the same memory/PostgreSQL state-machine vectors. Readiness requires all + shipped, validated CHECK definitions as well as the keys and reservation + index. Snapshot validation happens outside transactions; a mutation compares + the complete locked state, retrying at most 16 times. Contention exhaustion + raises STORAGE_UNAVAILABLE and leaves the durable intent available to resume. """ def __init__(self, *, pool: AsyncConnectionPool) -> None: @@ -83,37 +106,59 @@ async def create_schema(self) -> None: await self._check_schema_on(connection) async def _check_schema_on(self, connection: Any) -> None: - # These are the concurrency invariants, not an IF NOT EXISTS assumption. + # Verify storage bounds as well as concurrency invariants. IF NOT EXISTS + # must not silently bless a pre-created table with weaker constraints. rows = await ( await connection.execute( - "SELECT c.conname,c.contype,c.convalidated,c.condeferrable," + "SELECT c.conrelid='reporting_buyer_submission_scopes'::regclass," + " c.conname,c.contype,c.convalidated,c.condeferrable,c.connoinherit," " pg_get_constraintdef(c.oid) FROM pg_constraint c" " WHERE c.conrelid IN ('reporting_buyer_submission_scopes'::regclass," " 'reporting_buyer_submission_intents'::regclass)" ) ).fetchall() - constraints = {row[0]: tuple(row[1:]) for row in rows} + constraints = {(row[0], row[1]): tuple(row[2:]) for row in rows} required = { - "reporting_buyer_submission_scopes_pkey": ( + (True, "reporting_buyer_submission_scopes_pkey"): ( "p", True, False, + True, "PRIMARY KEY (scope_sha256)", ), - "reporting_buyer_submission_intents_pkey": ( + (False, "reporting_buyer_submission_intents_pkey"): ( "p", True, False, + True, "PRIMARY KEY (scope_sha256, submission_id)", ), - "reporting_buyer_submission_scope_fk": ( + (False, "reporting_buyer_submission_scope_fk"): ( "f", True, False, + True, "FOREIGN KEY (scope_sha256) REFERENCES " "reporting_buyer_submission_scopes(scope_sha256)", ), } + checks = ( + (True, "scope_digest", "scope_sha256 ~ '^[a-f0-9]{64}$'::text"), + (True, "scope_bound", "octet_length(canonical_identity) <= 32768"), + (False, "submission_id", "submission_id ~ '^reporting-submission:[a-f0-9]{64}$'::text"), + (False, "plan_digest", "plan_sha256 ~ '^[a-f0-9]{64}$'::text"), + (False, "confirmed_digest", "confirmed_sha256 ~ '^[a-f0-9]{64}$'::text"), + (False, "plan_bound", "octet_length(canonical_plan) <= 16777216"), + (False, "confirmed_bound", "octet_length(confirmed_results) <= 16777216"), + ) + for scopes_table, suffix, expression in checks: + required[(scopes_table, f"reporting_buyer_{suffix}")] = ( + "c", + True, + False, + False, + f"CHECK (({expression}))", + ) if any(constraints.get(name) != expected for name, expected in required.items()): raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) index = await ( @@ -127,51 +172,75 @@ async def _check_schema_on(self, connection: Any) -> None: if index is None or tuple(index) != (True, True, True, 1, "scope_sha256", "pending", True): raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) - async def _scope_on( - self, connection: Any, scope: ReportingSubmissionScope, *, create: bool = False - ) -> tuple[bool, str | None]: - if create: - await connection.execute( - "INSERT INTO reporting_buyer_submission_scopes" - " (scope_sha256,canonical_identity) VALUES (%s,%s)" - " ON CONFLICT (scope_sha256) DO NOTHING", - (scope.storage_key, scope.canonical_identity.decode()), - ) - row = await ( - await connection.execute( - "SELECT canonical_identity,current_submission_id" - " FROM reporting_buyer_submission_scopes WHERE scope_sha256=%s FOR UPDATE", - (scope.storage_key,), - ) - ).fetchone() + async def _snapshot_on( + self, connection: Any, key: str, selected_id: str | None, *, lock: bool = False + ) -> _Snapshot: + scope_query = ( + "SELECT canonical_identity,current_submission_id" + " FROM reporting_buyer_submission_scopes WHERE scope_sha256=%s FOR UPDATE" + if lock + else "SELECT canonical_identity,current_submission_id" + " FROM reporting_buyer_submission_scopes WHERE scope_sha256=%s" + ) + row = await (await connection.execute(scope_query, (key,))).fetchone() if row is None: - return False, None - if row[0] != scope.canonical_identity.decode(): - raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return _Snapshot(None, None, None, False) + intent_query = ( + "SELECT submission_id,canonical_plan,plan_sha256,confirmed_results," + " confirmed_sha256,pending FROM reporting_buyer_submission_intents" + " WHERE scope_sha256=%s AND (submission_id=%s OR submission_id=%s) FOR UPDATE" + if lock + else "SELECT submission_id,canonical_plan,plan_sha256,confirmed_results," + " confirmed_sha256,pending FROM reporting_buyer_submission_intents" + " WHERE scope_sha256=%s AND (submission_id=%s OR submission_id=%s)" + ) + rows = await (await connection.execute(intent_query, (key, row[1], selected_id))).fetchall() + by_id = {value[0]: tuple(value) for value in rows} + has_intents = bool(rows) if row[1] is None: - existing = await ( - await connection.execute( - "SELECT 1 FROM reporting_buyer_submission_intents" - " WHERE scope_sha256=%s LIMIT 1", - (scope.storage_key,), - ) - ).fetchone() - if existing is not None: - raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) - return True, row[1] + has_intents = ( + await ( + await connection.execute( + "SELECT 1 FROM reporting_buyer_submission_intents" + " WHERE scope_sha256=%s LIMIT 1", + (key,), + ) + ).fetchone() + ) is not None + return _Snapshot( + tuple(row), + by_id.get(row[1]), + by_id.get(row[1] if selected_id is None else selected_id), + has_intents, + ) - async def _read_on( - self, connection: Any, scope: ReportingSubmissionScope, submission_id: str - ) -> ReportingReceiptSubmission | None: - row = await ( - await connection.execute( - "SELECT submission_id,canonical_plan,plan_sha256,confirmed_results," - " confirmed_sha256,pending FROM reporting_buyer_submission_intents" - " WHERE scope_sha256=%s AND submission_id=%s", - (scope.storage_key, submission_id), - ) - ).fetchone() - return decode_submission(scope, row) if row is not None else None + async def _snapshot(self, key: str, selected_id: str | None) -> _Snapshot: + # Release the read-only MVCC snapshot and connection before CPU work. + # No scope row lock or caller-owned validation occurs in this phase. + async with self._pool.connection() as connection, connection.transaction(): + await connection.execute("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY") + return await self._snapshot_on(connection, key, selected_id) + + def _validate_snapshot( + self, scope: ReportingSubmissionScope, snapshot: _Snapshot + ) -> tuple[ReportingReceiptSubmission | None, ReportingReceiptSubmission | None]: + if snapshot.scope is None: + return None, None + if snapshot.scope[0] != scope.canonical_identity.decode() or ( + snapshot.scope[1] is None and snapshot.has_intents + ): + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + if snapshot.scope[1] is not None and snapshot.current is None: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + current = decode_submission(scope, snapshot.current) if snapshot.current else None + selected = ( + current + if snapshot.selected is snapshot.current + else decode_submission(scope, snapshot.selected) if snapshot.selected else None + ) + if selected is not None and selected.pending and selected != current: + raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) + return current, selected @_closed_storage_errors async def reserve(self, proposed: ReportingReceiptSubmission) -> ReportingReceiptSubmission: @@ -179,54 +248,58 @@ async def reserve(self, proposed: ReportingReceiptSubmission) -> ReportingReceip if proposed.confirmed_chunks: raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_PLAN) scope = proposed.scope - async with self._pool.connection() as connection, connection.transaction(): - _, current_id = await self._scope_on(connection, scope, create=True) - if current_id is not None: - current = await self._read_on(connection, scope, current_id) - if current is None: - raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) - if current.pending: - return current - previous = await self._read_on(connection, scope, proposed.submission_id) + key, identity = scope.storage_key, scope.canonical_identity.decode() + plan, plan_digest, confirmed, confirmed_digest = encode_submission(proposed) + for _ in range(_MAX_CAS_ATTEMPTS): + snapshot = await self._snapshot(key, proposed.submission_id) + current, previous = await asyncio.to_thread(self._validate_snapshot, scope, snapshot) + if current is not None and current.pending: + return current if previous is not None: if previous._plan != proposed._plan: raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) return previous - plan, plan_digest, confirmed, confirmed_digest = encode_submission(proposed) - await connection.execute( - "INSERT INTO reporting_buyer_submission_intents" - " (scope_sha256,submission_id,canonical_plan,plan_sha256," - " confirmed_results,confirmed_sha256,pending) VALUES (%s,%s,%s,%s,%s,%s,true)", - ( - scope.storage_key, - proposed.submission_id, - plan, - plan_digest, - confirmed, - confirmed_digest, - ), - ) - await connection.execute( - "UPDATE reporting_buyer_submission_scopes SET current_submission_id=%s" - " WHERE scope_sha256=%s", - (proposed.submission_id, scope.storage_key), - ) - return proposed + async with self._pool.connection() as connection, connection.transaction(): + expected = snapshot + if snapshot.scope is None: + await connection.execute( + "INSERT INTO reporting_buyer_submission_scopes" + " (scope_sha256,canonical_identity) VALUES (%s,%s)" + " ON CONFLICT (scope_sha256) DO NOTHING", + (key, identity), + ) + expected = _Snapshot((identity, None), None, None, False) + actual = await self._snapshot_on(connection, key, proposed.submission_id, lock=True) + if actual != expected: + continue + await connection.execute( + "INSERT INTO reporting_buyer_submission_intents" + " (scope_sha256,submission_id,canonical_plan,plan_sha256," + " confirmed_results,confirmed_sha256,pending) VALUES (%s,%s,%s,%s,%s,%s,true)", + ( + key, + proposed.submission_id, + plan, + plan_digest, + confirmed, + confirmed_digest, + ), + ) + await connection.execute( + "UPDATE reporting_buyer_submission_scopes SET current_submission_id=%s" + " WHERE scope_sha256=%s", + (proposed.submission_id, key), + ) + return proposed + raise ReportingSubmissionError(ReportingSubmissionCode.STORAGE_UNAVAILABLE) @_closed_storage_errors async def get( self, scope: ReportingSubmissionScope, submission_id: str | None = None ) -> ReportingReceiptSubmission | None: - async with self._pool.connection() as connection, connection.transaction(): - exists, current_id = await self._scope_on(connection, scope) - if not exists or current_id is None: - return None - current = await self._read_on(connection, scope, current_id) - if current is None: - raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) - if submission_id is None or submission_id == current_id: - return current - return await self._read_on(connection, scope, submission_id) + snapshot = await self._snapshot(scope.storage_key, submission_id) + _, selected = await asyncio.to_thread(self._validate_snapshot, scope, snapshot) + return selected @_closed_storage_errors async def confirm( @@ -236,20 +309,29 @@ async def confirm( chunk: int, response: SyncReportingReceiptsResponse, ) -> ReportingReceiptSubmission: - async with self._pool.connection() as connection, connection.transaction(): - exists, current_id = await self._scope_on(connection, scope) - state = await self._read_on(connection, scope, submission_id) if exists else None + # Caller-owned Pydantic objects are detached before the first await. + # Retries compare the same immutable response, even if its owner edits it. + frozen = freeze_response(response) + key = scope.storage_key + for _ in range(_MAX_CAS_ATTEMPTS): + snapshot = await self._snapshot(key, submission_id) + _, state = await asyncio.to_thread(self._validate_snapshot, scope, snapshot) if state is None: raise ReportingSubmissionError(ReportingSubmissionCode.NOT_FOUND) - if state.pending and current_id != submission_id: - raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) - updated = confirm_submission(state, chunk, response) - if updated is not state: - _, _, confirmed, digest = encode_submission(updated) - await connection.execute( - "UPDATE reporting_buyer_submission_intents" - " SET confirmed_results=%s,confirmed_sha256=%s,pending=%s" - " WHERE scope_sha256=%s AND submission_id=%s", - (confirmed, digest, updated.pending, scope.storage_key, submission_id), - ) - return updated + updated = await asyncio.to_thread(confirm_submission, state, chunk, frozen) + _, _, confirmed, digest = encode_submission(updated) + pending = updated.pending + async with self._pool.connection() as connection, connection.transaction(): + actual = await self._snapshot_on(connection, key, submission_id, lock=True) + if actual != snapshot: + continue + if updated is not state: + await connection.execute( + "UPDATE reporting_buyer_submission_intents" + " SET confirmed_results=%s,confirmed_sha256=%s,pending=%s" + " WHERE scope_sha256=%s AND submission_id=%s", + (confirmed, digest, pending, key, submission_id), + ) + return updated + # Contention never frees the lane or guesses whether another commit won. + raise ReportingSubmissionError(ReportingSubmissionCode.STORAGE_UNAVAILABLE) diff --git a/src/adcp/reporting/submissions/store.py b/src/adcp/reporting/submissions/store.py index 893224dda..77267bbb2 100644 --- a/src/adcp/reporting/submissions/store.py +++ b/src/adcp/reporting/submissions/store.py @@ -11,6 +11,7 @@ ReportingSubmissionError, ReportingSubmissionScope, confirm_submission, + freeze_response, validate_submission, ) from adcp.types import SyncReportingReceiptsResponse @@ -128,6 +129,7 @@ async def confirm( chunk: int, response: SyncReportingReceiptsResponse, ) -> ReportingReceiptSubmission: + frozen = freeze_response(response) async with self._lock: current = self._current(scope) state = self._submissions.get((scope.storage_key, submission_id)) @@ -138,6 +140,6 @@ async def confirm( state.pending and state.submission_id != current.submission_id ): raise ReportingSubmissionError(ReportingSubmissionCode.HISTORY_CORRUPT) - updated = confirm_submission(state, chunk, response) + updated = confirm_submission(state, chunk, frozen) self._submissions[(scope.storage_key, submission_id)] = updated return updated diff --git a/tests/conformance/reporting/_buyer_submission_benchmark.py b/tests/conformance/reporting/_buyer_submission_benchmark.py new file mode 100644 index 000000000..644c86a12 --- /dev/null +++ b/tests/conformance/reporting/_buyer_submission_benchmark.py @@ -0,0 +1,229 @@ +"""Reproducible buyer capacity/lock measurements, without timing assertions. + +Run with an isolated real PostgreSQL database via ADCP_PG_TEST_URL: + python -m tests.conformance.reporting._buyer_submission_benchmark + +Each cell starts with empty validation caches and a new store/schema. Inputs are +already constructed; elapsed time includes preparation, reservation, every +confirmation, and outcome access. The responder is in-process with zero network +delay; PostgreSQL cells exercise actual buyer persistence. No extrapolation is +reported as a measured result. Concurrent original gates may share the host. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import sys +import threading +import time +from contextlib import asynccontextmanager + +from adcp.reporting.submissions import ( + InMemoryReportingSubmissionIntentStore, + PgReportingSubmissionIntentStore, + models, + prepare_reporting_receipt_submission, + submit_reporting_receipts, +) +from adcp.reporting.submissions._validation_cache import ValidationCache +from adcp.types import ReportingReceipt + +from ._buyer_submission_support import SCOPE, ReceiptClient, TrustedAuthorizer, mixed, response_for +from ._generation_support import isolated_reporting_pool + + +def cold_caches(): + models.PLANS = ValidationCache(entries=16, byte_budget=32 * 1024 * 1024) + models.CONFIRMATIONS = ValidationCache(entries=256, byte_budget=32 * 1024 * 1024) + + +class MeasuredConnection: + def __init__(self, connection, measurements): + self.connection, self.measurements = connection, measurements + self.lock_started = self.lock_returned = None + + @asynccontextmanager + async def transaction(self): + try: + async with self.connection.transaction(): + yield + finally: + if self.lock_returned is not None: + released = time.perf_counter() + self.measurements.append( + { + "lock_query_seconds": self.lock_returned - self.lock_started, + "critical_section_seconds": released - self.lock_started, + "after_lock_return_to_release_seconds": released - self.lock_returned, + } + ) + + async def execute(self, query, params=None): + timed = query.startswith("SELECT canonical_identity,current_submission_id") and ( + "FOR UPDATE" in query + ) + if timed: + self.lock_started = time.perf_counter() + result = await self.connection.execute(query, params) + if timed: + self.lock_returned = time.perf_counter() + return result + + +class MeasuredPool: + def __init__(self, pool): + self.pool, self.measurements = pool, [] + + @asynccontextmanager + async def connection(self): + async with self.pool.connection() as connection: + yield MeasuredConnection(connection, self.measurements) + + +def lock_summary(measurements): + # Client-observed critical section includes lock acquisition/query transfer + # through committed transaction exit. It is an upper bound on lock hold when + # there is no external blocker, not a server-clock lock trace. + result = {"transactions": len(measurements)} + for name in ( + "lock_query_seconds", + "critical_section_seconds", + "after_lock_return_to_release_seconds", + ): + ordered = sorted(item[name] for item in measurements) + result[name] = { + "min": min(ordered), + "median": ordered[len(ordered) // 2], + "max": max(ordered), + "total": sum(ordered), + } + return result + + +async def measure(backend, count, store, measured_pool=None, *, wide=False): + cold_caches() + inputs, client = mixed(count), ReceiptClient() + if wide: + for item in inputs: + item.reporting_receipt_id += "x" * (255 - len(item.reporting_receipt_id)) + if isinstance(item, ReportingReceipt): + item.consumer_commit_ref = "c" * 255 + started, cpu = time.perf_counter(), time.process_time() + result = await submit_reporting_receipts( + client, authorizer=TrustedAuthorizer(client), store=store, receipts=inputs + ) + assert not result.pending and len(result.outcomes) == count + assert len(result.submitted_receipts) == count + expected = (count + 99) // 100 + assert len(client.requests) == result.submission.confirmed_chunks == expected + assert all( + 1 <= len(request.get("receipts", [])) + len(request.get("adjustment_receipts", [])) <= 100 + for request in client.requests + ) + record = { + "backend": backend, + "items": count, + "wide_identifiers": wide, + "chunks": expected, + "seconds": time.perf_counter() - started, + "buyer_process_cpu_seconds": time.process_time() - cpu, + "plan_bytes": len(result.submission._plan), + "confirmed_bytes": models._confirmation_bytes(result.submission._confirmed), + "cache_entries": [len(cache._values) for cache in (models.PLANS, models.CONFIRMATIONS)], + "cache_retained_bytes": [ + cache._retained_bytes for cache in (models.PLANS, models.CONFIRMATIONS) + ], + } + if measured_pool is not None: + record["locks"] = lock_summary(measured_pool.measurements) + print(json.dumps(record), flush=True) + + +async def paused_validator_waiter(): + async with isolated_reporting_pool() as pool: + measured = MeasuredPool(pool) + store = PgReportingSubmissionIntentStore(pool=measured) + await store.create_schema() + state = await store.reserve(prepare_reporting_receipt_submission(SCOPE, mixed(2000))) + entered, release = asyncio.Event(), threading.Event() + loop = asyncio.get_running_loop() + validation = [] + + class Paused(PgReportingSubmissionIntentStore): + def _validate_snapshot(self, scope, snapshot): + started = time.perf_counter() + result = super()._validate_snapshot(scope, snapshot) + validation.append(time.perf_counter() - started) + loop.call_soon_threadsafe(entered.set) + assert release.wait(20) + return result + + response = response_for(state.request(0)) + cold_caches() + measured.measurements.clear() + task = asyncio.create_task( + Paused(pool=measured).confirm(SCOPE, state.submission_id, 0, response) + ) + try: + await asyncio.wait_for(entered.wait(), 15) + started = time.perf_counter() + async with pool.connection() as connection, connection.transaction(): + await connection.execute( + "SELECT 1 FROM reporting_buyer_submission_scopes" + " WHERE scope_sha256=%s FOR UPDATE", + (SCOPE.storage_key,), + ) + elapsed = time.perf_counter() - started + assert not release.is_set() and not task.done() + release.set() + assert (await task).confirmed_chunks == 1 + print( + json.dumps( + { + "proof": "independent_pg_waiter_acquired_while_validator_paused", + "items": 2000, + "validation_seconds": validation, + "waiter_transaction_seconds": elapsed, + "locks": lock_summary(measured.measurements), + } + ), + flush=True, + ) + finally: + release.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--sizes", type=int, nargs="+", default=[100, 1000, 2000, 10000]) + parser.add_argument("--wide-identifiers", action="store_true") + parser.add_argument( + "--backends", nargs="+", choices=["memory", "postgres"], default=["memory", "postgres"] + ) + args = parser.parse_args() + print(json.dumps({"python": sys.version, "executable": sys.executable}), flush=True) + for backend in args.backends: + for count in args.sizes: + if backend == "memory": + await measure( + backend, + count, + InMemoryReportingSubmissionIntentStore(), + wide=args.wide_identifiers, + ) + else: + async with isolated_reporting_pool() as pool: + measured = MeasuredPool(pool) + store = PgReportingSubmissionIntentStore(pool=measured) + await store.create_schema() + await measure(backend, count, store, measured, wide=args.wide_identifiers) + if "postgres" in args.backends: + await paused_validator_waiter() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_cas.py b/tests/conformance/reporting/test_reporting_buyer_submission_cas.py new file mode 100644 index 000000000..78b3fd3c2 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submission_cas.py @@ -0,0 +1,276 @@ +"""Real PostgreSQL read/validate/lock/CAS races and cancellation boundaries.""" + +from __future__ import annotations + +import asyncio +import hashlib +import threading + +import pytest + +from adcp.reporting.submissions import ( + PgReportingSubmissionIntentStore, + ReportingSubmissionCode, + ReportingSubmissionError, + prepare_reporting_receipt_submission, +) +from adcp.reporting.submissions import pg as submission_pg + +from ._buyer_submission_support import SCOPE, mixed, receipt, response_for +from ._generation_support import isolated_reporting_pool + + +class PausedValidator(PgReportingSubmissionIntentStore): + """Pause CPU validation in its worker thread, without blocking the event loop.""" + + def __init__(self, *, pool, pauses=1): + super().__init__(pool=pool) + self.loop = asyncio.get_running_loop() + self.entered = [asyncio.Event() for _ in range(pauses)] + self.release = [threading.Event() for _ in range(pauses)] + self.validations = 0 + + def _validate_snapshot(self, scope, snapshot): + result = super()._validate_snapshot(scope, snapshot) + index = self.validations + self.validations += 1 + if index < len(self.entered): + self.loop.call_soon_threadsafe(self.entered[index].set) + assert self.release[index].wait(20), "test validator was not released" + return result + + def release_all(self): + for barrier in self.release: + barrier.set() + + +async def take_scope_lock(pool): + async with pool.connection() as connection, connection.transaction(): + await connection.execute( + "SELECT 1 FROM reporting_buyer_submission_scopes WHERE scope_sha256=%s FOR UPDATE", + (SCOPE.storage_key,), + ) + + +@pytest.mark.parametrize("winner", ["equivalent", "contradictory", "advanced", "new_lane"]) +async def test_paused_validation_does_not_hold_scope_lock_and_rechecks_concurrent_confirmation( + winner, +): + async with isolated_reporting_pool() as pool: + other = PgReportingSubmissionIntentStore(pool=pool) + await other.create_schema() + state = await other.reserve(prepare_reporting_receipt_submission(SCOPE, mixed(101))) + paused = PausedValidator(pool=pool) + response = response_for(state.request(0)) + task = asyncio.create_task(paused.confirm(SCOPE, state.submission_id, 0, response)) + try: + await asyncio.wait_for(paused.entered[0].wait(), 10) + # A distinct PG connection obtains FOR UPDATE while validation is + # still paused. Timeout is only a deadlock guard, not a speed claim. + await asyncio.wait_for(take_scope_lock(pool), 10) + assert not paused.release[0].is_set() + response.results.clear() # The first attempt already froze its body. + failed = {mixed(1)[0].reporting_receipt_id} if winner == "contradictory" else None + retained = await other.confirm( + SCOPE, + state.submission_id, + 0, + response_for(state.request(0), fail=failed, result="unchanged"), + ) + if winner in {"advanced", "new_lane"}: + retained = await other.confirm( + SCOPE, state.submission_id, 1, response_for(state.request(1)) + ) + latest = retained + if winner == "new_lane": + latest = await other.reserve( + prepare_reporting_receipt_submission(SCOPE, [receipt(99999)]) + ) + paused.release_all() + if winner == "contradictory": + with pytest.raises(ReportingSubmissionError) as error: + await task + assert error.value.code == ReportingSubmissionCode.INVALID_RESPONSE + else: + assert await task == retained + assert paused.validations >= 2 + assert await other.get(SCOPE) == latest + assert await other.get(SCOPE, state.submission_id) == retained + finally: + paused.release_all() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize( + "field", + [ + "canonical_plan", + "plan_sha256", + "confirmed_results", + "confirmed_sha256", + "pending", + "submission_id", + "canonical_identity", + "current_submission_id", + ], +) +async def test_cas_checks_every_state_field_and_never_overwrites_new_corruption(field): + async with isolated_reporting_pool() as pool: + from psycopg import sql + + other = PgReportingSubmissionIntentStore(pool=pool) + await other.create_schema() + state = await other.reserve(prepare_reporting_receipt_submission(SCOPE, mixed())) + paused = PausedValidator(pool=pool) + task = asyncio.create_task( + paused.confirm(SCOPE, state.submission_id, 0, response_for(state.request(0))) + ) + try: + await asyncio.wait_for(paused.entered[0].wait(), 10) + scopes = field in {"canonical_identity", "current_submission_id"} + table = ( + "reporting_buyer_submission_scopes" + if scopes + else ("reporting_buyer_submission_intents") + ) + value = { + "canonical_plan": state._plan.decode() + " ", + "plan_sha256": "0" * 64, + "confirmed_results": "[ ]", + "confirmed_sha256": "0" * 64, + "pending": False, + "submission_id": "reporting-submission:" + "e" * 64, + "canonical_identity": "{}", + "current_submission_id": None, + }[field] + async with pool.connection() as connection: + await connection.execute( + sql.SQL("UPDATE {} SET {}=%s").format( + sql.Identifier(table), sql.Identifier(field) + ), + (value,), + ) + if field in {"canonical_plan", "confirmed_results"}: + # Even a fresh correct digest must not bless changed bytes. + digest_field = ( + "plan_sha256" if field == "canonical_plan" else "confirmed_sha256" + ) + await connection.execute( + sql.SQL("UPDATE {} SET {}=%s").format( + sql.Identifier(table), sql.Identifier(digest_field) + ), + (hashlib.sha256(value.encode()).hexdigest(),), + ) + before = await ( + await connection.execute( + sql.SQL("SELECT * FROM {}").format(sql.Identifier(table)) + ) + ).fetchall() + paused.release_all() + with pytest.raises(ReportingSubmissionError) as error: + await task + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + async with pool.connection() as connection: + after = await ( + await connection.execute( + sql.SQL("SELECT * FROM {}").format(sql.Identifier(table)) + ) + ).fetchall() + assert after == before + finally: + paused.release_all() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +async def test_repeated_real_cas_losses_are_bounded_without_abandoning_the_intent(monkeypatch): + monkeypatch.setattr(submission_pg, "_MAX_CAS_ATTEMPTS", 2) + async with isolated_reporting_pool() as pool: + other = PgReportingSubmissionIntentStore(pool=pool) + await other.create_schema() + state = await other.reserve(prepare_reporting_receipt_submission(SCOPE, mixed(201))) + paused = PausedValidator(pool=pool, pauses=2) + task = asyncio.create_task( + paused.confirm(SCOPE, state.submission_id, 0, response_for(state.request(0))) + ) + try: + for chunk in range(2): + await asyncio.wait_for(paused.entered[chunk].wait(), 10) + retained = await other.confirm( + SCOPE, state.submission_id, chunk, response_for(state.request(chunk)) + ) + paused.release[chunk].set() + with pytest.raises(ReportingSubmissionError) as error: + await task + assert error.value.code == ReportingSubmissionCode.STORAGE_UNAVAILABLE + assert paused.validations == 2 + assert retained.pending and retained.confirmed_chunks == 2 + assert await other.get(SCOPE) == retained + assert ( + await other.reserve(prepare_reporting_receipt_submission(SCOPE, [receipt(99999)])) + == retained + ) + final = await other.confirm( + SCOPE, state.submission_id, 2, response_for(state.request(2)) + ) + assert not final.pending and final._confirmed[:2] == retained._confirmed + finally: + paused.release_all() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + +@pytest.mark.parametrize("phase", ["validation", "locked", "waiting"]) +async def test_cancel_at_new_boundary_releases_transactions_and_retains_uncertainty(phase): + async with isolated_reporting_pool() as pool: + other = PgReportingSubmissionIntentStore(pool=pool) + await other.create_schema() + state = await other.reserve(prepare_reporting_receipt_submission(SCOPE, mixed())) + entered, release = asyncio.Event(), asyncio.Event() + + class PausedLock(PgReportingSubmissionIntentStore): + async def _snapshot_on(self, connection, key, selected_id, *, lock=False): + if lock and phase == "waiting": + entered.set() + result = await super()._snapshot_on(connection, key, selected_id, lock=lock) + if lock and phase == "locked": + entered.set() + await release.wait() + return result + + paused = PausedValidator(pool=pool) if phase == "validation" else PausedLock(pool=pool) + + async def run_and_cancel(): + task = asyncio.create_task( + paused.confirm(SCOPE, state.submission_id, 0, response_for(state.request(0))) + ) + barrier = paused.entered[0] if phase == "validation" else entered + try: + await asyncio.wait_for(barrier.wait(), 10) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + finally: + if isinstance(paused, PausedValidator): + paused.release_all() + release.set() + task.cancel() + await asyncio.gather(task, return_exceptions=True) + + if phase == "waiting": + async with pool.connection() as connection, connection.transaction(): + await connection.execute( + "SELECT 1 FROM reporting_buyer_submission_scopes" + " WHERE scope_sha256=%s FOR UPDATE", + (SCOPE.storage_key,), + ) + await run_and_cancel() + else: + await run_and_cancel() + await asyncio.wait_for(take_scope_lock(pool), 10) + assert await other.get(SCOPE) == state + restored = await other.confirm( + SCOPE, state.submission_id, 0, response_for(state.request(0)) + ) + assert not restored.pending diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_migration.py b/tests/conformance/reporting/test_reporting_buyer_submission_migration.py index 175249b69..29bfc7768 100644 --- a/tests/conformance/reporting/test_reporting_buyer_submission_migration.py +++ b/tests/conformance/reporting/test_reporting_buyer_submission_migration.py @@ -54,6 +54,91 @@ async def test_existing_weakened_reservation_index_fails_rollout_closed(): assert error.value.__context__ is None +@pytest.mark.parametrize( + "table,suffix", + [ + ("scopes", "scope_digest"), + ("scopes", "scope_bound"), + ("intents", "submission_id"), + ("intents", "plan_digest"), + ("intents", "confirmed_digest"), + ("intents", "plan_bound"), + ("intents", "confirmed_bound"), + ], +) +@pytest.mark.parametrize("replacement", [None, "CHECK (true)"]) +async def test_each_missing_or_weakened_check_constraint_fails_rollout_closed( + table, suffix, replacement +): + async with isolated_reporting_pool() as pool: + from psycopg import sql + + store = PgReportingSubmissionIntentStore(pool=pool) + await store.create_schema() + table_name = sql.Identifier(f"reporting_buyer_submission_{table}") + constraint_name = sql.Identifier(f"reporting_buyer_{suffix}") + async with pool.connection() as connection: + await connection.execute( + sql.SQL("ALTER TABLE {} DROP CONSTRAINT {}").format(table_name, constraint_name) + ) + if replacement is not None: + await connection.execute( + sql.SQL("ALTER TABLE {} ADD CONSTRAINT {} CHECK (true)").format( + table_name, constraint_name + ) + ) + # create_schema must leave the weakened schema rejected; no automatic + # repair or previously completed migration can stand in for validation. + for _ in range(2): + with pytest.raises(ReportingSubmissionError) as error: + await store.create_schema() + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + assert error.value.__context__ is None and error.value.__cause__ is None + + +@pytest.mark.parametrize("modifier", ["NOT VALID", "NO INHERIT"]) +async def test_correct_check_expression_with_weakened_enforcement_is_rejected(modifier): + async with isolated_reporting_pool() as pool: + from psycopg import sql + + store = PgReportingSubmissionIntentStore(pool=pool) + await store.create_schema() + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE reporting_buyer_submission_intents" + " DROP CONSTRAINT reporting_buyer_plan_bound" + ) + await connection.execute( + sql.SQL( + "ALTER TABLE reporting_buyer_submission_intents" + " ADD CONSTRAINT reporting_buyer_plan_bound" + " CHECK (octet_length(canonical_plan) <= 16777216) {}" + ).format(sql.SQL(modifier)) + ) + with pytest.raises(ReportingSubmissionError) as error: + await store.create_schema() + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + + +async def test_identically_named_check_on_the_wrong_table_does_not_prove_readiness(): + async with isolated_reporting_pool() as pool: + store = PgReportingSubmissionIntentStore(pool=pool) + await store.create_schema() + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE reporting_buyer_submission_scopes" + " DROP CONSTRAINT reporting_buyer_scope_digest" + ) + await connection.execute( + "ALTER TABLE reporting_buyer_submission_intents" + " ADD CONSTRAINT reporting_buyer_scope_digest" + " CHECK (scope_sha256 ~ '^[a-f0-9]{64}$')" + ) + with pytest.raises(ReportingSubmissionError) as error: + await store.create_schema() + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + + @pytest.mark.parametrize( "mutation", ["plan", "confirmed", "identity", "pointer", "pending", "driver"] ) diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_validation.py b/tests/conformance/reporting/test_reporting_buyer_submission_validation.py new file mode 100644 index 000000000..534a4109b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_buyer_submission_validation.py @@ -0,0 +1,154 @@ +"""Scaled validation work, exact-byte cache isolation and bounded retention.""" + +from __future__ import annotations + +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.receipts.wire import ReceiptBatch +from adcp.reporting.submissions import ( + InMemoryReportingSubmissionIntentStore, + ReportingSubmissionCode, + ReportingSubmissionError, + models, + prepare_reporting_receipt_submission, + submit_reporting_receipts, +) +from adcp.reporting.submissions._validation_cache import ValidationCache + +from ._buyer_submission_support import SCOPE, ReceiptClient, TrustedAuthorizer, mixed, response_for + + +def cold_caches(monkeypatch): + monkeypatch.setattr(models, "PLANS", ValidationCache(entries=16, byte_budget=32 * 1024 * 1024)) + monkeypatch.setattr( + models, "CONFIRMATIONS", ValidationCache(entries=256, byte_budget=32 * 1024 * 1024) + ) + + +async def test_thousand_mixed_items_validate_schemas_once_per_chunk_and_revalidate_after_eviction( + monkeypatch, +): + cold_caches(monkeypatch) + parses = responses = 0 + parse = ReceiptBatch.parse + validate_response = models.validate_receipt_response + + def counting_parse(cls, body): + nonlocal parses + parses += 1 + return parse(body) + + def counting_response(body, batch): + nonlocal responses + responses += 1 + return validate_response(body, batch) + + monkeypatch.setattr(ReceiptBatch, "parse", classmethod(counting_parse)) + monkeypatch.setattr(models, "validate_receipt_response", counting_response) + client, store = ReceiptClient(), InMemoryReportingSubmissionIntentStore() + result = await submit_reporting_receipts( + client, authorizer=TrustedAuthorizer(client), store=store, receipts=mixed(1000) + ) + assert not result.pending and len(result.outcomes) == 1000 + assert len(client.requests) == parses == responses == 10 + for _ in range(3): + assert await store.get(SCOPE) == result.submission + assert (parses, responses) == (10, 10) + cold_caches(monkeypatch) + assert await store.get(SCOPE) == result.submission + assert (parses, responses) == (20, 20) + assert await store.get(SCOPE) == result.submission + assert (parses, responses) == (20, 20) + + +@pytest.mark.parametrize("changed", ["scope", "id", "plan", "mutable_plan", "mutable_prefix"]) +def test_warm_proof_never_uses_object_or_digest_identity(changed, monkeypatch): + cold_caches(monkeypatch) + state = prepare_reporting_receipt_submission(SCOPE, mixed()) + models.validate_submission(state) + if changed == "scope": + state = replace(state, scope=replace(SCOPE, consumer_id="another-consumer")) + elif changed == "id": + state = replace(state, submission_id="reporting-submission:" + "0" * 64) + elif changed == "plan": + plan = json.loads(state._plan) + plan["requests"][0]["account"]["account_id"] = "asserted-account" + state = replace(state, _plan=canonical_json_utf8_v1(plan)) + elif changed == "mutable_plan": + state = replace(state, _plan=bytearray(state._plan)) + else: + state = replace(state, _confirmed=[]) + with pytest.raises(ReportingSubmissionError) as error: + models.validate_submission(state) + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + + +@pytest.mark.parametrize("changed", ["body", "space", "order", "wrong_chunk"]) +async def test_warm_confirmation_proof_requires_exact_bytes_and_exact_chunk(changed, monkeypatch): + cold_caches(monkeypatch) + store = InMemoryReportingSubmissionIntentStore() + state = await store.reserve(prepare_reporting_receipt_submission(SCOPE, mixed(101))) + for chunk in range(2): + state = await store.confirm( + SCOPE, state.submission_id, chunk, response_for(state.request(chunk)) + ) + models.validate_submission(state) + first = json.loads(state._confirmed[0]) + if changed == "body": + first[0]["receipt"]["observed_manifest_sha256"] = "e" * 64 + altered = (canonical_json_utf8_v1(first), *state._confirmed[1:]) + elif changed == "space": + altered = (state._confirmed[0] + b" ", *state._confirmed[1:]) + elif changed == "order": + altered = (canonical_json_utf8_v1(list(reversed(first))), *state._confirmed[1:]) + else: + altered = tuple(reversed(state._confirmed)) + with pytest.raises(ReportingSubmissionError) as error: + models.validate_submission(replace(state, _confirmed=altered)) + assert error.value.code == ReportingSubmissionCode.HISTORY_CORRUPT + assert await store.get(SCOPE) == state + + +async def test_cached_evidence_has_no_mutable_views_or_raw_private_diagnostics(monkeypatch): + cold_caches(monkeypatch) + inputs = mixed() + client = ReceiptClient(fail={inputs[0].reporting_receipt_id}) + result = await submit_reporting_receipts( + client, + authorizer=TrustedAuthorizer(client), + store=InMemoryReportingSubmissionIntentStore(), + receipts=inputs, + ) + before = result.submission._plan, result.submission._confirmed + inputs.clear() + result.submission.request(0).receipts.clear() + result.outcomes[1].submitted_receipt.observed_manifest_sha256 = "e" * 64 + result.outcomes[1].receipt.observed_manifest_sha256 = "f" * 64 + assert (result.submission._plan, result.submission._confirmed) == before + for cache in (models.PLANS, models.CONFIRMATIONS): + for key, (value, _) in cache._values.items(): + assert type(key) is tuple and type(value) is tuple + assert all(type(part) is bytes for part in (*key, *value)) + assert b"PRIVATE_SENTINEL" not in b"".join((*key, *value)) + models.validate_submission(result.submission) + + +@pytest.mark.parametrize("entries,budget", [(2, 100000), (100, 2500)]) +def test_validation_proof_retention_has_entry_and_conservative_byte_bounds(entries, budget): + cache = ValidationCache(entries=entries, byte_budget=budget) + for index in range(10): + key, value = (str(index).encode(),), (b"validated",) + cache.put(key, value) + assert cache.get(key) == value + assert len(cache._values) <= entries + assert cache._retained_bytes <= budget + assert cache.get((b"0",)) is None + cache.put((b"too large",), (b"x" * budget,)) + assert cache.get((b"too large",)) is None + assert cache._retained_bytes == sum(size for _, size in cache._values.values()) + with pytest.raises(TypeError): + cache.put((b"mutable",), (bytearray(b"x"),)) From 0a0fb51fcbd56182f3ad71d9cd8a2dcdf6aee102 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 26 Sep 2026 14:51:34 +0000 Subject: [PATCH 3/3] fix(ci): allow time for Postgres packaging coverage Keep the full core selection with a 30-minute job budget and preserve the process lane at 15 minutes. Address buyer submission review comments on redacted errors and test assertions. --- .github/workflows/ci.yml | 13 +++++++++---- src/adcp/reporting/submissions/models.py | 1 + src/adcp/reporting/submissions/pg.py | 1 + .../test_reporting_buyer_submission_migration.py | 4 ++-- .../reporting/test_reporting_buyer_submissions.py | 2 +- 5 files changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 19ddb7f93..d74deaba7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -135,13 +135,18 @@ jobs: pg-conformance: name: Postgres conformance tests (Postgres 16, ${{ matrix.lane }}) runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: ${{ matrix.timeout_minutes }} permissions: contents: read strategy: fail-fast: false matrix: - lane: [core, process] + include: + # Core also builds and installs the reporting wheel/sdist fixtures. + - lane: core + timeout_minutes: 30 + - lane: process + timeout_minutes: 15 services: postgres: # CI-local ephemeral database. POSTGRES_HOST_AUTH_METHOD=trust @@ -184,8 +189,8 @@ jobs: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_test PG_LANE: ${{ matrix.lane }} run: | - # Keep every case and its deadline. Separate process-crash controls - # so setup and teardown also fit inside each unchanged job budget. + # Keep every case and its per-case deadline. The core lane includes + # installed artifact coverage and has a larger overall job budget. case "$PG_LANE" in core) python scripts/reporting_test_harness.py pytest tests/conformance/signing/test_pg_replay_store.py \ diff --git a/src/adcp/reporting/submissions/models.py b/src/adcp/reporting/submissions/models.py index a2c95a275..59c5d126e 100644 --- a/src/adcp/reporting/submissions/models.py +++ b/src/adcp/reporting/submissions/models.py @@ -117,6 +117,7 @@ def __post_init__(self) -> None: canonical_consumer(self.consumer_id) valid = all(value == value.strip() for value in (self.seller_id, self.account_id)) except (ValueError, TypeError, RuntimeError): + # Map provider validation details to the closed INVALID_SCOPE error below. pass if not valid: raise ReportingSubmissionError(ReportingSubmissionCode.INVALID_SCOPE) diff --git a/src/adcp/reporting/submissions/pg.py b/src/adcp/reporting/submissions/pg.py index 4fb0b3b8b..80ed6112f 100644 --- a/src/adcp/reporting/submissions/pg.py +++ b/src/adcp/reporting/submissions/pg.py @@ -92,6 +92,7 @@ async def create_schema(self) -> None: available = True except ImportError: + # The optional driver is absent; report PG_REQUIRED outside the handler. pass if not available: raise ReportingSubmissionError(ReportingSubmissionCode.PG_REQUIRED) diff --git a/tests/conformance/reporting/test_reporting_buyer_submission_migration.py b/tests/conformance/reporting/test_reporting_buyer_submission_migration.py index 29bfc7768..2794ce811 100644 --- a/tests/conformance/reporting/test_reporting_buyer_submission_migration.py +++ b/tests/conformance/reporting/test_reporting_buyer_submission_migration.py @@ -172,9 +172,9 @@ async def test_pg_corruption_and_driver_diagnostics_never_escape_or_permit_repla await connection.execute( "ALTER TABLE reporting_buyer_submission_intents DROP COLUMN confirmed_results" ) - for operation in (store.get(SCOPE), store.reserve(state)): + for operation in (lambda: store.get(SCOPE), lambda: store.reserve(state)): with pytest.raises(ReportingSubmissionError) as error: - await operation + await operation() assert error.value.code == ( ReportingSubmissionCode.STORAGE_UNAVAILABLE if mutation == "driver" diff --git a/tests/conformance/reporting/test_reporting_buyer_submissions.py b/tests/conformance/reporting/test_reporting_buyer_submissions.py index 3bc91408e..0d69ea8f9 100644 --- a/tests/conformance/reporting/test_reporting_buyer_submissions.py +++ b/tests/conformance/reporting/test_reporting_buyer_submissions.py @@ -240,7 +240,7 @@ def test_outbound_capture_is_immutable_and_each_public_model_view_is_detached(): assert first == same inputs[0].observed_adjustment_sha256 = "f" * 64 first.request(0).adjustment_receipts[0].observed_adjustment_sha256 = "e" * 64 - assert first == same + assert same.request(0).adjustment_receipts[0].observed_adjustment_sha256 == "b" * 64 assert first.request(0).adjustment_receipts[0].observed_adjustment_sha256 == "b" * 64 assert ( prepare_reporting_receipt_submission(SCOPE, list(reversed(mixed()))).submission_id