diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1cddb242f..139b8a3e8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,7 @@ on: - conductor/reporting-webhook-activity-1168b - conductor/reporting-status-notifications-1168c - conductor/1167b1-materializer-contracts + - conductor/1167b2-durable-managed-reporting # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -106,7 +107,7 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py examples/reporting_receipt_ingress.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' @@ -119,12 +120,12 @@ jobs: - name: Run tests if: matrix.python-version != '3.12' timeout-minutes: 45 - run: pytest tests/ -v + run: python scripts/reporting_test_harness.py pytest tests/ -v -ra - name: Run tests with coverage if: matrix.python-version == '3.12' timeout-minutes: 45 - run: pytest tests/ -v --cov=src/adcp --cov-report=term-missing + run: python scripts/reporting_test_harness.py pytest tests/ -v -ra --cov=src/adcp --cov-report=term-missing pg-conformance: name: Postgres conformance tests (Postgres 16, ${{ matrix.lane }}) @@ -182,7 +183,7 @@ jobs: # so setup and teardown also fit inside each unchanged job budget. case "$PG_LANE" in core) - pytest tests/conformance/signing/test_pg_replay_store.py \ + python scripts/reporting_test_harness.py pytest tests/conformance/signing/test_pg_replay_store.py \ tests/conformance/signing/test_pg_replay_store_e2e.py \ tests/conformance/decisioning/test_pg_buyer_agent_registry.py \ tests/conformance/decisioning/test_pg_idempotency_backend.py \ @@ -195,10 +196,11 @@ jobs: --ignore=tests/conformance/reporting/test_reporting_materializer_process.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_migration.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ - -v + --ignore-glob='tests/conformance/reporting/test_reporting_receipt_*.py' \ + -v -ra ;; process) - pytest tests/conformance/reporting/test_reporting_notification_process_matrix.py -v + python scripts/reporting_test_harness.py pytest tests/conformance/reporting/test_reporting_notification_process_matrix.py -v -ra ;; *) echo "Unknown Postgres conformance lane" @@ -211,7 +213,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: {} - needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer] + needs: [pg-conformance, pg-reporting-status, pg-reporting-materializer, pg-reporting-receipts, pg-reporting-receipt-compatibility] if: ${{ always() }} steps: - name: Require every Postgres conformance lane @@ -219,11 +221,15 @@ jobs: PG_RESULT: ${{ needs.pg-conformance.result }} STATUS_RESULT: ${{ needs.pg-reporting-status.result }} MATERIALIZER_RESULT: ${{ needs.pg-reporting-materializer.result }} + RECEIPTS_RESULT: ${{ needs.pg-reporting-receipts.result }} + RECEIPT_COMPATIBILITY_RESULT: ${{ needs.pg-reporting-receipt-compatibility.result }} run: | - if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ]; then + if [ "$PG_RESULT" != "success" ] || [ "$STATUS_RESULT" != "success" ] || [ "$MATERIALIZER_RESULT" != "success" ] || [ "$RECEIPTS_RESULT" != "success" ] || [ "$RECEIPT_COMPATIBILITY_RESULT" != "success" ]; then echo "Postgres conformance matrix result: $PG_RESULT" echo "Reporting status conformance result: $STATUS_RESULT" echo "Durable materializer conformance result: $MATERIALIZER_RESULT" + echo "Authenticated receipt conformance result: $RECEIPTS_RESULT" + echo "Receipt rolling compatibility result: $RECEIPT_COMPATIBILITY_RESULT" exit 1 fi echo "All Postgres conformance lanes passed" @@ -273,7 +279,7 @@ jobs: - name: Run status semantics, transaction, clock/process and rolling gates env: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_status_test - run: pytest tests/conformance/reporting/test_reporting_status_*.py -v + run: python scripts/reporting_test_harness.py pytest tests/conformance/reporting/test_reporting_status_*.py -v -ra pg-reporting-materializer: name: Durable materializer and frozen artifacts (Postgres 16) @@ -331,11 +337,12 @@ jobs: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_materializer_test ADCP_PYTHON310: ${{ steps.materializer-python310.outputs.python-path }} run: | - pytest tests/conformance/reporting/test_reporting_materializer_rolling.py \ + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_materializer_rolling.py \ tests/conformance/reporting/test_reporting_materializer_migration.py \ tests/conformance/reporting/test_reporting_materializer_process.py \ tests/conformance/reporting/test_reporting_materializer_installed_pg.py \ - -v -s | tee materializer-evidence.log + -v -s -ra | tee materializer-evidence.log - name: Preserve installed module and worker evidence if: always() uses: actions/upload-artifact@v7 @@ -344,6 +351,129 @@ jobs: path: materializer-evidence.log if-no-files-found: error + pg-reporting-receipts: + name: Authenticated receipts and installed replay (Postgres 16) + runs-on: ubuntu-latest + # Separate databases avoid shared account advisory locks. Keep five minutes + # outside the bounded suite for checkout, installations and cleanup. + timeout-minutes: 35 + permissions: + contents: read + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_receipts_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + id: receipt-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run receipt transactions, real mounts, crash and installed gates + shell: bash + timeout-minutes: 30 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_receipts_test + ADCP_PYTHON310: ${{ steps.receipt-python310.outputs.python-path }} + run: | + # Explicit pytest paths bypass --ignore; exclude rolling before invocation. + shopt -s nullglob + receipt_tests=() + for test_file in tests/conformance/reporting/test_reporting_receipt_*.py; do + if [[ "$test_file" != tests/conformance/reporting/test_reporting_receipt_rolling.py ]]; then + receipt_tests+=("$test_file") + fi + done + if (( ${#receipt_tests[@]} == 0 )); then + echo "No receipt test files selected" >&2 + exit 1 + fi + python scripts/reporting_test_harness.py pytest \ + "${receipt_tests[@]}" \ + -v -s -ra | tee pg-reporting-receipts-evidence.log + - name: Preserve commands, counts, installed provenance and skip reasons + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-receipts-evidence-${{ github.run_attempt }} + path: pg-reporting-receipts-evidence.log + if-no-files-found: error + + pg-reporting-receipt-compatibility: + name: Receipt rolling compatibility (eight actual artifacts) + runs-on: ubuntu-latest + # Separate databases avoid shared account advisory locks. Keep five minutes + # outside the bounded suite for checkout, installations and cleanup. + timeout-minutes: 35 + permissions: + contents: read + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_receipt_rolling_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 5s + --health-timeout 5s + --health-retries 10 + steps: + - uses: actions/checkout@v6 + - name: Fetch all eight exact approved historical artifacts + timeout-minutes: 2 + run: | + git fetch --no-tags --depth=1 origin \ + 3e76aa54623529a3dda01cd690b8a5c287c75641 \ + 3c405a21f978ed9d3208611bb4a7a8434a056933 \ + 037de4ac822ecefb2f95d32c15c297fb4c45d683 \ + 17ee407ae3978c8a2bb54437287afbf9dafb8130 \ + 0f34c666ac1961e9832fce43ef0ef6937b3c1dde \ + 967b6e286301d7e5d089aea6fdbb90bea8ee5a16 \ + 5487f2bdef23c5102118b305be9e868228f6ce61 \ + 3fd62121c96a074e3ea458c30c5224d6a586f169 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run all historical binaries against approved parent and receipt schema + shell: bash + timeout-minutes: 30 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_receipt_rolling_test + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_receipt_rolling.py \ + -v -s -ra | tee pg-reporting-receipt-compatibility-evidence.log + - name: Preserve commands, counts, installed provenance and skip reasons + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-receipt-compatibility-evidence-${{ github.run_attempt }} + path: pg-reporting-receipt-compatibility-evidence.log + if-no-files-found: error + conventional-commits: name: Validate conventional commit format runs-on: ubuntu-latest diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index 2a8f7a038..ce86531ea 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -3,7 +3,7 @@ name: PR Title Check on: pull_request: types: [opened, edited, synchronize, reopened] - branches: [main] + branches: [main, conductor/1167b2-durable-managed-reporting] permissions: contents: read diff --git a/docs/reporting-receipt-ingress.md b/docs/reporting-receipt-ingress.md new file mode 100644 index 000000000..578e48690 --- /dev/null +++ b/docs/reporting-receipt-ingress.md @@ -0,0 +1,211 @@ +# Authenticated durable reporting receipts + +`adcp.reporting.receipts.ReportingReceiptHandler` mounts the experimental +`sync_reporting_receipts` task over the SDK's MCP and A2A transports. It accepts +revision and official-adjustment receipts, persists each item outcome, and +replays the original completed response after a process restart. Use +`PgReportingReceiptStore` in production. It extends the existing durable +materializer store; no second materializer or required legacy store method is +introduced. The memory store is a conformance/reference implementation. + +This is **B2.2 of 4 within B2 of B1/B2** (Refs #1167). B2.3 owns frozen authorized +combined feeds, checkpoints and dependency closure. B2.4 owns versioned public +status, private ownership projection, complete ownership-walk validation and +per-offering production capability proof. Installing this slice does not make +Managed Delivery or Reconciled Billing ready. The existing new-store notification +advertisement veto also covers Core `reporting.ledger_changed`; Core polling is +unaffected. A deployment needing current Core notification advertisement may +retain the reviewed Core store for that surface. B2.4 owns deliberate admission +of the new store once all actual components and mounts are proven ready. + +## Composition and identity + +[The typed example](../examples/reporting_receipt_ingress.py) composes the store, +live account authorization callback and optional buyer registry, then uses +`serve(transport="both", auth=..., context_factory=auth_context_factory)`. +The same store participates in +[the durable materializer composition](../examples/reporting_durable_materializer.py). +A token validator supplies trusted principals. A registry-backed API/OAuth or +signed-agent adapter supplies trusted `AuthInfo` and its credential reference; +the registry is consulted again on every call, including exact replay. + +`resolve_account(reference, context, canonical_consumer)` must resolve the exact +account reference and check the current application ACL. Return the canonical +storage account ID; unknown and denied accounts raise +`ReportingReceiptError("UNAUTHORIZED")`. An opaque account ID is sufficient; +a natural account key is resolved through the same callback. Authorization must +not be cached solely by the batch key. A hydrated `RequestContext.account.id` +must equal the resolved account. + +The SDK cross-checks all present trusted principals, `AuthInfo`, signed agent +identity and active `BuyerAgent.agent_url`, using the reviewed canonical consumer +resolver. A raw `ToolContext.caller_identity` may identify the consumer. +`RequestContext.caller_identity` is an **opaque AccountStore cache key** and is +never parsed as a consumer. `tenant_id` and body fields cannot supply identity. +Absent, anonymous, inactive or conflicting identities fail closed. Two consumers +in one account and one consumer in two accounts have independent batch keys and +receipt visibility. + +## Wire and replay contract + +Requests negotiate `adcp_version: "3.2-rc.6"`. This is the wire release spelling; +`3.2.0-rc.6` is the bundle's semantic-version spelling. Unnegotiated traffic keeps +the SDK's existing AdCP 3.0 behavior; unsupported versions remain unsupported. +Pinned, unpinned and fallback MCP inventories use an isolated schema overlay. +Cached upstream schemas, generated models and older required protocols remain +unchanged. + +Before any batch or receipt mutation the SDK validates the entire request: + +- Supply `receipts`, `adjustment_receipts`, or both. Every supplied array is + nonempty; the combined size is 1–100. IDs are unique across both arrays. +- Do not supply `received_at`. PostgreSQL assigns it inside the item transaction. +- Follow the exact receipt schema, including evidence and timestamp formats. + Reporting's canonical JSON profile uses safe integers and decimal strings; + nonintegral numbers, nonfinite numbers, duplicate JSON keys, unsafe integers and + ambiguous requests and strings outside PostgreSQL JSONB's Unicode domain + (U+0000 or surrogate code points) are rejected. Receipt-specific SQL hashing + preserves the canonical UTF-16 key ordering for all accepted Unicode keys in + `context` and `ext`; earlier schema functions and manifests stay unchanged. +- Account and canonical consumer come from the trusted resolver and context. + Request identity fields cannot override them. + +Malformed shape produces task-level `INVALID_REQUEST` with zero writes. +Semantic failures are durable item results. Unknown and unauthorized referenced +records use the same safe `REPORTING_RECORD_UNAVAILABLE` item shape. Results are +revision receipts in input order followed by adjustment receipts in input order. +A failure of either kind has top-level `reporting_receipt_id` and `errors`; +a success has `receipt` or `adjustment_receipt`, including server `received_at`. + +The immutable batch header is keyed by `(account, canonical consumer, +idempotency_key)`. It stores the canonical **whole request**, digest and expected +count; `context` and `ext` participate in identity. Every ordinal, including a +failure, is durable. The final response is persisted and validated against the +original ordered items. Changed content conflicts before further work. Replays +return original `recorded`/failure results and original server timestamps, without +adding `replayed` or applying a response enhancer. + +The SDK generic idempotency wrapper and middleware are bypassed for this task: +the generic cache lacks the resolved account and can bypass revoked +authorization. Application middleware must not independently cache receipt +responses or rewrite admitted receipt parameters. Receipt pre-validation hooks +may inspect an isolated copy, but changing its content is rejected. + +The production HTTP mounts retain bounded raw bytes in the current request's +ASGI scope because A2A protobuf `Struct` normalizes JSON integers to floats. +The A2A wrapper checks a uniquely identified receipt's Unicode encoding before +protobuf decoding. Receipt parameter extraction remains bound to the selected +route and normal transport authentication; account and consumer authorization +still run in the handler. Raw bytes are never sourced from client metadata, +shared context caches or request-ID lookups. Both the normal body limiter and +the receipt capture bound apply. The raw request must identify +one unambiguous standard receipt invocation; lossy direct/custom transport inputs +fail closed. All normal whole-shape, numeric, account and consumer checks still +run. This does not replace the A2A stack or alter other tasks' version handling. +Exact integral HTTP numeric spellings such as protobuf's `1.0` are decoded from +their original text without floating-point rounding and canonicalized as `1`. +An already rounded direct/custom parameter value cannot supply that evidence +and is rejected. Fractional text that would round to an integer is also rejected. + +## Financial evidence and transactions + +Revision receipts match the exact account, consumer, obligation, revision, +materialization, verification profile, digest and control totals. Acceptance +requires readable evidence for that artifact at observation. Receipt chains are +single, complete, acyclic chains with no cross-target edges. Only the current +rejected leaf may be replaced. An accepted leaf is terminal. Adjustment receipts +match the exact official revision, canonical adjustment digest, finality and +correction/creation/observation ordering. PostgreSQL predicates also enforce these +relationships for ordinary SQL inserts. + +An accepted receipt remains authoritative for its referenced immutable artifact +after another materialization succeeds or fails, or after that artifact expires +or becomes corrupt/unreadable. Current readability is assessed separately. +Acceptance does not promise current availability; later health does not rewrite +acceptance. Consumer rejection never allocates a materializer attempt or dirties +retry work. + +For each item the account advisory lock is acquired before the batch row lock. +One connection-bound transaction chooses the next durable ordinal and commits +its receipt, private caller feed entry, ordinary status-dirty behavior, required +immutable captured status input and exact ordinal outcome. A semantic failure +commits only that ordinal outcome. A storage/capture/result failure rolls the +whole current ordinal back; previously committed sibling outcomes remain. +Memory rollback is unconditional, including first-use collections and sequence +heads, with notifications both off and on. + +Captured receipt inputs are isolated epoch-zero records with their own caller +sequence and B2.1's existing account capture sequence. They retain exact private +consumer and obligation/revision dependencies even when consumer status is +disabled. They do not activate a public projection, release historical events or +produce readiness. B2.1's terminal outcome + captured status input + work ACK + +enabled logical enqueue remains the original transaction. Its preactivation +work/events stay permanently quarantined across replay and later activation. +Ordinary terminal materialization writes still mark status dirty without +readiness, and argument bounds remain actionable closed errors. + +## Migration, rollout and recovery + +1. Back up and migrate with `await store.create_schema()` using a deployment + connection. The additive `reporting_receipt_ingestion.sql` and independent + `receipts/required_schema.json` ship in both wheel distribution paths. Do not + modify A/B/C manifests or the **187-object** B2.1 materializer manifest. + This migration also carries an additive Core fix: period-close leasing gains + a durable, total fairness order. Without it a worker that releases each turn + can re-lease one generation forever and never close any other account's + periods. The rank is held in a private `adcp_reporting_configuration_lease_turns` + table, outside the enumerated `reporting_*` catalog, so every existing + manifest and `reporting_configurations`' own shape stay byte-identical for + older readers. A generation with no rank row yet ranks as never leased, so + writers that predate the table starve nothing. Because the table is + deliberately invisible to the schema manifests, readiness cannot assert it: + run `create_schema()` before starting a worker, or the first lease fails + loudly on the missing relation rather than degrading silently. +2. Verify `await store.receipt_ingestion_ready()` on the actual installed store. + Empty, old, partial, disabled-trigger and mismatched schemas fail closed, + including completed replay. The migration is transactional, repeatable and + serialized; interrupted installation exposes no partial schema. +3. Drain legacy materialization writers before autonomous materializer activation. + Preserve [B2.1's legacy-pending recovery/import rules](reporting-durable-materializer.md). + Never invent an external idempotency history or reuse quarantined epoch-zero + evidence for later activation. Old ordinary writers are compatible during a + controlled rolling deployment; they are not autonomous workers to run in + parallel with the new service. +4. Mount the authenticated receipt handler and verify its actual MCP/A2A routes. + Deploy the same durable store to restarted instances. Keep account resolution + and registry revocation live. Do not advertise incomplete tier capabilities. +5. On timeout, cancellation, process death or uncertain commit, retry the identical + body and key. The durable prefix resumes with no duplicated receipt or changed + timestamp. Final-response failure retains earlier ordinals and rebuilds the + response from them. `IDEMPOTENCY_CONFLICT` requires recovering the original + submission; do not erase the batch to reuse its key. +6. For `RECEIPT_SCHEMA_UNREADY` repair the installed migration. For + `RECEIPT_HISTORY_CORRUPT` stop ingress and investigate the immutable evidence; + do not rewrite financial outcomes. `RECEIPT_STORAGE_UNAVAILABLE` is a safe + storage boundary error; retry the original body after service recovery. + Closed public errors omit provider bodies, credentials and SQL diagnostics. + +Keep batches, ordinals and captures at least as long as their referenced receipt +history and replay obligation. This slice does not prune them. Database owners +can bypass SQL protections, so migrations and repair remain operator-controlled; +application roles must not disable triggers or rewrite immutable rows. Rollback +means draining the new ingress and returning to compatible older ordinary +readers/writers while retaining the additive objects and records. Do not drop +financial history as a rollback strategy. B2.4 must separately drain/fence +incompatible status projectors before its activation. + +Historical reader/writer gates execute actual installed beta.15, records A, +foundation integration, outbox A, activity B, status C, B1 and the exact approved +B2.1 artifact on populated new records, comparing permitted old behavior before +and after this migration. The integrated pins are `17ee407a` (A), `0f34c666` (B), +`967b6e28` (C), `5487f2bd` (B1) and `3fd62121` (B2.1). These controls do not qualify +the earlier pre-integration A/B/C/B1/B2.1 snapshots. In particular, the old A +fingerprints and B2.1 scheduler ordering depend on database collation. The +integrated artifacts remove those dependencies; a C-only cluster would hide +portability regressions, so the gates require URL and drivers without a locale +pin. The pre-`17ee407a` A, pre-`0f34c666` B, pre-`967b6e28` C, pre-`5487f2bd` B1 +and pre-`3fd62121` B2.1 rolling exclusions must accompany release notes. +A's notification-readiness closure after C is compared on both sides and does +not excuse new regressions. Optional notifications may stay explicitly disabled; +complete polling still depends on the later B2.3/B2.4 components. Full buyer +adjustment automation and `client.reporting` remain named downstream #1172 work. diff --git a/examples/reporting_receipt_ingress.py b/examples/reporting_receipt_ingress.py new file mode 100644 index 000000000..740640ca1 --- /dev/null +++ b/examples/reporting_receipt_ingress.py @@ -0,0 +1,66 @@ +"""Authenticated durable receipts on the existing materializer store and mounts. + +Install migrations during deployment, drain legacy materialization writers, and +supply the application's token validator and fresh account ACL. Receipt traffic +negotiates AdCP 3.2-rc.6. This composition does not activate production tier claims. +""" + +from __future__ import annotations + +from adcp.decisioning.registry import BuyerAgentRegistry +from adcp.reporting.receipts import ( + PgReportingReceiptStore, + ReceiptAccountResolver, + ReportingReceiptHandler, +) +from adcp.server import serve +from adcp.server.auth import BearerTokenAuth, auth_context_factory + + +async def compose_receipt_ingress( + store: PgReportingReceiptStore, + *, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, +) -> ReportingReceiptHandler: + """Validate the installed schema and bind the live authorization boundary. + + The resolver receives the exact account reference and canonical consumer on + every request, including replay. Resolve natural account keys and check the + current ACL there; return a storage account ID or raise + ReportingReceiptError("UNAUTHORIZED"). It must never derive a consumer from + tenancy, request fields, or RequestContext.caller_identity's cache key. + + This same store can be passed to compose_materializer in + reporting_durable_materializer.py. No parallel materializer or readiness + switch is needed. Schema installation is an explicit deployment operation. + """ + await store.receipt_ingestion_ready() + return ReportingReceiptHandler( + store, resolve_account=resolve_account, buyer_agents=buyer_agents + ) + + +def serve_receipt_ingress( + handler: ReportingReceiptHandler, + *, + auth: BearerTokenAuth, + public_url: str, + allowed_hosts: tuple[str, ...], +) -> None: + """Use the production MCP and A2A mounts with one authenticated handler. + + Token validation supplies trusted Principal identities, never credentials in + request bodies. Registry-backed deployments populate AuthInfo credentials + through their verified adapter. Generic SDK idempotency middleware is + automatically bypassed for this task; the store owns durable replay. + """ + serve( + handler, + name="reporting-receipts", + transport="both", + auth=auth, + context_factory=auth_context_factory, + public_url=public_url, + allowed_hosts=allowed_hosts, + ) diff --git a/pyproject.toml b/pyproject.toml index 45fc80191..1a41bceed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -197,6 +197,7 @@ adcp = [ "reporting/outbox/*.json", "reporting/materializer/assets/*.json", "reporting/materializer/*.json", + "reporting/receipts/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/scripts/reporting_test_harness.py b/scripts/reporting_test_harness.py new file mode 100644 index 000000000..4302bdf55 --- /dev/null +++ b/scripts/reporting_test_harness.py @@ -0,0 +1,64 @@ +"""Run a reporting pytest gate in an audited private temporary directory. + +Usage: python scripts/reporting_test_harness.py pytest tests/... -v -s -ra +The child inherits the normal process umask. Failed diagnostics remain under the +printed root until the runner/workspace is removed; no live artifact is deleted. +""" + +from __future__ import annotations + +import json +import os +import stat +import subprocess +import sys +import tempfile +import time +from pathlib import Path + + +def main() -> int: + if len(sys.argv) < 2: + raise SystemExit("supply the pytest executable and gate arguments") + root = Path(tempfile.mkdtemp(prefix="adcp-reporting-gate-")).resolve() + root.chmod(0o700) + parent = root / "suite" + parent.mkdir(mode=0o700) + ancestors = [] + for path in (parent, root, *root.parents): + metadata = path.stat() + mode = stat.S_IMODE(metadata.st_mode) + if mode & 0o022 and not mode & stat.S_ISVTX: + raise SystemExit(f"unsafe reporting harness ancestor: {path}") + ancestors.append({"path": str(path), "mode": oct(mode), "uid": metadata.st_uid}) + mask = os.umask(0) + os.umask(mask) + command = [*sys.argv[1:], "--basetemp", str(parent / "run")] + print( + json.dumps( + { + "reporting_harness": str(root), + "umask": oct(mask), + "ancestors": ancestors, + "command": command, + } + ), + flush=True, + ) + started = time.monotonic() + result = subprocess.run(command, check=False) + print( + json.dumps( + { + "reporting_harness": str(root), + "exit_status": result.returncode, + "runtime_seconds": round(time.monotonic() - started, 3), + } + ), + flush=True, + ) + return result.returncode + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/adcp/reporting/ledger/_delivery_state.py b/src/adcp/reporting/ledger/_delivery_state.py index e96564201..2cd6baef7 100644 --- a/src/adcp/reporting/ledger/_delivery_state.py +++ b/src/adcp/reporting/ledger/_delivery_state.py @@ -188,11 +188,44 @@ def current_receipt( for item in records if isinstance(item, _RECEIPTS) and receipt_chain(item) == receipt_chain(requested) ] - superseded = {item.supersedes_reporting_receipt_id for item in chain} + if not chain: + return None + by_id = {item.reporting_receipt_id: item for item in chain} + superseded = [ + item.supersedes_reporting_receipt_id + for item in chain + if item.supersedes_reporting_receipt_id is not None + ] leaves = [item for item in chain if item.reporting_receipt_id not in superseded] - if chain and len(leaves) != 1: + if len(by_id) != len(chain) or len(set(superseded)) != len(superseded) or len(leaves) != 1: + fail("REPORTING_HISTORY_CORRUPT") + # One apparent leaf alone does not rule out a disconnected cycle, a missing + # predecessor, an accepted predecessor, or a cross-target/generation edge. + visited: set[str] = set() + node: ReportingReceiptRecord | None = leaves[0] + while node is not None: + if ( + node.reporting_receipt_id in visited + or type(node) is not type(requested) + or node.scope != requested.scope + ): + fail("REPORTING_HISTORY_CORRUPT") + if isinstance(node, ReportingAdjustmentReceiptRecord) and isinstance( + requested, ReportingAdjustmentReceiptRecord + ): + if node.adjusts_reporting_revision_id != requested.adjusts_reporting_revision_id: + fail("REPORTING_HISTORY_CORRUPT") + visited.add(node.reporting_receipt_id) + predecessor_id = node.supersedes_reporting_receipt_id + if predecessor_id is None: + node = None + else: + node = by_id.get(predecessor_id) + if node is None or node.status != "rejected": + fail("REPORTING_HISTORY_CORRUPT") + if len(visited) != len(chain): fail("REPORTING_HISTORY_CORRUPT") - return leaves[0] if leaves else None + return leaves[0] def replay( @@ -203,6 +236,8 @@ def replay( ) if existing is None: return None + if isinstance(existing, _RECEIPTS): + current_receipt(records, existing) comparison = existing if ( isinstance(record, _RECEIPTS) @@ -553,15 +588,22 @@ def _verify_receipt( fail("REPORTING_TIME_INVALID") if record.status != "accepted": return - if record.observed_row_count != revision.row_count or sorted( - record.observed_control_totals, key=lambda item: item.name - ) != sorted(target.verification.control_totals, key=lambda item: item.name): + if ( + record.observed_row_count != revision.row_count + or record.observed_row_count != target.verification.row_count + or sorted(record.observed_control_totals, key=lambda item: item.name) + != sorted(target.verification.control_totals, key=lambda item: item.name) + ): fail("RECEIPT_TOTALS_MISMATCH") profile = record.verification_profile if ( ( record.observed_canonical_content_digest is not None - and record.observed_canonical_content_digest != revision.canonical_content_digest + and ( + record.observed_canonical_content_digest != revision.canonical_content_digest + or record.observed_canonical_content_digest + != target.verification.canonical_content_digest + ) ) or ( record.observed_manifest_sha256 is not None diff --git a/src/adcp/reporting/ledger/delivery.py b/src/adcp/reporting/ledger/delivery.py index d07bad3c4..0ebb981c9 100644 --- a/src/adcp/reporting/ledger/delivery.py +++ b/src/adcp/reporting/ledger/delivery.py @@ -17,6 +17,7 @@ adjustment_payload, adjustment_sha256, change_id, + current_receipt, decode_record, fail, iso, @@ -281,8 +282,7 @@ def current_receipts(self) -> tuple[ReportingReceiptRecord, ...]: for item in self.records if isinstance(item, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)) ) - replaced = {item.supersedes_reporting_receipt_id for item in receipts} - return tuple(item for item in receipts if item.reporting_receipt_id not in replaced) + return tuple(item for item in receipts if current_receipt(self.records, item) == item) @property def terminal_acceptances(self) -> tuple[ReportingReceiptKey, ...]: diff --git a/src/adcp/reporting/ledger/delivery_pg.py b/src/adcp/reporting/ledger/delivery_pg.py index 274a98826..449fe8f14 100644 --- a/src/adcp/reporting/ledger/delivery_pg.py +++ b/src/adcp/reporting/ledger/delivery_pg.py @@ -137,12 +137,7 @@ async def _commit_record_on( if existing is not None: return cast(RecordT, existing), False context = await self._delivery_context(connection, candidate) - if self._clock is not None: - now = self._clock() - else: - time_row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() - assert time_row is not None - now = time_row[0] + now = await self._delivery_time_on(connection, candidate) stored = validate_transition(candidate, records, context, now) await self._insert(connection, stored) await self._append_reconciliation_change(connection, stored) @@ -168,6 +163,14 @@ async def _commit_record_on( await self._dirty_status(connection, scope, reason, after=evidence) return cast(RecordT, stored), True + async def _delivery_time_on(self, connection: Any, record: ReportingDeliveryRecord) -> datetime: + """Connection-bound clock seam; old public clock overrides remain supported.""" + if self._clock is not None: + return self._clock() + time_row = await (await connection.execute("SELECT clock_timestamp()")).fetchone() + assert time_row is not None + return cast(datetime, time_row[0]) + async def _append_reconciliation_change( self, connection: Any, record: ReportingDeliveryRecord ) -> None: diff --git a/src/adcp/reporting/ledger/pg.py b/src/adcp/reporting/ledger/pg.py index df2d275ab..d09d9341a 100644 --- a/src/adcp/reporting/ledger/pg.py +++ b/src/adcp/reporting/ledger/pg.py @@ -1810,22 +1810,55 @@ async def lease_period_close( self, *, worker_id: str, now: datetime, lease_seconds: float ) -> LeasedConfiguration | None: expires = _utc(now) + timedelta(seconds=lease_seconds) - async with self._connection() as connection: + async with self._connection() as connection, connection.transaction(): + # The fairness rank is the primary ordering term. The WHERE clause + # has already excluded every live lease, so among the survivors the + # expiry carries no fairness information: ordering by it first + # starves a crashed generation forever, because a peer that is + # leased and released each turn is always NULL and NULL sorts + # first. Without the rank at all, every released generation ties + # and whichever tuple the scan yields first is re-leased forever. + # Expiry and the generation key only break exact ties, which keeps + # the order total and independent of physical layout. + # + # The rank is joined from a private table rather than held on + # `reporting_configurations`, so the enumerated `reporting_*` + # catalog that older binaries validate stays byte-identical. A + # generation with no row yet has never been leased, which is what + # COALESCE to 0 means, so a newly accepted generation is served + # before any that already took a turn. row = await ( await connection.execute( "UPDATE reporting_configurations SET lease_worker_id = %s," " lease_expires_at = %s" " WHERE (account_id, delivery_config_id, delivery_config_version) = (" - " SELECT account_id, delivery_config_id, delivery_config_version" - " FROM reporting_configurations" - " WHERE lease_expires_at IS NULL OR lease_expires_at <= %s" - " ORDER BY lease_expires_at NULLS FIRST" - " FOR UPDATE SKIP LOCKED" + " SELECT c.account_id, c.delivery_config_id, c.delivery_config_version" + " FROM reporting_configurations c" + " LEFT JOIN adcp_reporting_configuration_lease_turns t" + " ON (t.account_id, t.delivery_config_id, t.delivery_config_version)" + " = (c.account_id, c.delivery_config_id, c.delivery_config_version)" + " WHERE c.lease_expires_at IS NULL OR c.lease_expires_at <= %s" + " ORDER BY COALESCE(t.lease_turn, 0), c.lease_expires_at NULLS FIRST," + " c.account_id, c.delivery_config_id, c.delivery_config_version" + " FOR UPDATE OF c SKIP LOCKED" " LIMIT 1)" " RETURNING account_id, delivery_config_id, delivery_config_version", (worker_id, expires, _utc(now)), ) ).fetchone() + if row is not None: + # Same transaction as the acquisition, so the rank can never + # advance without the lease or the lease without the rank. + await connection.execute( + "INSERT INTO adcp_reporting_configuration_lease_turns" + " (account_id, delivery_config_id, delivery_config_version, lease_turn)" + " VALUES (%s, %s, %s," + " nextval('adcp_reporting_configuration_lease_turn_seq'))" + " ON CONFLICT (account_id, delivery_config_id, delivery_config_version)" + " DO UPDATE SET lease_turn =" + " nextval('adcp_reporting_configuration_lease_turn_seq')", + (row[0], row[1], row[2]), + ) if row is None: return None return LeasedConfiguration( diff --git a/src/adcp/reporting/ledger/reporting_ledger.sql b/src/adcp/reporting/ledger/reporting_ledger.sql index 5a5fa258c..595b5d065 100644 --- a/src/adcp/reporting/ledger/reporting_ledger.sql +++ b/src/adcp/reporting/ledger/reporting_ledger.sql @@ -58,6 +58,45 @@ CREATE INDEX IF NOT EXISTS reporting_configurations_account_idx CREATE INDEX IF NOT EXISTS reporting_configurations_lease_idx ON reporting_configurations (lease_expires_at NULLS FIRST); +-- Durable round-robin fairness for period-close leasing. +-- +-- `reporting_configurations_lease_idx` above cannot deliver the "least recently +-- worked" order its own comment promises. `release_period_close` clears +-- `lease_expires_at`, so every released generation ties at NULL and the winner +-- is whichever tuple the scan happens to yield first: a worker that releases at +-- the end of every turn re-leases the same generation forever and never closes +-- any other account's periods. Ordering by expiry *first* is a second +-- starvation path -- the acquisition filter has already dropped every live +-- lease, so a peer that is released each turn is permanently NULL and outranks +-- a generation whose worker crashed, which then stays expired forever. +-- +-- The fairness rank therefore lives in its own private SDK table, stamped on +-- acquisition (release only clears the lease, so there is nowhere else to +-- record it) and persisted so it survives a worker restart. +-- +-- The `adcp_` prefix is deliberate and load-bearing. `schema_objects()` +-- enumerates every table in the current schema whose name starts with +-- `reporting_`, and the A/B+C conformance suites assert that the installed +-- object set is *exactly* their manifests. A new `reporting_*` table, or a new +-- column on `reporting_configurations`, would therefore break exact object +-- identity for every older binary: adding it to the required manifest would +-- make old installations fail validation, and leaving it out would break the +-- exhaustive comparison. Outside that prefix the table is invisible to the +-- catalog contract, so old manifests, `reporting_configurations`'s shape and +-- its retained evidence all stay byte-identical. +CREATE SEQUENCE IF NOT EXISTS adcp_reporting_configuration_lease_turn_seq AS BIGINT; + +-- No foreign key: the reference is resolved by the lazy LEFT JOIN instead, so +-- nothing is added to `reporting_configurations` and a generation that is +-- deleted simply stops being joined. +CREATE TABLE IF NOT EXISTS adcp_reporting_configuration_lease_turns ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + lease_turn BIGINT NOT NULL, + PRIMARY KEY (account_id, delivery_config_id, delivery_config_version) +); + CREATE TABLE IF NOT EXISTS reporting_obligations ( reporting_obligation_id TEXT COLLATE "C" NOT NULL PRIMARY KEY, account_id TEXT COLLATE "C" NOT NULL, diff --git a/src/adcp/reporting/ledger/reporting_receipt_ingestion.sql b/src/adcp/reporting/ledger/reporting_receipt_ingestion.sql new file mode 100644 index 000000000..6ebcd1f02 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_receipt_ingestion.sql @@ -0,0 +1,405 @@ +-- B2.2 additive receipt ingress. No older manifest, decoder, or queue is replaced. +-- Execute in the caller's transaction; an interrupted migration leaves no partial feature. +DO $migration$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.receipt_ingestion.schema')); + + -- Whole receipt requests may have arbitrary Unicode context/ext keys. + -- JCS sorts UTF-16 code units, which differs from the older closed-record + -- encoder's C ordering for supplementary characters. Keep this additive; + -- replacing reporting_payload_sha256 would invalidate approved manifests. + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_utf16(value TEXT) + RETURNS BYTEA LANGUAGE plpgsql IMMUTABLE STRICT AS $function$ + DECLARE + result BYTEA := ''::bytea; + point INTEGER; + i INTEGER; + BEGIN + FOR i IN 1..char_length(value) LOOP + point := ascii(substr(value,i,1)); + IF point > 65535 THEN + point := point - 65536; + result := result || decode(lpad(to_hex(55296 + (point >> 10)),4,'0') + || lpad(to_hex(56320 + (point & 1023)),4,'0'),'hex'); + ELSE + result := result || decode(lpad(to_hex(point),4,'0'),'hex'); + END IF; + END LOOP; + RETURN result; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_canonical(document JSONB) + RETURNS TEXT LANGUAGE plpgsql IMMUTABLE STRICT AS $function$ + DECLARE + shape TEXT := jsonb_typeof(document); + parts TEXT; + BEGIN + IF shape = 'object' THEN + SELECT coalesce(string_agg(to_json(e.key)::text || ':' + || reporting_receipt_ingestion_canonical(e.value), ',' + ORDER BY reporting_receipt_ingestion_utf16(e.key)), '') INTO parts + FROM jsonb_each(document) e; + RETURN '{' || parts || '}'; + ELSIF shape = 'array' THEN + SELECT coalesce(string_agg(reporting_receipt_ingestion_canonical(e.value), ',' + ORDER BY e.ordinality), '') INTO parts + FROM jsonb_array_elements(document) WITH ORDINALITY e; + RETURN '[' || parts || ']'; + END IF; + -- The approved scalar rules already match this restricted JCS profile. + RETURN reporting_canonical_json(document); + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_sha256(document JSONB) + RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $function$ + SELECT encode(sha256(convert_to(reporting_receipt_ingestion_canonical(document), 'UTF8')), 'hex') + $function$; + + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_batches ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL CHECK (idempotency_key ~ '^[A-Za-z0-9_.:-]{16,255}$'), + canonical_request TEXT NOT NULL, + request_sha256 TEXT COLLATE "C" NOT NULL CHECK (request_sha256 ~ '^[0-9a-f]{64}$'), + expected_count INTEGER NOT NULL CHECK (expected_count BETWEEN 1 AND 100), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + final_response JSONB, + final_sha256 TEXT COLLATE "C", + finalized_at TIMESTAMPTZ, + admission_epoch BIGINT NOT NULL DEFAULT 0 CHECK (admission_epoch = 0), + PRIMARY KEY (account_id, consumer_id, idempotency_key), + CHECK (request_sha256 = encode(sha256(convert_to(canonical_request, 'UTF8')), 'hex')), + CHECK (request_sha256 = reporting_receipt_ingestion_sha256(canonical_request::jsonb)), + CHECK (jsonb_typeof(canonical_request::jsonb) = 'object'), + CHECK ((canonical_request::jsonb->>'idempotency_key') IS NOT DISTINCT FROM idempotency_key), + CHECK ((final_response IS NULL) = (final_sha256 IS NULL)), + CHECK ((final_response IS NULL) = (finalized_at IS NULL)), + CHECK (final_response IS NULL OR final_sha256 = reporting_receipt_ingestion_sha256(final_response)) + ); + + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_results ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + ordinal INTEGER NOT NULL CHECK (ordinal BETWEEN 0 AND 99), + receipt_kind TEXT NOT NULL CHECK (receipt_kind IN ('revision_receipt','adjustment_receipt')), + reporting_receipt_id TEXT COLLATE "C" NOT NULL CHECK (reporting_receipt_id ~ '^[A-Za-z0-9_.:-]{16,255}$'), + receipt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'receipt' CHECK (receipt_namespace = 'receipt'), + receipt_record_id TEXT COLLATE "C", + result JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + recorded_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + PRIMARY KEY (account_id, consumer_id, idempotency_key, ordinal), + UNIQUE (account_id, consumer_id, idempotency_key, reporting_receipt_id), + FOREIGN KEY (account_id, consumer_id, idempotency_key) REFERENCES reporting_receipt_ingestion_batches, + FOREIGN KEY (account_id, consumer_id, receipt_namespace, receipt_record_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(result)), + CHECK (result ? 'result' AND result->>'result' IS NOT NULL + AND result->>'result' IN ('recorded','unchanged','failed')), + CHECK ((result->>'result' = 'failed') = (receipt_record_id IS NULL)), + CHECK (receipt_record_id IS NULL OR receipt_record_id = reporting_receipt_id) + ); + + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + max_sequence BIGINT NOT NULL CHECK (max_sequence > 0), + PRIMARY KEY (account_id, consumer_id) + ); + CREATE INDEX IF NOT EXISTS reporting_receipt_ingestion_chain + ON reporting_reconciliation_records(account_id, consumer_id, receipt_chain_key) + WHERE namespace = 'receipt'; + CREATE TABLE IF NOT EXISTS reporting_receipt_ingestion_boundaries ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL CHECK (sequence > 0), + account_sequence BIGINT NOT NULL CHECK (account_sequence > 0), + reporting_receipt_id TEXT COLLATE "C" NOT NULL, + receipt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'receipt' CHECK (receipt_namespace = 'receipt'), + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, consumer_id, sequence), + UNIQUE (account_id, consumer_id, reporting_receipt_id), + UNIQUE (account_id, account_sequence), + FOREIGN KEY (account_id, consumer_id, receipt_namespace, reporting_receipt_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(input)), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((input->>'admission_epoch')::bigint IS NOT DISTINCT FROM 0), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'reporting_receipt_id') IS NOT DISTINCT FROM reporting_receipt_id), + CHECK ((input->>'sequence')::bigint IS NOT DISTINCT FROM sequence), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK (input - ARRAY['version','admission_epoch','account_id','consumer_id', + 'sequence','account_sequence','reporting_receipt_id','as_of','core','reconciliation'] = '{}'::jsonb) + ); + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'receipt ingestion evidence is immutable' USING ERRCODE = '23514'; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_batch_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + request JSONB; + revisions JSONB; + adjustments JSONB; + expected JSONB; + n INTEGER; + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'receipt batch is immutable' USING ERRCODE = '23514'; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF TG_OP = 'UPDATE' THEN + IF (to_jsonb(NEW) - ARRAY['final_response','final_sha256','finalized_at']) + IS DISTINCT FROM (to_jsonb(OLD) - ARRAY['final_response','final_sha256','finalized_at']) + OR OLD.final_response IS NOT NULL OR NEW.final_response IS NULL THEN + RAISE EXCEPTION 'receipt batch is immutable' USING ERRCODE = '23514'; + END IF; + SELECT count(*), jsonb_agg(r.result ORDER BY r.ordinal) INTO n, expected + FROM reporting_receipt_ingestion_results r + WHERE (r.account_id,r.consumer_id,r.idempotency_key) + = (NEW.account_id,NEW.consumer_id,NEW.idempotency_key); + IF n <> NEW.expected_count OR NEW.final_response->'results' IS DISTINCT FROM expected + OR NEW.final_response->>'status' IS DISTINCT FROM 'completed' THEN + RAISE EXCEPTION 'receipt batch results are incomplete' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END IF; + request := NEW.canonical_request::jsonb; + revisions := coalesce(request->'receipts','[]'::jsonb); + adjustments := coalesce(request->'adjustment_receipts','[]'::jsonb); + IF jsonb_typeof(revisions) <> 'array' OR jsonb_typeof(adjustments) <> 'array' + OR (request ? 'receipts' AND jsonb_array_length(revisions) = 0) + OR (request ? 'adjustment_receipts' AND jsonb_array_length(adjustments) = 0) + OR jsonb_array_length(revisions) + jsonb_array_length(adjustments) <> NEW.expected_count + OR EXISTS (SELECT 1 FROM jsonb_array_elements(revisions || adjustments) i WHERE i ? 'received_at') + OR (SELECT count(DISTINCT i->>'reporting_receipt_id') + FROM jsonb_array_elements(revisions || adjustments) i) <> NEW.expected_count + OR NEW.final_response IS NOT NULL THEN + RAISE EXCEPTION 'receipt batch shape is invalid' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_wire(document JSONB) + RETURNS JSONB LANGUAGE SQL IMMUTABLE AS $function$ + SELECT jsonb_strip_nulls(document - ARRAY['scope','kind','rejection_codes']) + || CASE WHEN document->>'kind' = 'revision_receipt' + THEN jsonb_build_object('reporting_obligation_id',document#>>'{scope,reporting_obligation_id}') + ELSE '{}'::jsonb END + || CASE WHEN jsonb_array_length(document->'rejection_codes') > 0 + THEN jsonb_build_object('rejection_codes',document->'rejection_codes') ELSE '{}'::jsonb END + || CASE WHEN jsonb_typeof(document->'observed_canonical_content_digest') = 'object' + THEN jsonb_build_object('observed_canonical_content_digest', + document->'observed_canonical_content_digest' || '{"algorithm":"sha256"}'::jsonb) + ELSE '{}'::jsonb END + $function$; + + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_result_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + batch reporting_receipt_ingestion_batches; + request JSONB; + item JSONB; + expected_kind TEXT; + wire_key TEXT; + evidence JSONB; + n INTEGER; + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + SELECT * INTO batch FROM reporting_receipt_ingestion_batches b + WHERE (b.account_id,b.consumer_id,b.idempotency_key) + = (NEW.account_id,NEW.consumer_id,NEW.idempotency_key) FOR UPDATE; + SELECT count(*) INTO n FROM reporting_receipt_ingestion_results r + WHERE (r.account_id,r.consumer_id,r.idempotency_key) + = (NEW.account_id,NEW.consumer_id,NEW.idempotency_key); + IF batch.idempotency_key IS NULL OR batch.final_response IS NOT NULL + OR NEW.ordinal <> n OR NEW.ordinal >= batch.expected_count THEN + RAISE EXCEPTION 'receipt ordinal is unavailable' USING ERRCODE = '23514'; + END IF; + request := batch.canonical_request::jsonb; + n := jsonb_array_length(coalesce(request->'receipts','[]'::jsonb)); + IF NEW.ordinal < n THEN + expected_kind := 'revision_receipt'; wire_key := 'receipt'; + item := request->'receipts'->NEW.ordinal; + ELSE + expected_kind := 'adjustment_receipt'; wire_key := 'adjustment_receipt'; + item := request->'adjustment_receipts'->(NEW.ordinal - n); + END IF; + IF NEW.receipt_kind <> expected_kind + OR NEW.reporting_receipt_id IS DISTINCT FROM item->>'reporting_receipt_id' THEN + RAISE EXCEPTION 'receipt ordinal identity differs' USING ERRCODE = '23514'; + END IF; + IF NEW.result->>'result' = 'failed' THEN + IF NEW.result - ARRAY['result','reporting_receipt_id','errors'] <> '{}'::jsonb + OR NEW.result->>'reporting_receipt_id' IS DISTINCT FROM NEW.reporting_receipt_id + OR jsonb_typeof(NEW.result->'errors') IS DISTINCT FROM 'array' + OR jsonb_array_length(NEW.result->'errors') NOT BETWEEN 1 AND 16 THEN + RAISE EXCEPTION 'receipt failure shape differs' USING ERRCODE = '23514'; + END IF; + ELSE + SELECT reporting_receipt_ingestion_wire(r.payload) INTO evidence + FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.record_id,r.record_kind) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.reporting_receipt_id,NEW.receipt_kind); + IF evidence IS NULL OR NEW.result->wire_key IS DISTINCT FROM evidence + OR NEW.result - ARRAY['result',wire_key] <> '{}'::jsonb THEN + RAISE EXCEPTION 'receipt result evidence differs' USING ERRCODE = '23514'; + END IF; + END IF; + RETURN NEW; + END + $function$; + + -- Additional write predicates leave all approved reconciliation predicates + -- unchanged. They apply to raw SQL too, and only to newly written receipts. + CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_graph() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + DECLARE + revision reporting_revisions; + obligation reporting_obligations; + adjustment reporting_adjustments; + outcome JSONB; + observed TIMESTAMPTZ; + received TIMESTAMPTZ; + check_state TEXT; + chain_count INTEGER; + leaf_count INTEGER; + predecessor reporting_reconciliation_records; + visited TEXT[] := ARRAY[]::TEXT[]; + BEGIN + IF NEW.record_kind NOT IN ('revision_receipt','adjustment_receipt') THEN RETURN NEW; END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + observed := (NEW.payload->>'observed_at')::timestamptz; + received := (NEW.payload->>'received_at')::timestamptz; + IF observed IS NULL OR received IS NULL OR observed > received OR received > clock_timestamp() + OR NOT EXISTS (SELECT 1 FROM reporting_reconciliation_records b + WHERE (b.account_id,b.consumer_id,b.delivery_config_id,b.delivery_config_version) + = (NEW.account_id,NEW.consumer_id,NEW.delivery_config_id,NEW.delivery_config_version) + AND b.record_kind='destination_binding' AND b.payload->>'reconciliation_mode'='consumer_receipt') THEN + RAISE EXCEPTION 'receipt target is unavailable' USING ERRCODE = '23514'; + END IF; + SELECT * INTO revision FROM reporting_revisions v + WHERE (v.account_id,v.reporting_obligation_id,v.reporting_revision_id) + = (NEW.account_id,NEW.reporting_obligation_id,NEW.reporting_revision_id); + SELECT * INTO obligation FROM reporting_obligations o + WHERE (o.account_id,o.reporting_obligation_id,o.delivery_config_id,o.delivery_config_version) + = (NEW.account_id,NEW.reporting_obligation_id,NEW.delivery_config_id,NEW.delivery_config_version); + IF revision.reporting_revision_id IS NULL OR obligation.reporting_obligation_id IS NULL THEN + RAISE EXCEPTION 'receipt target is unavailable' USING ERRCODE = '23514'; + END IF; + SELECT count(*) INTO chain_count FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.receipt_chain_key) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.receipt_chain_key); + IF chain_count > 0 THEN + SELECT count(*) INTO leaf_count FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.receipt_chain_key) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.receipt_chain_key) + AND NOT EXISTS (SELECT 1 FROM reporting_reconciliation_records s + WHERE (s.account_id,s.consumer_id,s.namespace,s.supersedes_receipt_id) + = (r.account_id,r.consumer_id,'receipt',r.record_id)); + SELECT r.* INTO predecessor FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.record_id) + = (NEW.account_id,NEW.consumer_id,'receipt',NEW.supersedes_receipt_id); + IF leaf_count <> 1 OR predecessor.record_id IS NULL THEN + RAISE EXCEPTION 'receipt replacement is unavailable' USING ERRCODE = '23514'; + END IF; + LOOP + IF predecessor.record_id IS NULL OR predecessor.record_id = ANY(visited) + OR predecessor.receipt_status <> 'rejected' + OR (predecessor.record_kind,predecessor.delivery_config_id, + predecessor.delivery_config_version,predecessor.reporting_obligation_id, + predecessor.reporting_revision_id,predecessor.receipt_chain_key) + IS DISTINCT FROM (NEW.record_kind,NEW.delivery_config_id, + NEW.delivery_config_version,NEW.reporting_obligation_id, + NEW.reporting_revision_id,NEW.receipt_chain_key) + OR predecessor.reporting_adjustment_id IS DISTINCT FROM NEW.reporting_adjustment_id THEN + RAISE EXCEPTION 'receipt history is inconsistent' USING ERRCODE = '23514'; + END IF; + visited := array_append(visited, predecessor.record_id); + EXIT WHEN predecessor.supersedes_receipt_id IS NULL; + SELECT r.* INTO predecessor FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.namespace,r.record_id) + = (NEW.account_id,NEW.consumer_id,'receipt',predecessor.supersedes_receipt_id); + END LOOP; + IF cardinality(visited) <> chain_count THEN + RAISE EXCEPTION 'receipt history is inconsistent' USING ERRCODE = '23514'; + END IF; + ELSIF NEW.supersedes_receipt_id IS NOT NULL THEN + RAISE EXCEPTION 'receipt replacement is unavailable' USING ERRCODE = '23514'; + END IF; + IF NEW.record_kind='adjustment_receipt' THEN + SELECT * INTO adjustment FROM reporting_adjustments a + WHERE (a.account_id,a.adjusts_reporting_revision_id,a.reporting_adjustment_id) + = (NEW.account_id,NEW.reporting_revision_id,NEW.reporting_adjustment_id); + IF adjustment.reporting_adjustment_id IS NULL OR revision.finality <> 'official' + OR revision.finalized_at IS NULL + OR NOT (obligation.period_end <= revision.finalized_at + AND revision.finalized_at <= revision.created_at + AND revision.finalized_at <= adjustment.correction_observed_at + AND adjustment.correction_observed_at <= adjustment.created_at + AND adjustment.accounting_period_start < adjustment.accounting_period_end + AND adjustment.created_at <= observed) THEN + RAISE EXCEPTION 'receipt adjustment order is invalid' USING ERRCODE = '23514'; + END IF; + ELSE + SELECT r.payload INTO outcome FROM reporting_reconciliation_records r + WHERE (r.account_id,r.consumer_id,r.delivery_config_id,r.delivery_config_version, + r.reporting_obligation_id,r.reporting_revision_id,r.reporting_materialization_id) + = (NEW.account_id,NEW.consumer_id,NEW.delivery_config_id,NEW.delivery_config_version, + NEW.reporting_obligation_id,NEW.reporting_revision_id,NEW.reporting_materialization_id) + AND r.record_kind='materialization' AND r.payload->>'status' IN ('available','delivered'); + IF outcome IS NULL OR observed < (outcome->>'completed_at')::timestamptz + OR NEW.payload->>'verification_profile' IS DISTINCT FROM outcome#>>'{verification,verification_profile}' THEN + RAISE EXCEPTION 'receipt artifact is unavailable' USING ERRCODE = '23514'; + END IF; + IF NEW.receipt_status='accepted' THEN + SELECT r.payload->>'state' INTO check_state FROM reporting_reconciliation_records r + WHERE r.account_id=NEW.account_id AND r.consumer_id=NEW.consumer_id + AND r.record_kind='materialization_check' + AND r.reporting_materialization_id=NEW.reporting_materialization_id + AND (r.payload->>'checked_at')::timestamptz <= observed + ORDER BY (r.payload->>'checked_at')::timestamptz DESC LIMIT 1; + IF (check_state IS NOT NULL AND check_state <> 'readable') + OR (NEW.payload->>'observed_row_count')::bigint + IS DISTINCT FROM (outcome#>>'{verification,row_count}')::bigint + OR (jsonb_typeof(NEW.payload->'observed_canonical_content_digest')='object' + AND reporting_wire_digest(NEW.payload->'observed_canonical_content_digest') + IS DISTINCT FROM reporting_wire_digest(outcome#>'{verification,canonical_content_digest}')) THEN + RAISE EXCEPTION 'receipt artifact evidence differs' USING ERRCODE = '23514'; + END IF; + END IF; + END IF; + RETURN NEW; + END + $function$; + + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_batch ON reporting_receipt_ingestion_batches; + CREATE TRIGGER reporting_receipt_ingestion_batch BEFORE INSERT OR UPDATE OR DELETE + ON reporting_receipt_ingestion_batches FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_batch_guard(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_result ON reporting_receipt_ingestion_results; + CREATE TRIGGER reporting_receipt_ingestion_result BEFORE INSERT ON reporting_receipt_ingestion_results + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_result_guard(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_result_immutable ON reporting_receipt_ingestion_results; + CREATE TRIGGER reporting_receipt_ingestion_result_immutable BEFORE UPDATE OR DELETE + ON reporting_receipt_ingestion_results FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_boundary_immutable ON reporting_receipt_ingestion_boundaries; + CREATE TRIGGER reporting_receipt_ingestion_boundary_immutable BEFORE UPDATE OR DELETE + ON reporting_receipt_ingestion_boundaries FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + DROP TRIGGER IF EXISTS reporting_receipt_ingestion_graph ON reporting_reconciliation_records; + CREATE TRIGGER reporting_receipt_ingestion_graph BEFORE INSERT ON reporting_reconciliation_records + FOR EACH ROW EXECUTE FUNCTION reporting_receipt_ingestion_graph(); +END +$migration$; diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 805f8c413..549553ca2 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1654,26 +1654,40 @@ async def lease_period_close( async with self._lock: moment = _utc(now) - # Rank leasable generations the way the SQL store's - # `ORDER BY lease_expires_at NULLS FIRST` does -- unheld before - # expired, oldest expiry first -- then break the tie by whichever - # generation went longest without a turn. Without that last term a - # worker that releases at the end of every turn re-leases the same - # generation forever, and every other account's periods are never - # closed: starvation that only appears once two accounts can hold - # the same delivery_config_id. - ranked: list[tuple[tuple[int, float, int], ReportingConfigurationGenerationKey]] = [] + # Rank leasable generations exactly the way the SQL store's + # `ORDER BY lease_turn, lease_expires_at NULLS FIRST, ...` does: + # whichever generation went longest without a turn goes first. + # + # The turn has to be the *primary* term. The caller's `now` has + # already excluded every live lease, so among the survivors the + # expiry carries no fairness information -- and preferring unheld + # over expired ahead of the turn starves a crashed generation + # forever: a peer that is leased and released every turn is always + # unheld, so it wins every comparison while the generation whose + # worker died stays expired and never closes another period. + # Expiry and the generation key only break exact turn ties, so the + # order stays total and never depends on physical layout. + ranked: list[ + tuple[ + tuple[int, int, float, str, str, int], + ReportingConfigurationGenerationKey, + ] + ] = [] for key in self._configurations: turn = self._lease_turns.get(key, 0) held = self._leases.get(key) + tail = (key.account_id, key.delivery_config_id, key.delivery_config_version) if held is None: - ranked.append(((0, 0.0, turn), key)) + ranked.append(((turn, 0, 0.0, *tail), key)) elif _utc(held[1]) <= moment: - ranked.append(((1, _utc(held[1]).timestamp(), turn), key)) + ranked.append(((turn, 1, _utc(held[1]).timestamp(), *tail), key)) if not ranked: return None - # `min` keeps the first of equal ranks, so generations that have - # never been leased are handed out in the order they were accepted. + # The generation key is part of the rank, so the order is total and + # both stores make the same choice. Ranking by `min` alone would + # fall back to whichever generation this process happened to accept + # first, which the SQL store cannot reproduce and no adopter can + # observe consistently across a restart or a second worker. key = min(ranked, key=lambda item: item[0])[1] configuration = self._configurations[key] expires = moment + timedelta(seconds=lease_seconds) diff --git a/src/adcp/reporting/receipts/__init__.py b/src/adcp/reporting/receipts/__init__.py new file mode 100644 index 000000000..551286cc7 --- /dev/null +++ b/src/adcp/reporting/receipts/__init__.py @@ -0,0 +1,32 @@ +"""Durable seller receipt ingress. Optional PG driver is imported lazily.""" + +from typing import TYPE_CHECKING, Any + +from adcp.reporting.receipts.capture import ReportingReceiptBoundary +from adcp.reporting.receipts.errors import ReceiptErrorCode, ReportingReceiptError +from adcp.reporting.receipts.handler import ReceiptAccountResolver, ReportingReceiptHandler +from adcp.reporting.receipts.memory import InMemoryReportingReceiptStore +from adcp.reporting.receipts.store import ReportingReceiptBatchStore, ReportingReceiptCaptureStore + +if TYPE_CHECKING: + from adcp.reporting.receipts.pg import PgReportingReceiptStore + +__all__ = [ + "InMemoryReportingReceiptStore", + "PgReportingReceiptStore", + "ReceiptAccountResolver", + "ReceiptErrorCode", + "ReportingReceiptBatchStore", + "ReportingReceiptBoundary", + "ReportingReceiptCaptureStore", + "ReportingReceiptError", + "ReportingReceiptHandler", +] + + +def __getattr__(name: str) -> Any: + if name == "PgReportingReceiptStore": + from adcp.reporting.receipts.pg import PgReportingReceiptStore + + return PgReportingReceiptStore + raise AttributeError(name) diff --git a/src/adcp/reporting/receipts/capture.py b/src/adcp/reporting/receipts/capture.py new file mode 100644 index 000000000..966e99069 --- /dev/null +++ b/src/adcp/reporting/receipts/capture.py @@ -0,0 +1,94 @@ +"""Immutable receipt dirty inputs; projection activation belongs to B2.4.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from pydantic import TypeAdapter, ValidationError + +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger._delivery_state import decode_record, payload, principal +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.receipts.errors import ReportingReceiptError + +_CORE = TypeAdapter(ReportingStatusSnapshot) + + +@dataclass(frozen=True) +class ReportingReceiptBoundary: + caller: ReportingDeliveryPrincipal + sequence: int + account_sequence: int + reporting_receipt_id: str + as_of: datetime + core: ReportingStatusSnapshot = field(repr=False) + reconciliation: tuple[ReportingDeliveryRecord, ...] = field(repr=False) + + def __post_init__(self) -> None: + receipts = tuple( + r + for r in self.reconciliation + if isinstance(r, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)) + and r.reporting_receipt_id == self.reporting_receipt_id + ) + if ( + type(self.sequence) is not int + or self.sequence < 1 + or type(self.account_sequence) is not int + or self.account_sequence < self.sequence + or self.core.account_id != self.caller.account_id + or self.core.as_of != self.as_of + or self.core.consumer_ids != (self.caller.consumer_id,) + or any(s.consumer_id != self.caller.consumer_id for s in self.core.statuses) + or any( + i.consumer_id not in {None, self.caller.consumer_id} for i in self.core.lifecycles + ) + or any(principal(r) != self.caller for r in self.reconciliation) + or len(receipts) != 1 + or receipts[0].received_at != self.as_of + ): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + object.__setattr__(self, "as_of", aware_utc(self.as_of)) + + def to_storage(self) -> dict[str, Any]: + return { + "version": 1, + "admission_epoch": 0, + "account_id": self.caller.account_id, + "consumer_id": self.caller.consumer_id, + "sequence": self.sequence, + "account_sequence": self.account_sequence, + "reporting_receipt_id": self.reporting_receipt_id, + "as_of": self.as_of.isoformat(), + "core": _CORE.dump_python(self.core, mode="json"), + "reconciliation": [payload(r) for r in self.reconciliation], + } + + +def decode_receipt_boundary(value: dict[str, Any]) -> ReportingReceiptBoundary: + result = None + try: + if type(value) is dict and type(value.get("version")) is int and value["version"] == 1: + result = ReportingReceiptBoundary( + ReportingDeliveryPrincipal(value["account_id"], value["consumer_id"]), + value["sequence"], + value["account_sequence"], + value["reporting_receipt_id"], + datetime.fromisoformat(value["as_of"]), + _CORE.validate_python(value["core"]), + tuple(decode_record(r) for r in value["reconciliation"]), + ) + except (ValueError, TypeError, KeyError, ValidationError): + # Raise the closed history error below without exposing decoder exception context. + pass + if result is None or result.to_storage() != value: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return result diff --git a/src/adcp/reporting/receipts/errors.py b/src/adcp/reporting/receipts/errors.py new file mode 100644 index 000000000..338dea1e6 --- /dev/null +++ b/src/adcp/reporting/receipts/errors.py @@ -0,0 +1,31 @@ +"""Closed, payload-free receipt ingress failures.""" + +from __future__ import annotations + +from typing import Literal + +ReceiptErrorCode = Literal[ + "INVALID_REQUEST", + "UNAUTHORIZED", + "IDEMPOTENCY_CONFLICT", + "RECEIPT_SCHEMA_UNREADY", + "RECEIPT_HISTORY_CORRUPT", + "RECEIPT_STORAGE_UNAVAILABLE", +] + +_MESSAGES: dict[ReceiptErrorCode, str] = { + "INVALID_REQUEST": "supply a valid receipt batch with 1..100 distinct receipt IDs", + "UNAUTHORIZED": "the reporting account or authenticated consumer is unavailable", + "IDEMPOTENCY_CONFLICT": "reuse the original receipt batch body or choose a new batch key", + "RECEIPT_SCHEMA_UNREADY": "install and verify the isolated receipt ingestion schema", + "RECEIPT_HISTORY_CORRUPT": "retained receipt batch evidence requires operator repair", + "RECEIPT_STORAGE_UNAVAILABLE": "receipt storage is unavailable; retry the same batch and key", +} + + +class ReportingReceiptError(RuntimeError): + """An actionable classification; never includes request, driver or auth details.""" + + def __init__(self, code: ReceiptErrorCode) -> None: + self.code = code + super().__init__(_MESSAGES[code]) diff --git a/src/adcp/reporting/receipts/handler.py b/src/adcp/reporting/receipts/handler.py new file mode 100644 index 000000000..70239e119 --- /dev/null +++ b/src/adcp/reporting/receipts/handler.py @@ -0,0 +1,153 @@ +"""One authenticated composition path for direct MCP/A2A and hydrated contexts.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from typing import Any + +from adcp.decisioning.context import AuthInfo, RequestContext +from adcp.decisioning.registry import BuyerAgent, BuyerAgentRegistry, HttpSigCredential +from adcp.exceptions import ADCPTaskError +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox.identity import canonical_consumer, resolve_reporting_consumer +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.store import ReportingReceiptBatchStore +from adcp.reporting.receipts.wire import TASK, validate_receipt_request +from adcp.server.base import ADCPHandler, ToolContext +from adcp.types import Error, SyncReportingReceiptsRequest + +ReceiptAccountResolver = Callable[[dict[str, Any], ToolContext, str], Awaitable[str]] +"""Resolve AND reauthorize the exact account reference for this consumer on every call. + +Return the canonical storage account ID, never a RequestContext cache key. A +natural-key account reference is resolved by this same application ACL boundary. +Raises ReportingReceiptError('UNAUTHORIZED') for unknown or denied accounts. +""" + + +async def _consumer(context: ToolContext, registry: BuyerAgentRegistry | None) -> str: + identities: set[str] = set() + auths: list[AuthInfo] = [] + agents: list[BuyerAgent] = [] + auth_value = getattr(context, "auth_info", None) + if auth_value is not None: + if not isinstance(auth_value, AuthInfo): + raise ReportingReceiptError("UNAUTHORIZED") + auths.append(auth_value) + agent_value = getattr(context, "buyer_agent", None) + if agent_value is not None: + if not isinstance(agent_value, BuyerAgent): + raise ReportingReceiptError("UNAUTHORIZED") + agents.append(agent_value) + auth_principal = getattr(context, "auth_principal", None) + if auth_principal is not None: + identities.add(canonical_consumer(auth_principal)) + # caller_identity is deliberately never inspected on RequestContext. + if not isinstance(context, RequestContext) and context.caller_identity is not None: + identities.add(canonical_consumer(context.caller_identity)) + for key in ("adcp.auth_info", "auth_info"): + auth = context.metadata.get(key) + if auth is not None: + if not isinstance(auth, AuthInfo): + raise ReportingReceiptError("UNAUTHORIZED") + auths.append(auth) + agent = context.metadata.get("adcp.buyer_agent") + if agent is not None: + if not isinstance(agent, BuyerAgent): + raise ReportingReceiptError("UNAUTHORIZED") + agents.append(agent) + if registry is not None: + if not auths: + raise ReportingReceiptError("UNAUTHORIZED") + for auth in auths: + credential = auth.credential + if credential is None: + raise ReportingReceiptError("UNAUTHORIZED") + resolved = ( + await registry.resolve_by_agent_url(credential.agent_url) + if isinstance(credential, HttpSigCredential) + else await registry.resolve_by_credential(credential) + ) + if resolved is None: + raise ReportingReceiptError("UNAUTHORIZED") + agents.append(resolved) + for agent in agents: + if agent.status != "active": + raise ReportingReceiptError("UNAUTHORIZED") + identities.add(resolve_reporting_consumer(auth_info=None, agent=agent)) + for auth in auths: + # A credential-only API/OAuth AuthInfo is allowed when a registry or + # trusted hydrated agent supplied the consumer. It is never an alias. + if ( + auth.principal is not None + or auth.agent_url is not None + or isinstance(auth.credential, HttpSigCredential) + ): + identities.add(resolve_reporting_consumer(auth_info=auth)) + if len(identities) != 1: + raise ReportingReceiptError("UNAUTHORIZED") + return identities.pop() + + +class ReportingReceiptHandler(ADCPHandler[ToolContext]): + """Mount only the durable receipt task; tier activation remains separately gated. + + ``resolve_account`` is an application ACL, called even for a completed + batch. ``buyer_agents`` optionally re-resolves API/OAuth/signed commercial + identity on every call. Neither transport tenancy nor body fields supply + the consumer. Authentication middleware must populate trusted context. + """ + + advertised_tools = {TASK} + + def __init__( + self, + store: ReportingReceiptBatchStore, + *, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + ) -> None: + super().__init__() + if not isinstance(store, ReportingReceiptBatchStore): + raise TypeError("receipt ingress requires an atomic ReportingReceiptBatchStore") + self.receipt_store = store + self._receipt_account_resolver = resolve_account + self._receipt_registry = buyer_agents + + async def sync_reporting_receipts( + self, + params: SyncReportingReceiptsRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + request = ( + params + if isinstance(params, dict) + else params.model_dump(mode="json", exclude_unset=True) + ) + try: + validate_receipt_request(request) + if context is None: + raise ReportingReceiptError("UNAUTHORIZED") + try: + consumer = await _consumer(context, self._receipt_registry) + except ReportingNotificationError: + raise ReportingReceiptError("UNAUTHORIZED") from None + account = await self._receipt_account_resolver( + dict(request["account"]), context, consumer + ) + if isinstance(context, RequestContext) and context.account.id != account: + raise ReportingReceiptError("UNAUTHORIZED") + try: + caller = ReportingDeliveryPrincipal(account, consumer) + except (ValueError, TypeError): + raise ReportingReceiptError("UNAUTHORIZED") from None + return await self.receipt_store.ingest_receipt_batch(request, caller=caller) + except ReportingReceiptError as error: + code, message = error.code, str(error) + except Exception: + unavailable = ReportingReceiptError("RECEIPT_STORAGE_UNAVAILABLE") + code, message = unavailable.code, str(unavailable) + # Leave the exception scope before translating. Credential/ACL adapters + # may raise provider errors; neither logs nor exception chains retain them. + raise ADCPTaskError(operation=TASK, errors=[Error(code=code, message=message)]) diff --git a/src/adcp/reporting/receipts/memory.py b/src/adcp/reporting/receipts/memory.py new file mode 100644 index 000000000..577966a7d --- /dev/null +++ b/src/adcp/reporting/receipts/memory.py @@ -0,0 +1,203 @@ +"""Account-serialized receipt batches on the existing unconditional rollback model.""" + +from __future__ import annotations + +import json +from copy import deepcopy +from dataclasses import dataclass, replace +from datetime import datetime +from typing import Any + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.status_snapshot import settle_memory_snapshot +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import private_snapshot +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore +from adcp.reporting.receipts.capture import ReportingReceiptBoundary +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.records import ( + failed_result, + prepare_receipt, + receipt_record, + success_result, +) +from adcp.reporting.receipts.wire import ( + ReceiptBatch, + validate_receipt_response, + validate_receipt_results, +) +from adcp.server.helpers import inject_context + + +@dataclass +class _BatchState: + batch: ReceiptBatch + expected_count: int + created_at: datetime + results: tuple[dict[str, Any], ...] = () + response: bytes | None = None + + +class InMemoryReportingReceiptStore(InMemoryReportingMaterializerStore): + """Reference participant. No production tier or notification readiness claim.""" + + # Lazily initialized deliberately: first-use allocations must roll back too. + _receipt_batches: dict[tuple[str, str, str], _BatchState] + _receipt_boundaries: list[ReportingReceiptBoundary] + _receipt_status_heads: dict[ReportingDeliveryPrincipal, int] + + def _receipt_batch( + self, caller: ReportingDeliveryPrincipal, batch: ReceiptBatch + ) -> _BatchState: + if not hasattr(self, "_receipt_batches"): + self._receipt_batches = {} + key = caller.account_id, caller.consumer_id, batch.key + state = self._receipt_batches.get(key) + if state is None: + state = _BatchState(batch, len(batch.items), self._clock()) + self._receipt_batches[key] = state + elif state.batch.canonical_request != batch.canonical_request: + raise ReportingReceiptError("IDEMPOTENCY_CONFLICT") + if state.expected_count != len(batch.items) or len(state.results) > state.expected_count: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + validate_receipt_results(list(state.results), batch) + return state + + def _append_receipt_result(self, state: _BatchState, result: dict[str, Any]) -> None: + state.results = (*state.results, deepcopy(result)) + + def _save_receipt_response(self, state: _BatchState) -> None: + response = inject_context( + state.batch.request, {"status": "completed", "results": deepcopy(list(state.results))} + ) + validate_receipt_response(response, state.batch) + state.response = canonical_json_utf8_v1(response) + + def _assemble_receipt_response(self, state: _BatchState) -> dict[str, Any]: + if state.response is None: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + response: dict[str, Any] = json.loads(state.response) + validate_receipt_response(response, state.batch) + if response["results"] != list(state.results): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return response + + async def ingest_receipt_batch( + self, request: dict[str, Any], *, caller: ReportingDeliveryPrincipal + ) -> dict[str, Any]: + batch = ReceiptBatch.parse(request) + while True: + # Each turn chooses the next *durable* ordinal under the account + # lock. Concurrent/resumed callers cooperate; no process-local batch + # lock, pending task, or TTL is a correctness dependency. + async with self._mutation(): + state = self._receipt_batch(caller, batch) + if state.response is not None: + return self._assemble_receipt_response(state) + if len(state.results) == state.expected_count: + self._save_receipt_response(state) + return self._assemble_receipt_response(state) + kind, item = batch.items[len(state.results)] + candidate = None + try: + revision_id = item.get( + "reporting_revision_id", item.get("adjusts_reporting_revision_id") + ) + revision = ( + self._revisions.get(revision_id) if isinstance(revision_id, str) else None + ) + obligation_id = item.get("reporting_obligation_id") + if ( + kind == "adjustment_receipt" + and revision is not None + and revision.account_id == caller.account_id + ): + obligation_id = revision.reporting_obligation_id + obligation = ( + self._obligations.get(obligation_id) + if isinstance(obligation_id, str) + else None + ) + candidate = receipt_record(kind, item, caller, obligation) + records = tuple(c.record for c in self._caller_changes(caller)) + stored, added = prepare_receipt( + candidate, + records, + self._delivery_context(candidate), + tuple( + r + for r in self._revisions.values() + if obligation is not None + and r.account_id == caller.account_id + and r.reporting_obligation_id == obligation.reporting_obligation_id + ), + self._clock(), + ) + result = success_result(stored, added) + except LedgerConflictError as error: + result = failed_result(item, error) + added = False + # Nothing in the semantic-error catch above mutates domain state. + # Any insertion/capture/result failure below escapes and rolls + # this entire ordinal back, with notifications both off and on. + if added: + assert candidate is not None + stored, inserted = self._commit_record_unlocked(candidate) + assert inserted + result = success_result(stored, True) + self._append_receipt_result(state, result) + + def _commit_record_unlocked( + self, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + stored, added = super()._commit_record_unlocked(record, notify=notify, dirty=dirty) + if added and isinstance( + stored, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + ): + self._capture_receipt(stored) + return stored, added + + def _capture_receipt(self, receipt: ReportingReceiptRecord) -> None: + assert receipt.received_at is not None + caller = receipt.scope.principal + if not hasattr(self, "_receipt_status_heads"): + self._receipt_status_heads = {} + if not hasattr(self, "_receipt_boundaries"): + self._receipt_boundaries = [] + sequence = self._receipt_status_heads.get(caller, 0) + 1 + self._receipt_status_heads[caller] = sequence + # Preserve cross-kind account order using B2.1's original capture clock. + # No materializer work, retry decision, or readiness event is created. + account_sequence = self._materializer_account_heads.get(caller.account_id, 0) + 1 + self._materializer_account_heads[caller.account_id] = account_sequence + core = replace(settle_memory_snapshot(self, caller.account_id), as_of=receipt.received_at) + self._receipt_boundaries.append( + ReportingReceiptBoundary( + caller, + sequence, + account_sequence, + receipt.reporting_receipt_id, + receipt.received_at, + private_snapshot(core, caller), + tuple(c.record for c in self._caller_changes(caller)), + ) + ) + + async def read_receipt_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingReceiptBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ReportingReceiptError("INVALID_REQUEST") + async with self._lock: + return tuple( + b + for b in getattr(self, "_receipt_boundaries", ()) + if b.caller == caller and b.sequence > after + )[:limit] diff --git a/src/adcp/reporting/receipts/pg.py b/src/adcp/reporting/receipts/pg.py new file mode 100644 index 000000000..673deec8a --- /dev/null +++ b/src/adcp/reporting/receipts/pg.py @@ -0,0 +1,386 @@ +"""Durable mixed-batch ordinals on the materializer's account-locked connection.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Coroutine +from datetime import datetime +from functools import wraps +from importlib.resources import files +from typing import Any, ParamSpec, TypeVar, cast + +from adcp.reporting.canonical_json import canonical_json_sha256_v1 +from adcp.reporting.ledger._delivery_state import RecordT +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.pg import ( + _OBLIGATION_COLUMNS, + _REVISION_COLUMNS, + _json, + _obligation_from_row, + _revision_from_row, +) +from adcp.reporting.ledger.status_snapshot import settle_snapshot_on +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import private_snapshot +from adcp.reporting.materializer.pg import PgReportingMaterializerStore, _now +from adcp.reporting.receipts.capture import ReportingReceiptBoundary, decode_receipt_boundary +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.records import ( + failed_result, + prepare_receipt, + receipt_record, + success_result, +) +from adcp.reporting.receipts.schema import validate_receipt_schema +from adcp.reporting.receipts.wire import ( + ReceiptBatch, + ReceiptKind, + validate_receipt_response, + validate_receipt_results, +) +from adcp.server.helpers import inject_context + +_P = ParamSpec("_P") +_R = TypeVar("_R") + + +def _storage_errors( + fn: Callable[_P, Coroutine[Any, Any, _R]], +) -> Callable[_P, Coroutine[Any, Any, _R]]: + @wraps(fn) + async def wrapped(*args: _P.args, **kwargs: _P.kwargs) -> _R: + try: + return await fn(*args, **kwargs) + except ReportingReceiptError: + raise + except Exception: + unavailable = ReportingReceiptError("RECEIPT_STORAGE_UNAVAILABLE") + # Outside the driver exception scope: no SQL/provider detail in __context__. + raise unavailable + + return wrapped + + +class PgReportingReceiptStore(PgReportingMaterializerStore): + """One composition: old public stores plus optional ingestion and capture. + + No network I/O, session lock, or second pool participates in an ordinal. + All actual receipt timestamps come from PostgreSQL, including when an old + public conformance clock was supplied to the inherited constructor. + """ + + @_storage_errors + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + root = files("adcp.reporting.ledger") + await connection.execute(root.joinpath("reporting_materializer.sql").read_text()) + await connection.execute(root.joinpath("reporting_receipt_ingestion.sql").read_text()) + + @_storage_errors + async def receipt_ingestion_ready(self) -> bool: + async with self._connection() as connection: + await validate_receipt_schema(connection, notifications=self._notifications_enabled) + return True + + async def _delivery_time_on(self, connection: Any, record: ReportingDeliveryRecord) -> datetime: + if isinstance(record, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord)): + return await _now(connection) + return await super()._delivery_time_on(connection, record) + + async def _commit_record_on( + self, connection: Any, record: RecordT, *, notify: bool = True, dirty: bool = True + ) -> tuple[RecordT, bool]: + stored, added = await super()._commit_record_on( + connection, record, notify=notify, dirty=dirty + ) + if added and isinstance( + stored, (ReportingRevisionReceiptRecord, ReportingAdjustmentReceiptRecord) + ): + await self._capture_receipt_on(connection, stored) + return stored, added + + async def _receipt_batch_on( + self, connection: Any, caller: ReportingDeliveryPrincipal, batch: ReceiptBatch + ) -> tuple[Any, ...]: + key = caller.account_id, caller.consumer_id, batch.key + row = await ( + await connection.execute( + "SELECT canonical_request,request_sha256,expected_count,final_response,final_sha256" + " FROM reporting_receipt_ingestion_batches" + " WHERE account_id=%s AND consumer_id=%s AND idempotency_key=%s FOR UPDATE", + key, + ) + ).fetchone() + if row is None: + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_batches" + " (account_id,consumer_id,idempotency_key,canonical_request," + " request_sha256,expected_count)" + " VALUES (%s,%s,%s,%s,%s,%s)", + (*key, batch.canonical_request.decode(), batch.digest, len(batch.items)), + ) + return batch.canonical_request.decode(), batch.digest, len(batch.items), None, None + if row[0] != batch.canonical_request.decode() or row[1] != batch.digest: + raise ReportingReceiptError("IDEMPOTENCY_CONFLICT") + if row[2] != len(batch.items): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return tuple(row) + + async def _receipt_results_on( + self, connection: Any, caller: ReportingDeliveryPrincipal, batch: ReceiptBatch + ) -> list[dict[str, Any]]: + rows = await ( + await connection.execute( + "SELECT ordinal,receipt_kind,reporting_receipt_id,result,content_sha256" + " FROM reporting_receipt_ingestion_results" + " WHERE account_id=%s AND consumer_id=%s AND idempotency_key=%s ORDER BY ordinal", + (caller.account_id, caller.consumer_id, batch.key), + ) + ).fetchall() + items = batch.items + if len(rows) > len(items) or any( + r[0] != i + or r[1] != items[i][0] + or r[2] != items[i][1]["reporting_receipt_id"] + or canonical_json_sha256_v1(r[3]) != r[4] + for i, r in enumerate(rows) + ): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + results = [r[3] for r in rows] + validate_receipt_results(results, batch) + return results + + async def _prepare_receipt_on( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + kind: ReceiptKind, + item: dict[str, Any], + ) -> tuple[ReportingReceiptRecord, dict[str, Any], bool]: + revision_id = item.get("reporting_revision_id", item.get("adjusts_reporting_revision_id")) + revision = await ( + await connection.execute( + # SDK-owned column list; every request value is bound below. + f"SELECT {_REVISION_COLUMNS} FROM reporting_revisions" # nosec B608 + " WHERE account_id=%s AND reporting_revision_id=%s", + (caller.account_id, revision_id), + ) + ).fetchone() + obligation_id = item.get("reporting_obligation_id") + if kind == "adjustment_receipt" and revision is not None: + obligation_id = _revision_from_row(revision).reporting_obligation_id + row = await ( + await connection.execute( + # SDK-owned column list; every request value is bound below. + f"SELECT {_OBLIGATION_COLUMNS} FROM reporting_obligations" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (caller.account_id, obligation_id), + ) + ).fetchone() + obligation = _obligation_from_row(row) if row else None + candidate = receipt_record(kind, item, caller, obligation) + records = await self._records(connection, caller) + revisions = await ( + await connection.execute( + # SDK-owned column list; every request value is bound below. + f"SELECT {_REVISION_COLUMNS} FROM reporting_revisions" # nosec B608 + " WHERE account_id=%s AND reporting_obligation_id=%s", + (caller.account_id, obligation_id), + ) + ).fetchall() + stored, added = prepare_receipt( + candidate, + records, + await self._delivery_context(connection, candidate), + tuple(_revision_from_row(r) for r in revisions), + await _now(connection), + ) + return candidate, success_result(stored, added), added + + async def _insert_receipt_result_on( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + batch: ReceiptBatch, + ordinal: int, + result: dict[str, Any], + ) -> None: + kind, item = batch.items[ordinal] + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_results" + " (account_id,consumer_id,idempotency_key,ordinal,receipt_kind,reporting_receipt_id," + " receipt_record_id,result,content_sha256) VALUES (%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s)", + ( + caller.account_id, + caller.consumer_id, + batch.key, + ordinal, + kind, + item["reporting_receipt_id"], + item["reporting_receipt_id"] if result["result"] != "failed" else None, + _json(result), + canonical_json_sha256_v1(result), + ), + ) + + async def _save_receipt_response_on( + self, + connection: Any, + caller: ReportingDeliveryPrincipal, + batch: ReceiptBatch, + results: list[dict[str, Any]], + ) -> dict[str, Any]: + response = inject_context(batch.request, {"status": "completed", "results": results}) + validate_receipt_response(response, batch) + await connection.execute( + "UPDATE reporting_receipt_ingestion_batches" + " SET final_response=%s::jsonb,final_sha256=%s,finalized_at=clock_timestamp()" + " WHERE account_id=%s AND consumer_id=%s AND idempotency_key=%s", + ( + _json(response), + canonical_json_sha256_v1(response), + caller.account_id, + caller.consumer_id, + batch.key, + ), + ) + return response + + def _assemble_receipt_response( + self, + response: dict[str, Any], + digest: str, + results: list[dict[str, Any]], + batch: ReceiptBatch, + ) -> dict[str, Any]: + validate_receipt_response(response, batch) + if canonical_json_sha256_v1(response) != digest or response["results"] != results: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return cast(dict[str, Any], json.loads(_json(response))) + + @_storage_errors + async def ingest_receipt_batch( + self, request: dict[str, Any], *, caller: ReportingDeliveryPrincipal + ) -> dict[str, Any]: + batch = ReceiptBatch.parse(request) + checked = False + while True: + async with self._connection() as connection, connection.transaction(): + # Account advisory lock precedes the batch row; no lock pool or + # second connection can deadlock a size-one application pool. + await self._lock_account(connection, caller.account_id) + if not checked: + await validate_receipt_schema( + connection, notifications=self._notifications_enabled + ) + checked = True + row = await self._receipt_batch_on(connection, caller, batch) + results = await self._receipt_results_on(connection, caller, batch) + if row[3] is not None: + return self._assemble_receipt_response(row[3], row[4], results, batch) + if len(results) == len(batch.items): + response = await self._save_receipt_response_on( + connection, caller, batch, results + ) + return self._assemble_receipt_response( + response, canonical_json_sha256_v1(response), results, batch + ) + kind, item = batch.items[len(results)] + candidate = None + try: + candidate, result, added = await self._prepare_receipt_on( + connection, caller, kind, item + ) + except LedgerConflictError as error: + result, added = failed_result(item, error), False + # Only pure semantic preparation is caught. SQL/capture/result + # failures roll back this ordinal instead of recording a failure + # beside a partially inserted receipt. + if added: + assert candidate is not None + stored, inserted = await self._commit_record_on(connection, candidate) + assert inserted + result = success_result(stored, True) + await self._insert_receipt_result_on( + connection, caller, batch, len(results), result + ) + + async def _capture_receipt_on(self, connection: Any, receipt: ReportingReceiptRecord) -> None: + assert receipt.received_at is not None + caller = receipt.scope.principal + core = await settle_snapshot_on( + self, connection, account_id=caller.account_id, as_of=receipt.received_at + ) + row = await ( + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_heads" + " (account_id,consumer_id,max_sequence)" + " VALUES (%s,%s,1) ON CONFLICT (account_id,consumer_id) DO UPDATE" + " SET max_sequence=reporting_receipt_ingestion_heads.max_sequence+1" + " RETURNING max_sequence", + (caller.account_id, caller.consumer_id), + ) + ).fetchone() + account = await ( + await connection.execute( + "UPDATE reporting_materializer_accounts SET captured_sequence=captured_sequence+1" + " WHERE account_id=%s RETURNING captured_sequence", + (caller.account_id,), + ) + ).fetchone() + if row is None or account is None: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + value = ReportingReceiptBoundary( + caller, + row[0], + account[0], + receipt.reporting_receipt_id, + receipt.received_at, + private_snapshot(core, caller), + await self._records(connection, caller), + ).to_storage() + await connection.execute( + "INSERT INTO reporting_receipt_ingestion_boundaries" + " (account_id,consumer_id,sequence,account_sequence,reporting_receipt_id," + " as_of,input,content_sha256)" + " VALUES (%s,%s,%s,%s,%s,%s,%s::jsonb,%s)", + ( + caller.account_id, + caller.consumer_id, + row[0], + account[0], + receipt.reporting_receipt_id, + receipt.received_at, + _json(value), + canonical_json_sha256_v1(value), + ), + ) + + @_storage_errors + async def read_receipt_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingReceiptBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ReportingReceiptError("INVALID_REQUEST") + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + await validate_receipt_schema(connection, notifications=self._notifications_enabled) + rows = await ( + await connection.execute( + "SELECT input,content_sha256=reporting_receipt_ingestion_sha256(input)" + " FROM reporting_receipt_ingestion_boundaries" + " WHERE account_id=%s AND consumer_id=%s AND sequence>%s" + " ORDER BY sequence LIMIT %s", + (caller.account_id, caller.consumer_id, after, limit), + ) + ).fetchall() + if any(not r[1] for r in rows): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + return tuple(decode_receipt_boundary(r[0]) for r in rows) diff --git a/src/adcp/reporting/receipts/records.py b/src/adcp/reporting/receipts/records.py new file mode 100644 index 000000000..6c22b6787 --- /dev/null +++ b/src/adcp/reporting/receipts/records.py @@ -0,0 +1,106 @@ +"""Pure preparation against exact, account-locked retained evidence.""" + +from __future__ import annotations + +from datetime import datetime +from typing import Any, cast + +from pydantic import TypeAdapter, ValidationError + +from adcp.reporting.evidence import ReportingCanonicalDigest +from adcp.reporting.ledger._delivery_state import DeliveryContext, fail, replay, validate_transition +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingControlTotalRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDeliveryScope, + ReportingReceiptRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.models import ( + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.receipts.wire import ReceiptKind +from adcp.reporting.revision_selection import select_reporting_revision + +_REVISION_RECEIPT = TypeAdapter(ReportingRevisionReceiptRecord) +_ADJUSTMENT_RECEIPT = TypeAdapter(ReportingAdjustmentReceiptRecord) + + +def receipt_record( + kind: ReceiptKind, + item: dict[str, Any], + caller: ReportingDeliveryPrincipal, + obligation: ReportingObligationRecord | None, +) -> ReportingReceiptRecord: + if obligation is None or obligation.account_id != caller.account_id: + fail("REPORTING_RECORD_UNAVAILABLE") + body = dict(item) + body.pop("reporting_obligation_id", None) + body["scope"] = ReportingDeliveryScope( + obligation.generation_key, caller.consumer_id, obligation.reporting_obligation_id + ) + value: ReportingReceiptRecord | None = None + try: + if kind == "revision_receipt": + body["observed_control_totals"] = tuple( + ReportingControlTotalRecord(**total) for total in body["observed_control_totals"] + ) + digest = body.get("observed_canonical_content_digest") + if digest is not None: + body["observed_canonical_content_digest"] = ReportingCanonicalDigest( + **{k: v for k, v in digest.items() if k != "algorithm"} + ) + adapter = _REVISION_RECEIPT if kind == "revision_receipt" else _ADJUSTMENT_RECEIPT + value = adapter.validate_python(body) + except (ValueError, TypeError, ValidationError): + # Reject the record below without attaching private validation details as context. + pass + if value is None: + fail("INVALID_REPORTING_RECORD") + return value + + +def prepare_receipt( + record: ReportingReceiptRecord, + records: tuple[ReportingDeliveryRecord, ...], + context: DeliveryContext, + revisions: tuple[ReportingRevisionRecord, ...], + now: datetime, +) -> tuple[ReportingReceiptRecord, bool]: + existing = replay(record, records) + if existing is not None: + return cast(ReportingReceiptRecord, existing), False + if context.obligation is None: + fail("REPORTING_RECORD_UNAVAILABLE") + obligation = context.obligation + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + fail("REPORTING_HISTORY_CORRUPT") + # Historical snapshot receipts remain independently repairable after official + # publication. Evidence is bound to the supplied artifact, never the latest try. + return cast(ReportingReceiptRecord, validate_transition(record, records, context, now)), True + + +def success_result(record: ReportingReceiptRecord, added: bool) -> dict[str, Any]: + key = "receipt" if isinstance(record, ReportingRevisionReceiptRecord) else "adjustment_receipt" + return {"result": "recorded" if added else "unchanged", key: receipt_to_wire(record)} + + +def failed_result(item: dict[str, Any], error: LedgerConflictError) -> dict[str, Any]: + # The shared state machine uses closed codes. Never persist exception text or + # database details, and never distinguish an invisible target from a missing one. + return { + "result": "failed", + "reporting_receipt_id": item["reporting_receipt_id"], + "errors": [{"code": error.code, "message": "reporting receipt could not be recorded"}], + } diff --git a/src/adcp/reporting/receipts/required_schema.json b/src/adcp/reporting/receipts/required_schema.json new file mode 100644 index 000000000..3ec4c8a1f --- /dev/null +++ b/src/adcp/reporting/receipts/required_schema.json @@ -0,0 +1,410 @@ +{ + "column:reporting_receipt_ingestion_batches.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_batches.admission_epoch": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_receipt_ingestion_batches.canonical_request": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_receipt_ingestion_batches.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_batches.created_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_receipt_ingestion_batches.expected_count": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_receipt_ingestion_batches.final_response": { + "enabled": true, + "fingerprint": "1f539b84d6adb1a8577add320aabb81fdd5c614ad01b46c7b77eb55b995a6556" + }, + "column:reporting_receipt_ingestion_batches.final_sha256": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_receipt_ingestion_batches.finalized_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_receipt_ingestion_batches.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_batches.request_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_receipt_ingestion_boundaries.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_receipt_ingestion_boundaries.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_receipt_ingestion_boundaries.receipt_namespace": { + "enabled": true, + "fingerprint": "3c136011bb3ec377d8e534e35c5e0e338531ff0a5cae3e41945a2eec630c09e6" + }, + "column:reporting_receipt_ingestion_boundaries.reporting_receipt_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_boundaries.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_receipt_ingestion_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_heads.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_heads.max_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_receipt_ingestion_results.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.ordinal": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_receipt_ingestion_results.receipt_kind": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_receipt_ingestion_results.receipt_namespace": { + "enabled": true, + "fingerprint": "3c136011bb3ec377d8e534e35c5e0e338531ff0a5cae3e41945a2eec630c09e6" + }, + "column:reporting_receipt_ingestion_results.receipt_record_id": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_receipt_ingestion_results.recorded_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_receipt_ingestion_results.reporting_receipt_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_receipt_ingestion_results.result": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_admission_epoch_check": { + "enabled": true, + "fingerprint": "abd3e698b241244e9e751e940838ef5bcecdd79d21aab371f4b595818b5b1f95" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_canonical_request_check": { + "enabled": true, + "fingerprint": "912af24f35b588c366cb2e7520d612baab36ec0e91e5b2489a8a8ef4fc3e2d68" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check": { + "enabled": true, + "fingerprint": "5fcb92d5ab815be1be7ba3958714ac48a157ee8e3780601e1807108704249a12" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check1": { + "enabled": true, + "fingerprint": "66f19a2cc809660aef7b568e84764eb65c35964a4523a6c245afc61e90678500" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check2": { + "enabled": true, + "fingerprint": "927d933d82230e62d52dd8e37bdc6a1879f7765f2060d9c9fcd3ad39494744c8" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check3": { + "enabled": true, + "fingerprint": "6540b709a9e52537489a92313773f473f9ef7e1e1d9d59cebeaf0490a49006f1" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check4": { + "enabled": true, + "fingerprint": "47d0a4b1d9cd02cfd02f0e8d88e2010a6f7c70c9d1a59ed6eedfd283e3e4ea09" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_check5": { + "enabled": true, + "fingerprint": "1b2547b965ab254cabcecf3edab1bdf0c65848b46c52a9c0268629f5a5f1f44f" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_expected_count_check": { + "enabled": true, + "fingerprint": "723d4bcaf0cee6c6aa069b6ed53e19524ba88a12da5a0c4de7809ee446b5bb6e" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_idempotency_key_check": { + "enabled": true, + "fingerprint": "bb45ae8ff7c580eb4c52bca8b48aae27441c134868327a8627baf671e651dc6d" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_pkey": { + "enabled": true, + "fingerprint": "bd14bccb4c52b10981823f13e78f2d623d0f82b2d8103fe009b0ead4e7c4d7ed" + }, + "constraint:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_request_sha256_check": { + "enabled": true, + "fingerprint": "e4296548e9e39b78a6b347016c0bccde754cd6a0afbbe2ed963a125a1bca1918" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_b_account_id_consumer_id_recei_fkey": { + "enabled": true, + "fingerprint": "e7f6d3026b2a48db5725fedb3296c4ea22741e47af12b0c59e30bb4e29f6e2ce" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "4da8231cf59acee96ee91b4f06240f128843063504be0c21755ed8748bfbc403" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "460b11d8840751c3e894245e33c24e704407f4b3ca42bbd9e832cb48a0c3b24d" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check1": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check2": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check3": { + "enabled": true, + "fingerprint": "53c1080ff5edc81171541e32d6cdaf69dfa2376bd4cc5bb31774ab2d4b773ff2" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check4": { + "enabled": true, + "fingerprint": "65d0dca50d7ca308123252fcc784fb427fd857d8bf3092bd85fc8fff5787d7ec" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check5": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_check6": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_input_check": { + "enabled": true, + "fingerprint": "1403f246fdde17015118d212847ed5d8169df3adc0364598cbd5e82be066e1bf" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_input_check1": { + "enabled": true, + "fingerprint": "7e7cefdce39915938ec708771cbe2b61b774ad362d41d2d3ef2a7896c9baa177" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_input_check2": { + "enabled": true, + "fingerprint": "908f04e9bfa67eef862e12260105785952531a776178a8b57ec271ff21b3aec5" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_pkey": { + "enabled": true, + "fingerprint": "00dd5f9ba4e0d0face7ab27fe5e52ffa9c4101943c95afb8ec1badb401c7010d" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_receipt_namespace_check": { + "enabled": true, + "fingerprint": "2c7db1fb7a2ce91e34550c4a9925c8dadadc94ef57d8e020e4ba8d7875c94b15" + }, + "constraint:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_receipt_ingestion_heads.reporting_receipt_ingestion_heads_max_sequence_check": { + "enabled": true, + "fingerprint": "ee47acec8d450eed7f99b637596847d21bc449a12317a605fd6b6cb0fad6aa79" + }, + "constraint:reporting_receipt_ingestion_heads.reporting_receipt_ingestion_heads_pkey": { + "enabled": true, + "fingerprint": "4d853add149814edf9eadd3f752bf43f1f7164cba2bd079f0349d4d540a7f20c" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_idemp_fkey": { + "enabled": true, + "fingerprint": "b95192b8aa7ab2d51fc9ba9dc1c88ee5c32d39da4725efb06cfb67b494588383" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_idempo_key": { + "enabled": true, + "fingerprint": "3e31e8cc82fc1bc507bc26bd28159937ac3f8ab8fc55a9926fc476568e3e49b1" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_recei_fkey": { + "enabled": true, + "fingerprint": "9af695720cddd0dcbc5cff566319d54a58086960a9ab861d8908b3d7a43db762" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_check": { + "enabled": true, + "fingerprint": "e665b17e336517d76d15cf779c34c687a7ebb443ccb89e1fa2ae90cbb74120d7" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_check1": { + "enabled": true, + "fingerprint": "3c812b76d750d500699370919d0e256db83181eab776d254f897a5f68fb5ff96" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_check2": { + "enabled": true, + "fingerprint": "af7466439087470d4590ea91c3d7d7de23bf47af1424ed14c254c96727784149" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_ordinal_check": { + "enabled": true, + "fingerprint": "681e853666470c336498661c1ed30b8e5ed60024aa72dead813393d62eca83ce" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_pkey": { + "enabled": true, + "fingerprint": "330a35cc748968f52093f6d1707fbcf4ac136f7d76327874eedb54000f2bb793" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_receipt_kind_check": { + "enabled": true, + "fingerprint": "807349018715c3cfc653a1889df10225848b212a749d4977dfb37f4799674474" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_receipt_namespace_check": { + "enabled": true, + "fingerprint": "2c7db1fb7a2ce91e34550c4a9925c8dadadc94ef57d8e020e4ba8d7875c94b15" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_reporting_receipt_id_check": { + "enabled": true, + "fingerprint": "db3e198b5d35078fd20db3918eca7084516ec6eb8d49efa1a6eec5a0a744c6f2" + }, + "constraint:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_result_check": { + "enabled": true, + "fingerprint": "c57c0272ed7fd01b0fbb2972e8d5144bbf4566e21a3fda58d7d521dcecdf499a" + }, + "function:reporting_receipt_ingestion_batch_guard()": { + "enabled": true, + "fingerprint": "ebaa3104d98c18b32d36b7af443877f80d2a6ea4d564be68053ad937ba80b4b3" + }, + "function:reporting_receipt_ingestion_canonical(document jsonb)": { + "enabled": true, + "fingerprint": "83894ca562527560f07f5f8e27e4bccf728e851f5f12652ebc82f8f04630ad3e" + }, + "function:reporting_receipt_ingestion_graph()": { + "enabled": true, + "fingerprint": "570d827fe99bcd57d5f2614c0ed146d5d9cbd51d0c816ee6262cbdff140ab5f7" + }, + "function:reporting_receipt_ingestion_immutable()": { + "enabled": true, + "fingerprint": "099d3850fab91c795d5516e3e99066e1d02598a71e5d71e8dee9c69d998abb57" + }, + "function:reporting_receipt_ingestion_result_guard()": { + "enabled": true, + "fingerprint": "628db026db8a5a70ccd3e9c66c3b5ef344c80da2a662cc97c488a103dac3dc4c" + }, + "function:reporting_receipt_ingestion_sha256(document jsonb)": { + "enabled": true, + "fingerprint": "6f95f80903a080155fd6ecc522fdc99a6f656914f3973b44b1cb46a3c393f228" + }, + "function:reporting_receipt_ingestion_utf16(value text)": { + "enabled": true, + "fingerprint": "e136f3cbdb9aed5e8d48defef32a573c6d4b26a3f52a952022d1246adc0dffca" + }, + "function:reporting_receipt_ingestion_wire(document jsonb)": { + "enabled": true, + "fingerprint": "1a01db876edf8d3f077375aa09edc56c1feb5ab658d7a5aa3cdb17c461675cb0" + }, + "index:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batches_pkey": { + "enabled": true, + "fingerprint": "65c0ec48dceaace248f672b59680b283319529a1fd41125a15a96733092c5387" + }, + "index:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "54844be45542b67fafaef6dd517b51e5a7147877035e47f711af914b90422f26" + }, + "index:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "e6c0904fc74d63aef2188c36ee5788d297d8f6175a0d10d5fe658a8d8030cbe3" + }, + "index:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundaries_pkey": { + "enabled": true, + "fingerprint": "fe4b6f5a34e2c778f77138231b76fd27e8ee56639e357293b9f4700553db0da7" + }, + "index:reporting_receipt_ingestion_heads.reporting_receipt_ingestion_heads_pkey": { + "enabled": true, + "fingerprint": "a6fbc55d5bd3cc94f0cfdb3c3ea72c1611cccb2aeccbd67c0576076149dd2804" + }, + "index:reporting_receipt_ingestion_results.reporting_receipt_ingestion_r_account_id_consumer_id_idempo_key": { + "enabled": true, + "fingerprint": "bb57e83db85651fd68160cc0b437f0a730a4587d9f7f832b7b10c1e6141b0281" + }, + "index:reporting_receipt_ingestion_results.reporting_receipt_ingestion_results_pkey": { + "enabled": true, + "fingerprint": "16d6c560b7bb8eb6e36ae8f59aaff25aa8f2efb906f2f6ce9795c4a753f8baff" + }, + "index:reporting_reconciliation_records.reporting_receipt_ingestion_chain": { + "enabled": true, + "fingerprint": "956676e509d0b5c196f9abaa090eb4a57e8b764838572951105b9096b68b36b1" + }, + "table:reporting_receipt_ingestion_batches": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_receipt_ingestion_boundaries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_receipt_ingestion_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_receipt_ingestion_results": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_receipt_ingestion_batches.reporting_receipt_ingestion_batch": { + "enabled": true, + "fingerprint": "d5c809b674a594c41801296da98b81ba108bd08414bfc3bb711b266a7d589ef0" + }, + "trigger:reporting_receipt_ingestion_boundaries.reporting_receipt_ingestion_boundary_immutable": { + "enabled": true, + "fingerprint": "1ccb241ae1c0a4a467c64cc246d1ae159f5c78621b7b5a549f862e4b808958d1" + }, + "trigger:reporting_receipt_ingestion_results.reporting_receipt_ingestion_result": { + "enabled": true, + "fingerprint": "053262f24d8e8762ff76615ce66db34be3403992d0aeec598c27ee709ee5835a" + }, + "trigger:reporting_receipt_ingestion_results.reporting_receipt_ingestion_result_immutable": { + "enabled": true, + "fingerprint": "3e71c52e6576becdc89161b72a6b986f83e0301400e9049ed88a64d3b1889a56" + }, + "trigger:reporting_reconciliation_records.reporting_receipt_ingestion_graph": { + "enabled": true, + "fingerprint": "05a5ce8a48fb20c3682b8baede61ad9a2391e06908a91a6148760eec5550f685" + } +} diff --git a/src/adcp/reporting/receipts/schema.py b/src/adcp/reporting/receipts/schema.py new file mode 100644 index 000000000..cb874c93e --- /dev/null +++ b/src/adcp/reporting/receipts/schema.py @@ -0,0 +1,27 @@ +"""Isolated ingestion readiness; never widens an A/B/C/B1/B2.1 manifest.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.materializer.schema import validate_materializer_schema +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts.errors import ReportingReceiptError + + +async def validate_receipt_schema(connection: Any, *, notifications: bool = False) -> None: + ready = False + try: + required = json.loads( + files("adcp.reporting.receipts").joinpath("required_schema.json").read_text() + ) + installed = await schema_objects(connection) + if required and all(installed.get(key) == expected for key, expected in required.items()): + await validate_materializer_schema(connection, notifications=notifications) + ready = True + except Exception: + ready = False + if not ready: + raise ReportingReceiptError("RECEIPT_SCHEMA_UNREADY") diff --git a/src/adcp/reporting/receipts/store.py b/src/adcp/reporting/receipts/store.py new file mode 100644 index 000000000..a163893f4 --- /dev/null +++ b/src/adcp/reporting/receipts/store.py @@ -0,0 +1,24 @@ +"""Optional batch participant; no change to any required legacy store protocol.""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.receipts.capture import ReportingReceiptBoundary + + +@runtime_checkable +class ReportingReceiptBatchStore(Protocol): + """Trusted callers only. Each ordinal co-commits with all of its domain effects.""" + + async def ingest_receipt_batch( + self, request: dict[str, Any], *, caller: ReportingDeliveryPrincipal + ) -> dict[str, Any]: ... + + +@runtime_checkable +class ReportingReceiptCaptureStore(Protocol): + async def read_receipt_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingReceiptBoundary, ...]: ... diff --git a/src/adcp/reporting/receipts/transport.py b/src/adcp/reporting/receipts/transport.py new file mode 100644 index 000000000..a854526cc --- /dev/null +++ b/src/adcp/reporting/receipts/transport.py @@ -0,0 +1,200 @@ +"""Request-local lossless receipt JSON, before transport/model normalization.""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable +from decimal import Decimal, InvalidOperation +from typing import Any + +from adcp.reporting.canonical_json import MAX_SAFE_INTEGER +from adcp.reporting.receipts.wire import TASK + +# The standard serve() body limiter applies first. Direct mounts also bound +# this additional receipt-only capture; a larger body is rejected, never rounded. +MAX_RECEIPT_BODY_BYTES = 10 * 1024 * 1024 +RAW_RECEIPT_BODY_SCOPE_KEY = "adcp.receipt_ingress.raw_body" + + +def receipt_body_receive( + scope: dict[str, Any], + receive: Callable[[], Awaitable[Any]], + *, + limit: int = MAX_RECEIPT_BODY_BYTES, +) -> Callable[[], Awaitable[Any]]: + """Tee only the bytes delivered to this mounted HTTP request's decoder. + + No caller metadata, global cache, request ID lookup or auth shortcut is + involved. The existing upstream body cap still applies first. + """ + captured: bytearray | None = bytearray() + + async def capture() -> Any: + nonlocal captured + message = await receive() + if message.get("type") == "http.request": + chunk = message.get("body", b"") + if captured is not None: + if len(captured) + len(chunk) > limit: + captured = None + else: + captured.extend(chunk) + if not message.get("more_body", False): + scope[RAW_RECEIPT_BODY_SCOPE_KEY] = ( + bytes(captured) if captured is not None else None + ) + return message + + return capture + + +def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]: + result: dict[str, Any] = {} + for key, value in pairs: + if key in result: + raise ValueError("ambiguous JSON object") + result[key] = value + return result + + +def _exact_integer(value: Decimal) -> int: + """Accept protobuf's 1.0 spelling only when its *raw text* is an exact integer. + + A binary float cannot establish this: 1.000000000000000000001 has already + rounded to 1.0 there. Decimal examines the authenticated HTTP bytes without + rounding and bounds the value before any potentially large integer allocation. + """ + try: + if ( + value.is_finite() + and value.copy_abs() <= MAX_SAFE_INTEGER + and value == value.to_integral_value() + ): + return int(value) + except (InvalidOperation, ValueError, OverflowError): + # Conversion failed; report the bounded integer error outside this handler. + pass + raise ValueError("receipt JSON requires exact safe integers") + + +def _receipt_numbers(value: Any) -> Any: + if isinstance(value, Decimal): + return _exact_integer(value) + if type(value) is dict: + return {key: _receipt_numbers(item) for key, item in value.items()} + if type(value) is list: + return [_receipt_numbers(item) for item in value] + return value + + +def _raw_number(raw: str) -> Decimal: + try: + return Decimal(raw) + except InvalidOperation: + # Preserve route identification even for an unrepresentable exponent. + # This sentinel can only produce an invalid empty request below; it is + # never handed to a handler as an accepted numeric value. + return Decimal("NaN") + + +def _raw_json(body: str | bytes) -> Any: + return json.loads( + body, + object_pairs_hook=_unique_object, + parse_float=_raw_number, + parse_constant=_raw_number, + ) + + +def _a2a_receipt_invocation(body: bytes | None) -> dict[str, Any] | None: + try: + if body is None: + return None + envelope = _raw_json(body) + if ( + type(envelope) is not dict + or envelope.get("jsonrpc") != "2.0" + or envelope.get("method") + not in {"message/send", "message/stream", "SendMessage", "SendStreamingMessage"} + ): + return None + parts = envelope["params"]["message"]["parts"] + invocations = [] + for part in parts: + if type(part) is not dict: + return None + data = part.get("data") + if data is None and isinstance(part.get("text"), str): + try: + data = _raw_json(part["text"]) + except ValueError: + continue + if type(data) is dict and data.get("skill"): + invocations.append(data) + if len(invocations) == 1 and invocations[0]["skill"] == TASK: + return dict(invocations[0]) + except (ValueError, TypeError, KeyError, RecursionError): + # Malformed envelopes cannot establish a unique receipt invocation. + pass + return None + + +def a2a_receipt_parameters(body: bytes | None) -> dict[str, Any] | None: + """Distinguish an exact receipt route from invalid receipt parameters. + + None means no uniquely identified standard invocation. An empty dictionary + means that invocation's parameters are invalid and must reach the ordinary + whole-shape rejection. Neither case can authorize a domain write. + """ + invocation = _a2a_receipt_invocation(body) + if invocation is None: + return None + try: + params = invocation.get("parameters") + if type(params) is dict: + return dict(_receipt_numbers(params)) + except (ValueError, TypeError, RecursionError): + # The receipt route is known; return invalid parameters for ordinary rejection. + pass + return {} + + +def a2a_receipt_has_invalid_unicode(body: bytes | None) -> bool: + """Scope the upstream protobuf codec error boundary to this exact raw route.""" + if body is None or _a2a_receipt_invocation(body) is None: + return False + try: + pending = [_raw_json(body)] + while pending: + value = pending.pop() + if isinstance(value, str) and any(0xD800 <= ord(c) <= 0xDFFF for c in value): + return True + if type(value) is dict: + pending.extend(value) + pending.extend(value.values()) + elif type(value) is list: + pending.extend(value) + except (ValueError, TypeError, RecursionError): + # Failed parsing establishes no Unicode diagnosis; ordinary validation still applies. + pass + return False + + +def mcp_receipt_parameters(body: bytes | None) -> dict[str, Any]: + """The already selected tools/call must match this exact receipt invocation.""" + try: + if body is None: + return {} + envelope = _raw_json(body) + if ( + type(envelope) is dict + and envelope.get("jsonrpc") == "2.0" + and envelope.get("method") == "tools/call" + and envelope["params"]["name"] == TASK + and type(envelope["params"].get("arguments")) is dict + ): + return dict(_receipt_numbers(envelope["params"]["arguments"])) + except (ValueError, TypeError, KeyError, RecursionError): + # The selected MCP route must receive invalid parameters, never a partial body. + pass + return {} diff --git a/src/adcp/reporting/receipts/wire.py b/src/adcp/reporting/receipts/wire.py new file mode 100644 index 000000000..db0b9f065 --- /dev/null +++ b/src/adcp/reporting/receipts/wire.py @@ -0,0 +1,189 @@ +"""Lossless raw batch admission and isolated transport-schema overlays. + +Never edit cached upstream schemas or generated models. The portable canonical +schema retains conditionals omitted by model generation and compact MCP profiles. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from functools import lru_cache +from typing import Any, Literal, cast + +from jsonschema import Draft7Validator, FormatChecker + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.validation.schema_loader import get_portable_schema + +TASK = "sync_reporting_receipts" +ReceiptKind = Literal["revision_receipt", "adjustment_receipt"] +_IDENTITY_FIELDS = frozenset( + { + "consumer", + "consumer_id", + "principal", + "principal_id", + "buyer", + "buyer_agent", + "buyer_agent_id", + "governance_agent", + "governance_principal", + "caller_identity", + "tenant_id", + "auth_info", + "auth_principal", + "agent_url", + } +) + + +def receipt_schema( + direction: Literal["request", "sync"], *, version: str | None = None +) -> dict[str, Any]: + """Independent mutable schema for pinned, unpinned and model-fallback mounts.""" + schema = get_portable_schema(TASK, direction, version=version) + if schema is None: + raise ReportingReceiptError("RECEIPT_SCHEMA_UNREADY") + if direction == "request": + schema["anyOf"] = [{"required": ["receipts"]}, {"required": ["adjustment_receipts"]}] + rules = schema.setdefault("allOf", []) + rules.append( + {"not": {"anyOf": [{"required": [name]} for name in sorted(_IDENTITY_FIELDS)]}} + ) + for name in ("receipts", "adjustment_receipts"): + array = schema["properties"][name] + array.update(minItems=1, maxItems=100) + array["items"] = {"allOf": [array["items"], {"not": {"required": ["received_at"]}}]} + # JSON Schema has no cross-array sum operator. These bounded implications + # express the exact combined limit without a proprietary validator keyword. + rules.extend( + { + "if": {"required": ["receipts"], "properties": {"receipts": {"minItems": n}}}, + "then": {"properties": {"adjustment_receipts": {"maxItems": 100 - n}}}, + } + for n in range(1, 101) + ) + return schema + + +@lru_cache(maxsize=2) +def _validator(direction: Literal["request", "sync"]) -> Any: + return Draft7Validator(receipt_schema(direction), format_checker=FormatChecker()) + + +def validate_receipt_request(request: object) -> None: + """Preflight the entire supplied shape before auth-dependent lookup or writes.""" + valid = False + try: + if type(request) is dict: + supplied = [ + request[name] for name in ("receipts", "adjustment_receipts") if name in request + ] + if supplied and all(type(items) is list and items for items in supplied): + items = [item for array in supplied for item in array] + ids = [item.get("reporting_receipt_id") for item in items if type(item) is dict] + valid = ( + 1 <= len(items) <= 100 + and len(ids) == len(items) + and all(isinstance(i, str) for i in ids) + and len(set(ids)) == len(ids) + and not _IDENTITY_FIELDS.intersection(request) + and _validator("request").is_valid(request) + ) + if valid: + # Also reject non-JSON values and non-finite/lossy JCS numbers. + canonical_json_utf8_v1(request) + # JSONB requires Unicode scalar values without U+0000. + # Reject unsupported keys/values before a header on both + # stores, including strings inside arbitrary context/ext. + pending: list[Any] = [request] + while pending: + value = pending.pop() + if isinstance(value, str) and any( + c == "\x00" or 0xD800 <= ord(c) <= 0xDFFF for c in value + ): + raise ValueError("unsupported JSON string") + if type(value) is dict: + pending.extend(value) + pending.extend(value.values()) + elif type(value) is list: + pending.extend(value) + except (ValueError, TypeError, OverflowError, RecursionError): + valid = False + if not valid: + raise ReportingReceiptError("INVALID_REQUEST") + + +def validate_receipt_results( + results: object, batch: ReceiptBatch, *, complete: bool = False +) -> None: + """Validate every durable prefix before resuming any domain writes.""" + valid = False + try: + if type(results) is list and len(results) <= len(batch.items): + if not results and not complete: + return + valid = _validator("sync").is_valid({"status": "completed", "results": results}) + ids = [ + ( + result["reporting_receipt_id"] + if result["result"] == "failed" + else result.get("receipt", result.get("adjustment_receipt"))[ + "reporting_receipt_id" + ] + ) + for result in results + ] + expected = [item["reporting_receipt_id"] for _, item in batch.items] + valid = valid and ids == (expected if complete else expected[: len(results)]) + for (kind, _), result in zip(batch.items, results): + if result["result"] != "failed": + key = "receipt" if kind == "revision_receipt" else "adjustment_receipt" + valid = valid and key in result + except (ValueError, TypeError, KeyError): + valid = False + if not valid: + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + + +def validate_receipt_response(response: object, batch: ReceiptBatch) -> None: + """Validate exact ID coverage/order as well as the final wire schema.""" + if type(response) is not dict or not _validator("sync").is_valid(response): + raise ReportingReceiptError("RECEIPT_HISTORY_CORRUPT") + validate_receipt_results(response["results"], batch, complete=True) + + +@dataclass(frozen=True) +class ReceiptBatch: + """Private immutable whole-request identity, including context and extensions.""" + + canonical_request: bytes = field(repr=False) + + @classmethod + def parse(cls, request: dict[str, Any]) -> ReceiptBatch: + validate_receipt_request(request) + return cls(canonical_json_utf8_v1(request)) + + @property + def request(self) -> dict[str, Any]: + return cast(dict[str, Any], json.loads(self.canonical_request)) + + @property + def key(self) -> str: + return str(self.request["idempotency_key"]) + + @property + def digest(self) -> str: + return hashlib.sha256(self.canonical_request).hexdigest() + + @property + def items(self) -> tuple[tuple[ReceiptKind, dict[str, Any]], ...]: + request = self.request + arrays: tuple[tuple[ReceiptKind, str], ...] = ( + ("revision_receipt", "receipts"), + ("adjustment_receipt", "adjustment_receipts"), + ) + return tuple((kind, item) for kind, name in arrays for item in request.get(name, [])) diff --git a/src/adcp/server/a2a_server.py b/src/adcp/server/a2a_server.py index 729921e3a..6b7602810 100644 --- a/src/adcp/server/a2a_server.py +++ b/src/adcp/server/a2a_server.py @@ -159,17 +159,73 @@ def _get_decisioning_adcp_error_types() -> tuple[type[BaseException], ...]: class _A2ARequestContextMiddleware: """Make the originating HTTP request available during A2A dispatch.""" - def __init__(self, app: Any) -> None: + def __init__(self, app: Any, *, receipt_ingress: bool = False) -> None: self.app = app + self.receipt_ingress = receipt_ingress async def __call__(self, scope: Any, receive: Any, send: Any) -> None: if scope["type"] != "http": await self.app(scope, receive, send) return - token = _A2A_REQUEST_CONTEXT.set(Request(scope, receive=receive)) + downstream_receive = receive + if self.receipt_ingress: + from collections import deque + + from adcp.reporting.receipts.transport import ( + MAX_RECEIPT_BODY_BYTES, + RAW_RECEIPT_BODY_SCOPE_KEY, + a2a_receipt_has_invalid_unicode, + receipt_body_receive, + ) + + captured_receive = receipt_body_receive(scope, receive) + prefix = bytearray() + trailing = None + more_body = False + # Read ahead only to the bounded receipt capture limit, then replay + # the same bytes. Coalesce small chunks so one-byte/empty chunks + # cannot grow an unbounded queue of ASGI message dictionaries. + while True: + message = await captured_receive() + chunk = message.get("body", b"") + if ( + message.get("type") != "http.request" + or len(prefix) + len(chunk) > MAX_RECEIPT_BODY_BYTES + ): + trailing, more_body = message, True + break + prefix.extend(chunk) + more_body = message.get("more_body", False) + if not more_body: + break + buffered: deque[Any] = deque() + if prefix or trailing is None: + buffered.append( + {"type": "http.request", "body": bytes(prefix), "more_body": more_body} + ) + if trailing is not None: + buffered.append(trailing) + + async def replay_receive() -> Any: + return buffered.popleft() if buffered else await captured_receive() + + downstream_receive = replay_receive + if a2a_receipt_has_invalid_unicode(scope.get(RAW_RECEIPT_BODY_SCOPE_KEY)): + from starlette.responses import JSONResponse + + await JSONResponse( + { + "jsonrpc": "2.0", + "id": None, + "error": {"code": -32700, "message": "Invalid receipt JSON encoding"}, + }, + status_code=400, + )(scope, downstream_receive, send) + return + token = _A2A_REQUEST_CONTEXT.set(Request(scope, receive=downstream_receive)) try: - await self.app(scope, receive, send) + await self.app(scope, downstream_receive, send) finally: _A2A_REQUEST_CONTEXT.reset(token) @@ -558,13 +614,44 @@ def _parse_request(self, context: RequestContext) -> tuple[str | None, dict[str, JSON fallback). """ request = _A2A_REQUEST_CONTEXT.get() + parsed = None if request is not None: cached = request.scope.get(_A2A_PARSED_REQUEST_SCOPE_KEY) if cached is not None: - return cast(tuple[str | None, dict[str, Any]], cached) - if self._message_parser is not None: - return self._message_parser(context) - return self._default_parse_request(context) + parsed = cast(tuple[str | None, dict[str, Any]], cached) + if parsed is None: + if self._message_parser is not None: + parsed = self._message_parser(context) + else: + try: + parsed = self._default_parse_request(context) + except ValueError: + # The protobuf JSON printer refuses non-finite numbers. + # A standard raw receipt invocation still reaches mandatory + # strict preflight and a closed INVALID_REQUEST, without + # logging a protobuf serialization exception. + if request is not None: + from adcp.reporting.receipts.transport import ( + RAW_RECEIPT_BODY_SCOPE_KEY, + a2a_receipt_parameters, + ) + + raw = a2a_receipt_parameters(request.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY)) + if raw is not None: + return "sync_reporting_receipts", raw + raise + if parsed[0] == "sync_reporting_receipts" and request is not None: + from adcp.reporting.receipts.transport import ( + RAW_RECEIPT_BODY_SCOPE_KEY, + a2a_receipt_parameters, + ) + + # The protobuf representation has already lost numeric lexemes. + # Only the standard raw invocation can authorize the batch body. + return parsed[0], ( + a2a_receipt_parameters(request.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY)) or {} + ) + return parsed def _default_parse_request(self, context: RequestContext) -> tuple[str | None, dict[str, Any]]: """Built-in parser. Supports two formats: @@ -1419,7 +1506,10 @@ def agent_card_url(request: Request) -> str: # during executor dispatch. This is installed for direct # ``create_a2a_server`` adopters as well as the unified ``serve`` path, # independent of whether bearer-auth middleware is configured. - app.add_middleware(_A2ARequestContextMiddleware) + app.add_middleware( + _A2ARequestContextMiddleware, + receipt_ingress="sync_reporting_receipts" in executor.supported_skills, + ) # Startup log lives on the create_a2a_server path (symmetric with # MCP's _register_handler_tools). Moved out of diff --git a/src/adcp/server/idempotency/store.py b/src/adcp/server/idempotency/store.py index ebbee1f47..621b4e7d1 100644 --- a/src/adcp/server/idempotency/store.py +++ b/src/adcp/server/idempotency/store.py @@ -37,6 +37,7 @@ import weakref from collections.abc import AsyncIterator, Awaitable, Callable from contextlib import asynccontextmanager +from contextvars import ContextVar from functools import wraps from typing import Any @@ -52,6 +53,10 @@ logger = logging.getLogger(__name__) +# Set only by the SDK transport dispatcher, never from request metadata. This +# also covers middleware whose wrapped callable is named call_next/execute. +_RECEIPT_BATCH_DISPATCH: ContextVar[bool] = ContextVar("receipt_batch_dispatch", default=False) + # Registry of functions returned by IdempotencyStore.wrap. Read by # adcp.decisioning.validate_idempotency.is_wrapped() to reconcile the # adopter's declared IdempotencySupported capability against actual @@ -215,6 +220,15 @@ def wrap(self, handler: HandlerFn) -> HandlerFn: @wraps(handler) async def _wrapped(*args: Any, **kwargs: Any) -> Any: + if ( + _RECEIPT_BATCH_DISPATCH.get() + or getattr(handler, "__name__", None) == "sync_reporting_receipts" + ): + # Receipt ingress owns durable whole-batch replay after exact + # account/consumer authorization. This generic key omits the + # resolved account and cached hits bypass that authorization. + # It also adds replayed=True, changing the immutable response. + return await handler(*args, **kwargs) handler_self, hash_source, context = _resolve_call_args(args, kwargs) operation = getattr(handler, "__name__", "handler") diff --git a/src/adcp/server/mcp_tools.py b/src/adcp/server/mcp_tools.py index 716deb62a..ebb8cd421 100644 --- a/src/adcp/server/mcp_tools.py +++ b/src/adcp/server/mcp_tools.py @@ -540,9 +540,21 @@ def _widen_media_buy_output_schema_for_legacy_statuses( "properties": { "account": {"type": "object"}, "idempotency_key": {"type": "string"}, - "receipts": {"type": "array"}, + "receipts": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": {"type": "object", "not": {"required": ["received_at"]}}, + }, + "adjustment_receipts": { + "type": "array", + "minItems": 1, + "maxItems": 100, + "items": {"type": "object", "not": {"required": ["received_at"]}}, + }, }, - "required": ["account", "idempotency_key", "receipts"], + "required": ["account", "idempotency_key"], + "anyOf": [{"required": ["receipts"]}, {"required": ["adjustment_receipts"]}], }, }, { @@ -2401,7 +2413,16 @@ def get_tools_for_handler( # The in-memory registry shares memoized schema subtrees to keep server # startup compact. Public definitions remain ordinary independently # mutable JSON values, matching the pre-memoization behavior. - return [_copy_json_without_aliases(tool) for tool in selected] + definitions = [_copy_json_without_aliases(tool) for tool in selected] + for definition in definitions: + if definition["name"] == "sync_reporting_receipts": + from adcp.reporting.receipts.wire import receipt_schema + + # Also runs when model generation fell back to hand-written + # stubs. Do not mutate cached upstream/generated schema inputs. + definition["inputSchema"] = receipt_schema("request") + definition["outputSchema"] = receipt_schema("sync") + return definitions if not list_validator_keys(version=resolved_version): raise ValueError( @@ -2419,6 +2440,13 @@ def get_tools_for_handler( if input_schema is None: continue definition = copy.deepcopy(tool) + if name == "sync_reporting_receipts": + from adcp.reporting.receipts.wire import receipt_schema + + definition["inputSchema"] = receipt_schema("request", version=resolved_version) + definition["outputSchema"] = receipt_schema("sync", version=resolved_version) + versioned.append(definition) + continue definition["inputSchema"] = input_schema output_schema = get_mcp_schema(name, "sync", version=resolved_version) if output_schema is not None: @@ -2745,11 +2773,48 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) raw_params = params # Preserve original wire params for context echo. + if method_name == "sync_reporting_receipts": + from adcp.reporting.receipts.errors import ReportingReceiptError + from adcp.reporting.receipts.wire import ReceiptBatch + + # Financial batches require the raw shape even with generic + # validation disabled. Coercion must not erase supplied empty + # arrays, read-only fields, timestamps, or request extensions. + try: + receipt_batch = ReceiptBatch.parse(raw_params) + raw_params = receipt_batch.request + except ReportingReceiptError as exc: + raise ADCPTaskError( + operation=method_name, errors=[Error(code=exc.code, message=str(exc))] + ) from None + if pre_validation_hooks: try: params = _apply_pre_validation_hooks( - pre_validation_hooks, method_name, dict(params) + pre_validation_hooks, + method_name, + ( + copy.deepcopy(params) + if method_name == "sync_reporting_receipts" + else dict(params) + ), ) + if method_name == "sync_reporting_receipts": + # A rewrite would change the identity the buyer authorized. + # Hooks may inspect this task; financial mutations fail closed. + try: + if ReceiptBatch.parse(params) != receipt_batch: + raise ReportingReceiptError("INVALID_REQUEST") + except ReportingReceiptError: + raise ADCPTaskError( + operation=method_name, + errors=[ + Error( + code="INVALID_REQUEST", + message="receipt requests cannot be rewritten", + ) + ], + ) from None except PreValidationHookError as exc: raise ADCPTaskError( operation=method_name, @@ -2959,7 +3024,7 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) ], ) from exc - if isinstance(params, dict): + if isinstance(params, dict) and method_name != "sync_reporting_receipts": params = _apply_unknown_field_policy( method_name, params, @@ -2987,7 +3052,9 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) ) call_params: Any = params - if params_model is not None and isinstance(params, dict): + if method_name == "sync_reporting_receipts": + call_params = raw_params + elif params_model is not None and isinstance(params, dict): try: call_params = params_model.model_validate(params) except ValidationError as exc: @@ -3142,7 +3209,7 @@ async def call_tool(params: dict[str, Any], context: ToolContext | None = None) # envelope is enhanced on the dedicated error paths # (``build_mcp_error_result`` / ``_send_adcp_error``), so skip # it here to avoid a double pass. - if "adcp_error" not in result: + if "adcp_error" not in result and method_name != "sync_reporting_receipts": _apply_response_enhancer(response_enhancer, method_name, result, ctx) if response_mode is not None and response_mode != "off" and isinstance(result, dict): diff --git a/src/adcp/server/serve.py b/src/adcp/server/serve.py index 618d260d9..96e03ce4f 100644 --- a/src/adcp/server/serve.py +++ b/src/adcp/server/serve.py @@ -369,6 +369,7 @@ def _get_starlette_request_for_dispatch() -> Any: try: return _ADCP_MCP_REQUEST_CONTEXT.get() except LookupError: + # No SDK dispatch is bound here; try the legacy test-harness context below. pass # MCP SDK v1 exposed this contextvar. MCP SDK v2 removed it, but keep @@ -539,8 +540,36 @@ async def _dispatch_with_middleware( if not middleware: return await call_handler() + receipt_request = None + if skill_name == "sync_reporting_receipts": + from adcp.exceptions import ADCPTaskError + from adcp.reporting.receipts.errors import ReportingReceiptError + from adcp.reporting.receipts.wire import ReceiptBatch + from adcp.types import Error + + try: + receipt_request = ReceiptBatch.parse(params) + except ReportingReceiptError as error: + raise ADCPTaskError( + operation=skill_name, errors=[Error(code=error.code, message=str(error))] + ) from None + async def _step(index: int) -> Any: if index >= len(middleware): + if receipt_request is not None: + try: + if ReceiptBatch.parse(params) != receipt_request: + raise ReportingReceiptError("INVALID_REQUEST") + except ReportingReceiptError: + raise ADCPTaskError( + operation=skill_name, + errors=[ + Error( + code="INVALID_REQUEST", + message="receipt requests cannot be rewritten", + ) + ], + ) from None return await call_handler() mw = middleware[index] @@ -549,6 +578,14 @@ async def call_next() -> Any: return await mw(skill_name, params, context, call_next) + if skill_name == "sync_reporting_receipts": + from adcp.server.idempotency.store import _RECEIPT_BATCH_DISPATCH + + token = _RECEIPT_BATCH_DISPATCH.set(True) + try: + return await _step(0) + finally: + _RECEIPT_BATCH_DISPATCH.reset(token) return await _step(0) @@ -1635,7 +1672,8 @@ def _run_mcp_http( import anyio import uvicorn - host = getattr(mcp.settings, "host", "0.0.0.0") + # Intentional server listener fallback; deployment settings select the host. + host = getattr(mcp.settings, "host", "0.0.0.0") # nosec B104 port = int(mcp.settings.port) log_level = getattr(mcp.settings, "log_level", "INFO").lower() @@ -1722,7 +1760,8 @@ def _build_a2a_app( from adcp.server.a2a_server import create_a2a_server from adcp.server.discovery import resolve_base_url - resolved_base_url = resolve_base_url("0.0.0.0", port, base_url) + # The URL resolver translates the wildcard listener into a usable public URL. + resolved_base_url = resolve_base_url("0.0.0.0", port, base_url) # nosec B104 app = create_a2a_server( handler, @@ -1817,7 +1856,8 @@ def _serve_a2a( auth=auth, public_url=public_url, ) - sock = _bind_reusable_socket("0.0.0.0", resolved_port) + # Intentional public serving socket; authentication is composed by the app. + sock = _bind_reusable_socket("0.0.0.0", resolved_port) # nosec B104 try: # Same bind-boundary INFO as the MCP path so A2A adopters # also see one framework-controlled line confirming the @@ -2151,7 +2191,8 @@ def _serve_mcp_and_a2a( import uvicorn resolved_port = port or int(os.environ.get("PORT", "3001")) - resolved_host = host or os.environ.get("ADCP_HOST", "0.0.0.0") + # Intentional server default, configurable by argument or deployment env. + resolved_host = host or os.environ.get("ADCP_HOST", "0.0.0.0") # nosec B104 log_level = "info" app = _build_mcp_and_a2a_app( @@ -2443,7 +2484,10 @@ def create_mcp_server( from mcp.server.transport_security import TransportSecuritySettings resolved_port = port or int(os.environ.get("PORT", "3001")) - resolved_host = host if host is not None else (os.environ.get("ADCP_HOST") or "0.0.0.0") + # Intentional server default, configurable by argument or deployment env. + resolved_host = ( + host if host is not None else (os.environ.get("ADCP_HOST") or "0.0.0.0") # nosec B104 + ) mcp: Any = MCPServer(name, instructions=instructions) mcp.settings = _ADCPMCPSettingsProxy(mcp.settings) object.__setattr__(mcp.settings, "host", resolved_host) @@ -2609,6 +2653,10 @@ def streamable_http_app( class ADCPStreamableHTTPASGIApp: async def __call__(self, scope: Any, receive: Any, send: Any) -> None: + if getattr(mcp, "_adcp_receipt_ingress", False): + from adcp.reporting.receipts.transport import receipt_body_receive + + receive = receipt_body_receive(scope, receive, limit=max_request_body_size) token = _ADCP_MCP_REQUEST_CONTEXT.set(Request(scope, receive)) try: await streamable_http(scope, receive, send) @@ -2740,7 +2788,21 @@ def _register_tool( build_mcp_error_result, ) + if name == "sync_reporting_receipts": + object.__setattr__(mcp, "_adcp_receipt_ingress", True) + async def fn(**kwargs: Any) -> dict[str, Any]: + if name == "sync_reporting_receipts": + request_context = _get_starlette_request_for_dispatch() + if request_context is not None: + from adcp.reporting.receipts.transport import ( + RAW_RECEIPT_BODY_SCOPE_KEY, + mcp_receipt_parameters, + ) + + kwargs = mcp_receipt_parameters( + request_context.scope.get(RAW_RECEIPT_BODY_SCOPE_KEY) + ) # Caller identity: FastMCP does not expose an authenticated principal # at the SDK level (``Context.client_id`` is a session hint, not an # authenticated user). Sellers wire auth via HTTP middleware on diff --git a/tests/conformance/reporting/_materializer_installed.py b/tests/conformance/reporting/_materializer_installed.py index c56349360..4fd2d0358 100644 --- a/tests/conformance/reporting/_materializer_installed.py +++ b/tests/conformance/reporting/_materializer_installed.py @@ -6,7 +6,7 @@ import importlib.util import json import sys -from dataclasses import fields +from dataclasses import fields, replace from datetime import datetime, timedelta, timezone from importlib.resources import files from pathlib import Path @@ -26,6 +26,7 @@ async def main(): "adcp.reporting.revision_selection", "adcp.reporting.ledger", "adcp.reporting.outbox", + "adcp.reporting.receipts", ): module = importlib.import_module(name) assert len(module.__all__) == len(set(module.__all__)) @@ -72,6 +73,8 @@ async def main(): assert files("adcp.reporting.outbox").joinpath("required_status_selector_schema.json").is_file() assert files("adcp.reporting.ledger").joinpath("reporting_materializer.sql").is_file() assert files("adcp.reporting.materializer").joinpath("required_schema.json").is_file() + assert files("adcp.reporting.ledger").joinpath("reporting_receipt_ingestion.sql").is_file() + assert files("adcp.reporting.receipts").joinpath("required_schema.json").is_file() start = datetime(2026, 9, 1, tzinfo=timezone.utc) schedule = ReportingScheduleSpec("PT1H", "PT1H", period_anchor=start) period = derive_period(schedule, account_timezone="UTC", ordinal=0) @@ -210,6 +213,59 @@ async def main(): boundaries = await durable.read_materializer_boundaries(caller=scope.principal) assert len(boundaries) == boundaries[0].sequence == boundaries[0].account_sequence == 1 assert durable._materializer_outbox is None + from adcp.reporting.ledger import ReportingRevisionReceiptRecord, receipt_to_wire + from adcp.reporting.receipts import InMemoryReportingReceiptStore, ReportingReceiptHandler + from adcp.server import ToolContext + + receipt_store = InMemoryReportingReceiptStore(notifications=False) + await receipt_store.put_configuration(configuration) + await receipt_store.commit_obligation(obligation) + receipt_binding = replace(binding, reconciliation_mode="consumer_receipt") + await receipt_store.put_destination_binding(receipt_binding) + await receipt_store.commit_revision(revision, rows) + destination = example.development_destination(receipt_binding) + receipt_service = ReportingMaterializerService( + receipt_store, + ReportingDestinationIO(destination.registry, destination.resolver), + destination.writer, + ) + receipt_operation_1 = await receipt_service.run_once() + assert (receipt_operation_1).state == "verified" + snapshot = await receipt_store.read_reconciliation_snapshot(caller=scope.principal) + outcome = next(r for r in snapshot.records if r.kind == "materialization") + verification = outcome.verification + receipt = ReportingRevisionReceiptRecord( + scope, + "installed-receipt-0001", + revision.reporting_revision_id, + outcome.reporting_materialization_id, + "accepted", + verification.verification_profile, + verification.row_count, + verification.control_totals, + datetime.now(timezone.utc), + observed_canonical_content_digest=verification.canonical_content_digest, + ) + request = { + "adcp_version": "3.2-rc.6", + "account": {"account_id": configuration.account_id}, + "idempotency_key": "installed-batch-0001", + "receipts": [receipt_to_wire(receipt)], + } + + async def authorize(reference, context, consumer): + assert reference == {"account_id": configuration.account_id} + assert consumer == scope.consumer_id + return configuration.account_id + + handler = ReportingReceiptHandler(receipt_store, resolve_account=authorize) + context = ToolContext(caller_identity=scope.consumer_id) + recorded = await handler.sync_reporting_receipts(request, context) + assert recorded["results"][0]["result"] == "recorded" + receipt_operation_2 = await handler.sync_reporting_receipts(request, context) + assert receipt_operation_2 == recorded + assert len(await receipt_store.read_receipt_boundaries(caller=scope.principal)) == 1 + assert receipt_store._materializer_outbox is None workspace = Path(config["workspace"]).resolve() assert all(not Path(path).resolve().is_relative_to(workspace) for path in sys.path) assert all( @@ -227,6 +283,7 @@ async def main(): "installed": True, "assets": actual, "durable": True, + "receipts": True, } ) ) diff --git a/tests/conformance/reporting/_materializer_process.py b/tests/conformance/reporting/_materializer_process.py index a6267a225..6574bf24b 100644 --- a/tests/conformance/reporting/_materializer_process.py +++ b/tests/conformance/reporting/_materializer_process.py @@ -6,6 +6,7 @@ import json import os import sys +from dataclasses import replace from pathlib import Path from uuid import uuid4 @@ -114,6 +115,18 @@ async def finish_materialization(self, *args, **kwargs): return result verifier = reference_verifier() + if settings.get("legacy_definition"): + verifier = replace( + verifier, + key=replace( + verifier.key, + definition=replace( + verifier.key.definition, + monetary_metric_units=(), + monetary_control_total_units=(), + ), + ), + ) registry = ReportingRevisionVerifierRegistry((verifier,)) writer = ReferenceReportingDestinationWriter((verifier.key.capability,)) reference = ReferenceReportingResolver(writer, registry, (decode_record(settings["binding"]),)) diff --git a/tests/conformance/reporting/_receipt_frozen.py b/tests/conformance/reporting/_receipt_frozen.py new file mode 100644 index 000000000..fbd82911c --- /dev/null +++ b/tests/conformance/reporting/_receipt_frozen.py @@ -0,0 +1,285 @@ +"""Installed historical reader/writer probe, copied out and run with python -I.""" + +import asyncio +import hashlib +import importlib +import json +import sys +import traceback +from dataclasses import replace +from datetime import datetime, timezone +from importlib.resources import files +from pathlib import Path + + +async def main(settings): + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.ledger as ledger + + workspace = Path(settings["workspace"]).resolve() + origins = {} + for name, expected in settings["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + assert not path.is_relative_to(workspace) and "site-packages" in str(path) + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=2, open=False + ) as pool: + if settings["artifact"] == "b21": + from adcp.reporting.materializer import PgReportingMaterializerStore + + store_type = PgReportingMaterializerStore + else: + store_type = getattr( + ledger, "PgReportingReconciliationStore", ledger.PgReportingLedgerStore + ) + options = ( + {"notifications": settings["notifications"]} + if settings["artifact"] not in {"beta15", "records", "integration"} + else {} + ) + store = store_type(pool=pool, **options) + if settings["action"] == "install": + assert settings["artifact"] == "b21" + await store.create_schema() + # C's projection is an independently installed optional component. + # Use this actual parent binary's reviewed Core composition when + # preparing the rolling baseline; the materializer migration does + # not implicitly install or activate C's projector. + from adcp.reporting.outbox import PgStatusNotificationStore + + await PgStatusNotificationStore( + ledger.PgReportingReconciliationStore(pool=pool, notifications=True) + ).create_schema() + assert await store.materializer_ready() + manifest = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() + ) + assert len(manifest) == 187 + return {"installed": True, "manifest_objects": len(manifest), "origins": origins} + account, consumer = settings["account"], settings["consumer"] + configs = await store.list_configurations(account_id=account) + assert len(configs) == 1 + obligations = await store.get_obligation( + account_id=account, reporting_obligation_id=settings["obligation"] + ) + assert obligations is not None + revisions = await store.list_revisions( + account_id=account, reporting_obligation_id=settings["obligation"] + ) + assert len(revisions) == 1 and revisions[0].finality == "official" + status, projector_turns = None, 0 + if settings["notifications"] and settings["artifact"] in {"c", "b1", "b21"}: + from adcp.reporting.outbox import PgStatusNotificationStore + + status = PgStatusNotificationStore( + ledger.PgReportingReconciliationStore(pool=pool, notifications=True) + ) + await status.baseline(account_id=account) + + async def project_ordinary_status(): + nonlocal projector_turns + if status is not None: + while (await status.project_one(account_id=account)).did_work: + projector_turns += 1 + assert projector_turns < 32 + + # All artifacts execute their own default-off ordinary Core read/write + # path on the exact parent before and after the additive migration. + await store.put_configuration( + replace(configs[0], status_retention_days=configs[0].status_retention_days + 1) + ) + for readable in (False, True): + await store.set_revision_readable( + account_id=account, + reporting_revision_id=revisions[0].reporting_revision_id, + readable=readable, + ) + await project_ordinary_status() + receipt_count, ordinary_materializer = 0, False + if hasattr(ledger, "ReportingDeliveryPrincipal"): + caller = ledger.ReportingDeliveryPrincipal(account, consumer) + snapshot = await store.read_reconciliation_snapshot(caller=caller) + available = [ + r for r in snapshot.records if r.kind == "materialization" and r.status != "failed" + ] + assert len(available) == 1 + receipts = [ + r for r in snapshot.records if r.kind in {"revision_receipt", "adjustment_receipt"} + ] + receipt_count = len(receipts) + assert receipt_count == settings["receipt_count"] + if receipt_count: + assert {r.kind for r in receipts} == {"revision_receipt", "adjustment_receipt"} + for receipt in receipts: + assert (await store.get_receipt(receipt.key)) == receipt + assert receipt.status == "accepted" and receipt.received_at is not None + # Public immutable terminal N+1 compatibility is deliberately + # separate from autonomous retry allocation. This actual old writer + # is run in isolation, never alongside an autonomous worker. + attempts = [r for r in snapshot.records if r.kind == "materialization_attempt"] + now = datetime.now(timezone.utc) + attempt = replace( + attempts[0], + reporting_materialization_id=f"frozen-{settings['phase']}", + attempt=len(attempts) + 1, + created_at=now, + ) + await store.commit_materialization_attempt(attempt) + outcome = replace( + available[0], + reporting_materialization_id=attempt.reporting_materialization_id, + status="failed", + resource=None, + verification=None, + failure_code="WRITE_FAILED", + completed_at=now, + ) + receipt_operation_1 = await store.commit_materialization(outcome) + assert (receipt_operation_1)[1] + receipt_operation_2 = await store.commit_materialization(outcome) + assert not (receipt_operation_2)[1] + read = await store.get_materialization(outcome.key) + assert read is not None and read.outcome == outcome + ordinary_materializer = True + await project_ordinary_status() + readiness = None + if settings["artifact"] in {"a", "b", "c", "b1", "b21"}: + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox._schema import validate_schema + + async with pool.connection() as connection: + try: + await validate_schema(connection) + readiness = True + except ReportingNotificationError: + readiness = False + assert readiness == (settings["artifact"] != "a") + materializer = None + if settings["artifact"] == "b21": + assert await store.materializer_ready() + boundaries = await store.read_materializer_boundaries(caller=caller) + assert len(boundaries) == 1 + assert boundaries[0].to_storage()["version"] == 1 + # The approved boundary decoder is closed: epoch is stored on its + # referenced work/event rows, not inside the captured input blob. + async with pool.connection() as connection: + epoch = await ( + await connection.execute( + "SELECT admission_epoch,state FROM reporting_materializer_work" + " WHERE account_id=%s AND consumer_id=%s" + " AND reporting_materialization_id=%s", + (account, consumer, boundaries[0].reporting_materialization_id), + ) + ).fetchone() + assert epoch == (0, "acked") + materializer = len( + json.loads( + files("adcp.reporting.materializer") + .joinpath("required_schema.json") + .read_text() + ) + ) + assert materializer == 187 + workers = {} + if settings["notifications"]: + from adcp.reporting.ledger.notification_models import decode_event + from adcp.reporting.outbox import ( + PgReportingOutbox, + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ) + + class EmptySubscriptions: + async def list_active(self, **kwargs): + assert kwargs["notification_type"] != "reporting.delivery_ready" + return () + + async def get_active(self, **kwargs): + raise AssertionError("empty membership has no HTTP delivery") + + outboxes = {"ordinary": PgReportingOutbox(pool=pool)} + if status is not None: + outboxes["status"] = status.outbox + for name, outbox in outboxes.items(): + events = await outbox.list_events(account_id=account) + assert events and all( + event.notification_type != "reporting.delivery_ready" for event in events + ) + identities = { + (e.account_id, e.consumer_namespace, e.notification_id): e for e in events + } + seen = set() + + class ObservedOutbox: + def __getattr__(self, attribute): + return getattr(outbox, attribute) + + async def claim_expansion(self, **kwargs): + lease = await outbox.claim_expansion(**kwargs) + if lease is not None: + identity = ( + lease.account_id, + lease.consumer_namespace, + lease.notification_id, + ) + assert identity in identities and identity not in seen + assert decode_event(lease.event) == identities[identity] + seen.add(identity) + return lease + + worker = ReportingNotificationWorker( + outbox=ObservedOutbox(), + subscriptions=EmptySubscriptions(), + cipher=ReportingEnvelopeCipher(b"e" * 32), + ) + while await worker.expand_one(account_id=account): + assert len(seen) <= len(events) + receipt_operation_3 = await worker.deliver_one(account_id=account) + assert not receipt_operation_3 + assert await outbox.list_events(account_id=account) == events + workers[name] = len(seen) + if settings["phase"] == "before": + assert workers["ordinary"] > 0 # An actual old Core worker claimed real work. + if status is not None: + assert projector_turns > 0 and workers["status"] > 0 + return { + "ordinary_core": True, + "ordinary_materializer": ordinary_materializer, + "receipt_count": receipt_count, + "notification_readiness": readiness, + "materializer_manifest": materializer, + "projector_turns": projector_turns, + "workers": workers, + "origins": origins, + "sha": settings["sha"], + "artifact": settings["artifact"], + } + + +if __name__ == "__main__": + try: + result = asyncio.run(main(json.load(sys.stdin))) + except Exception as error: + result = { + "failure": type(error).__name__, + "safe_code": ( + error.code + if type(error).__module__ == "adcp.reporting.ledger.notification_models" + and type(error).__name__ == "ReportingNotificationError" + else None + ), + "frames": [ + [Path(frame.filename).name, frame.lineno] + for frame in traceback.extract_tb(error.__traceback__) + ], + } + print(json.dumps(result)) diff --git a/tests/conformance/reporting/_receipt_installed.py b/tests/conformance/reporting/_receipt_installed.py new file mode 100644 index 000000000..ef9381a26 --- /dev/null +++ b/tests/conformance/reporting/_receipt_installed.py @@ -0,0 +1,83 @@ +"""Installed Python 3.10 SQL and authenticated HTTP replay, run outside the checkout.""" + +import asyncio +import hashlib +import importlib +import importlib.util +import json +import sys +import traceback +from pathlib import Path +from types import SimpleNamespace + + +async def main(settings): + from psycopg_pool import AsyncConnectionPool + + from adcp.reporting.receipts import PgReportingReceiptStore, ReportingReceiptError + + installed = settings["installed"] + workspace = Path(installed["workspace"]).resolve() + assert list(sys.version_info[:2]) == installed["python"] + origins = {} + for name, expected in installed["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=1, open=False + ) as pool: + store = PgReportingReceiptStore(pool=pool, notifications=settings["notifications"]) + if settings["action"] == "install": + try: + await store.receipt_ingestion_ready() + except ReportingReceiptError as error: + assert error.code == "RECEIPT_SCHEMA_UNREADY" + else: + raise AssertionError("empty schema must fail closed") + await store.create_schema() + await store.create_schema() + assert await store.receipt_ingestion_ready() + result = {"installed": True} + else: + spec = importlib.util.spec_from_file_location("installed_transport", settings["helper"]) + transport = importlib.util.module_from_spec(spec) + spec.loader.exec_module(transport) + mount = transport.MountedReceipts(SimpleNamespace(store=store), hydrated=True) + caller = settings["caller"] + scenario = SimpleNamespace( + obligation=SimpleNamespace(account_id=caller["account_id"]), + binding=SimpleNamespace(consumer_id=caller["consumer_id"]), + ) + mount.authorize(scenario) + async with mount.client() as client: + _, result = await mount.mcp(client, settings["request"]) + assert [r["result"] for r in result["results"]] == ["recorded", "recorded"] + receipt_operation_1 = await mount.a2a(client, settings["request"]) + assert (receipt_operation_1)[1] == result + mount.grants.clear() + for call in (mount.mcp, mount.a2a): + _, denied = await call(client, settings["request"]) + assert transport.error_code(denied) == "UNAUTHORIZED" + assert len(mount.auth_calls) == 4 + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + return {"result": result, "origins": origins, "python": installed["python"]} + + +if __name__ == "__main__": + try: + result = asyncio.run(main(json.load(sys.stdin))) + except Exception as error: + result = { + "failure": type(error).__name__, + "frames": [ + [Path(frame.filename).name, frame.lineno] + for frame in traceback.extract_tb(error.__traceback__) + ], + } + print(json.dumps(result)) diff --git a/tests/conformance/reporting/_receipt_process.py b/tests/conformance/reporting/_receipt_process.py new file mode 100644 index 000000000..7a336fe11 --- /dev/null +++ b/tests/conformance/reporting/_receipt_process.py @@ -0,0 +1,81 @@ +"""Real receipt worker. Pause while holding an ordinal transaction, then SIGKILL.""" + +import asyncio +import hashlib +import importlib +import json +import sys +from pathlib import Path + + +async def main(): + settings = json.loads(await asyncio.to_thread(sys.stdin.readline)) + from psycopg import AsyncConnection + from psycopg_pool import AsyncConnectionPool + + from adcp.reporting.ledger import ReportingDeliveryPrincipal + from adcp.reporting.receipts import PgReportingReceiptStore + + origins = {} + if settings.get("installed"): + installed = settings["installed"] + workspace = Path(installed["workspace"]).resolve() + assert list(sys.version_info[:2]) == installed["python"] + for name, expected in installed["modules"].items(): + module = importlib.import_module(name) + path = Path(module.__file__).resolve() + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert "site-packages" in module.__file__ + assert not Path(module.__file__).resolve().is_relative_to(workspace) + + async def pause(point): + if settings.get("pause") == point: + print(json.dumps({"point": point}), flush=True) + receipt_operation_1 = await asyncio.to_thread(sys.stdin.readline) + assert json.loads(receipt_operation_1)["continue"] + + class Connection(AsyncConnection): + async def execute(self, query, params=None, **kwargs): + result = await super().execute(query, params, **kwargs) + if isinstance(query, str): + for prefix, point in ( + ("INSERT INTO reporting_receipt_ingestion_batches", "header"), + ("INSERT INTO reporting_reconciliation_records", "receipt"), + ("INSERT INTO reporting_reconciliation_changes", "feed"), + ("INSERT INTO reporting_receipt_ingestion_boundaries", "capture"), + ("INSERT INTO reporting_receipt_ingestion_results", "ordinal"), + ("UPDATE reporting_receipt_ingestion_batches", "final"), + ): + if query.startswith(prefix): + await pause(point) + return result + + class Store(PgReportingReceiptStore): + async def _receipt_results_on(self, connection, caller, batch): + result = await super()._receipt_results_on(connection, caller, batch) + if len(result) == 1: + await pause("between_ordinals") + return result + + async with AsyncConnectionPool( + settings["conninfo"], + kwargs=settings["kwargs"], + min_size=1, + max_size=1, + connection_class=Connection, + open=False, + ) as pool: + store = Store(pool=pool, notifications=settings["notifications"]) + result = await store.ingest_receipt_batch( + settings["request"], caller=ReportingDeliveryPrincipal(**settings["caller"]) + ) + await pause("committed") + print(json.dumps({"point": "done", "result": result, "origins": origins}), flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_receipt_support.py b/tests/conformance/reporting/_receipt_support.py new file mode 100644 index 000000000..25b82b946 --- /dev/null +++ b/tests/conformance/reporting/_receipt_support.py @@ -0,0 +1,271 @@ +"""Shared receipt-ingress vectors; production PG receipt time is database time.""" + +from contextlib import asynccontextmanager +from dataclasses import dataclass, replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingDeliveryScope, +) +from adcp.reporting.ledger.delivery import adjustment_to_wire, receipt_to_wire +from adcp.reporting.ledger.models import derive_period +from adcp.reporting.ledger.producer import revision_content_sha256 +from adcp.reporting.receipts import InMemoryReportingReceiptStore + +from ._durable_materializer_support import DurableHarness +from ._generation_support import END, configuration, isolated_reporting_pool +from ._reconciliation_support import Clock, scenario + + +@asynccontextmanager +async def receipt_harness(backend, *, notifications=False, pool=None): + clock = Clock() + if backend == "memory": + yield DurableHarness( + InMemoryReportingReceiptStore(clock=clock, notifications=notifications), clock + ) + return + from adcp.reporting.receipts import PgReportingReceiptStore + + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingReceiptStore(pool=pool, clock=clock, notifications=notifications) + await store.create_schema() + yield DurableHarness(store, clock, pool) + + +@pytest.fixture( + params=[("memory", False), ("memory", True), ("postgres", False), ("postgres", True)] +) +async def receipts(request): + backend, notifications = request.param + async with receipt_harness(backend, notifications=notifications) as h: + yield h + + +async def receipt_case(h, **kwargs): + s = await scenario(h.store, **kwargs) + await h.store.commit_materialization(s.outcome) + return s + + +def request_for(s, *, key="receipt-batch-0001", **changes): + return { + "adcp_version": "3.2-rc.6", + "account": {"account_id": s.obligation.account_id}, + "idempotency_key": key, + "receipts": [receipt_to_wire(s.receipt)], + **changes, + } + + +async def adjustment_for(h, s, **changes): + adjustment = ReportingAdjustmentRecord( + "adjustment-1", + s.obligation.account_id, + s.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1.50"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "EUR"), + ), + ) + adjustment = replace(adjustment, **changes) + await h.store.commit_adjustment(adjustment) + return { + "reporting_receipt_id": "adjustment-receipt-0001", + "reporting_adjustment_id": adjustment.reporting_adjustment_id, + "adjusts_reporting_revision_id": adjustment.adjusts_reporting_revision_id, + "status": "accepted", + "observed_adjustment_sha256": adjustment_to_wire(adjustment)["canonical_adjustment_sha256"], + "observed_at": (END + timedelta(seconds=7)).isoformat().replace("+00:00", "Z"), + } + + +async def batch_state(h): + if h.pool is None: + return tuple( + (len(s.results), s.response is not None) + for s in getattr(h.store, "_receipt_batches", {}).values() + ) + async with h.pool.connection() as c: + return tuple( + await ( + await c.execute( + "SELECT (SELECT count(*) FROM reporting_receipt_ingestion_results r" + " WHERE (r.account_id,r.consumer_id,r.idempotency_key)=" + "(b.account_id,b.consumer_id,b.idempotency_key))," + " final_response IS NOT NULL FROM reporting_receipt_ingestion_batches b" + " ORDER BY account_id,consumer_id,idempotency_key" + ) + ).fetchall() + ) + + +@dataclass(frozen=True) +class ForeignTargets: + """Existing, wholly VALID records that belong to a different exact target. + + Nothing here is corrupt or unauthorized at the application boundary: every + row is a legitimate artifact for its own obligation, revision, consumer or + adjustment. A receipt that points at one of them from another target must + still be refused, and refused indistinguishably from an absent record. + """ + + obligation: object + revision: object + materialization_id: str + adjustment: object + consumer_id: str + consumer_materialization_id: str + + +async def foreign_targets(h, s): + """Commit one more complete valid period and one more valid consumer.""" + config = replace( + configuration(s.obligation.account_id), + feed_purpose=s.binding.feed_purpose, + required_finality=s.obligation.required_finality, + ) + period = derive_period(config.schedule, account_timezone=config.account_timezone, ordinal=1) + obligation = replace( + s.obligation, + reporting_obligation_id="rpo_foreign", + period=period, + scope_resolved_at=period.end, + automated_recovery_deadline_at=period.expected_at + config.automated_recovery_window, + ) + await h.store.commit_obligation(obligation) + scope = ReportingDeliveryScope( + obligation.generation_key, s.binding.consumer_id, obligation.reporting_obligation_id + ) + await h.store.bind_obligation_delivery( + replace(s.delivery, scope=scope, resource_retained_until=period.end + timedelta(days=400)) + ) + rows = [ + { + "media_buy_id": obligation.media_buy_ids[0], + "impressions": 5, + "spend": "12.50", + "currency": "EUR", + } + ] + revision = replace( + s.revision, + reporting_revision_id="revision-foreign", + reporting_obligation_id=obligation.reporting_obligation_id, + observed_at=period.end, + data_through=period.end, + created_at=period.end + timedelta(seconds=1), + finalized_at=period.end if s.revision.finality == "official" else None, + revision_content_sha256=revision_content_sha256( + reporting_revision_id="revision-foreign", + row_count=s.revision.row_count, + control_totals=s.revision.control_totals, + reporting_rows=rows, + control_total_evidence=s.revision.managed_control_totals, + ), + ) + await h.store.commit_revision(revision, rows) + completed = period.end + timedelta(seconds=3) + await h.store.commit_materialization_attempt( + replace( + s.attempt, + scope=scope, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id="materialization-foreign", + created_at=period.end + timedelta(seconds=2), + ) + ) + await h.store.commit_materialization( + replace( + s.outcome, + scope=scope, + reporting_revision_id=revision.reporting_revision_id, + reporting_materialization_id="materialization-foreign", + completed_at=completed, + resource=replace(s.outcome.resource, expires_at=completed + timedelta(days=400)), + verification=replace(s.outcome.verification, verified_at=completed), + ) + ) + adjustment = ReportingAdjustmentRecord( + "adjustment-foreign", + obligation.account_id, + revision.reporting_revision_id, + "source_correction", + period.end, + period.end + timedelta(days=30), + (("spend", "-1.50"),), + period.end + timedelta(seconds=5), + period.end + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "EUR"), + ), + ) + await h.store.commit_adjustment(adjustment) + # One more valid consumer holding its own artifact for the caller's revision. + consumer_id = f"{s.binding.consumer_id}-foreign" + other = ReportingDeliveryScope( + s.obligation.generation_key, consumer_id, s.obligation.reporting_obligation_id + ) + await h.store.put_destination_binding(replace(s.binding, consumer_id=consumer_id)) + await h.store.bind_obligation_delivery(replace(s.delivery, scope=other)) + await h.store.commit_materialization_attempt( + replace(s.attempt, scope=other, reporting_materialization_id="materialization-consumer") + ) + await h.store.commit_materialization( + replace(s.outcome, scope=other, reporting_materialization_id="materialization-consumer") + ) + return ForeignTargets( + obligation, + revision, + "materialization-foreign", + adjustment, + consumer_id, + "materialization-consumer", + ) + + +async def extra_materialization(h, s, materialization_id, attempt_number): + """Commit one more valid attempt/materialization for s's exact scope.""" + created = s.attempt.created_at + timedelta(seconds=attempt_number) + completed = s.outcome.completed_at + timedelta(seconds=attempt_number) + await h.store.commit_materialization_attempt( + replace( + s.attempt, + reporting_materialization_id=materialization_id, + attempt=attempt_number, + created_at=created, + ) + ) + await h.store.commit_materialization( + replace( + s.outcome, + reporting_materialization_id=materialization_id, + completed_at=completed, + resource=replace(s.outcome.resource, expires_at=completed + timedelta(days=400)), + verification=replace(s.outcome.verification, verified_at=completed), + ) + ) + return materialization_id + + +async def foreign_account(h, *, account_id="acct_b"): + """Commit one more COMPLETE, VALID account and return its exact identifiers. + + Obligations, revisions and adjustments are globally keyed, so the same + identifier cannot exist under two accounts. The account relationship can + therefore only be probed by naming another account's real identifiers from + this account's receipt, which is what the returned values are for. + """ + other = await receipt_case(h, account_id=account_id) + adjustment = await adjustment_for(h, other, reporting_adjustment_id=f"adjustment-{account_id}") + materialization = await extra_materialization(h, other, f"materialization-{account_id}", 2) + return other, adjustment, materialization diff --git a/tests/conformance/reporting/_receipt_transport.py b/tests/conformance/reporting/_receipt_transport.py new file mode 100644 index 000000000..1b25cb4f7 --- /dev/null +++ b/tests/conformance/reporting/_receipt_transport.py @@ -0,0 +1,270 @@ +"""Real unified production mounts, with deterministic trusted auth adapters.""" + +import json +from contextlib import asynccontextmanager + +import httpx +from asgi_lifespan import LifespanManager + +from adcp.decisioning import Account, AuthInfo, RequestContext +from adcp.decisioning.registry import ( + ApiKeyCredential, + BuyerAgent, + HttpSigCredential, + OAuthCredential, +) +from adcp.reporting.receipts import ReportingReceiptError, ReportingReceiptHandler +from adcp.server.auth import BearerTokenAuth, Principal, auth_context_factory +from adcp.server.idempotency import IdempotencyStore, MemoryBackend +from adcp.server.serve import _build_mcp_and_a2a_app + + +class Registry: + def __init__(self): + self.agents = {} + self.calls = [] + + async def resolve_by_agent_url(self, agent_url): + self.calls.append(("signed", agent_url)) + return self.agents.get(agent_url) + + async def resolve_by_credential(self, credential): + key = ( + credential.key_id if isinstance(credential, ApiKeyCredential) else credential.client_id + ) + self.calls.append((credential.kind, key)) + return self.agents.get(key) + + +class ForbiddenGenericCache(MemoryBackend): + async def get(self, *args, **kwargs): + raise AssertionError("receipt ingress reached the generic idempotency cache") + + +class MountedReceipts: + def __init__(self, h, *, hydrated=False, registry_kind=None, version=None): + self.h = h + self.hydrated = hydrated + self.registry_kind = registry_kind + self.tokens = {} + self.grants = set() + self.accounts = {} + self.auth_calls = [] + self.contexts = [] + self.registry = Registry() if registry_kind is not None else None + self.sessions = {} + self.counter = 0 + self.idempotency = IdempotencyStore(backend=ForbiddenGenericCache()) + self.handler = ReportingReceiptHandler( + h.store, resolve_account=self.resolve_account, buyer_agents=self.registry + ) + # Exercise both the direct method decorator and the common generic + # middleware whose wrapped function is named execute, not the task. + self.handler.sync_reporting_receipts = self.idempotency.wrap( + self.handler.sync_reporting_receipts + ) + if version is not None: + self.handler.adcp_version = version + + def authorize(self, s, *, token="token-one"): + account, consumer = s.obligation.account_id, s.binding.consumer_id + self.tokens[token] = Principal( + caller_identity=consumer, + tenant_id="one-shared-tenant", + metadata={"account_id": account, "credential_id": token}, + ) + self.grants.add((account, consumer)) + self.accounts[account] = account + if self.registry is not None: + agent = BuyerAgent(consumer, "Fixture buyer", "active") + self.registry.agents[token] = self.registry.agents[consumer] = agent + + async def resolve_account(self, reference, context, consumer): + self.auth_calls.append((dict(reference), consumer)) + account = self.accounts.get(reference.get("account_id")) + if (account, consumer) not in self.grants: + raise ReportingReceiptError("UNAUTHORIZED") + return account + + def context(self, meta): + raw = auth_context_factory(meta) + auth = raw.metadata.get("adcp.auth_info") + agent = None + if self.registry_kind is not None and raw.caller_identity is not None: + key = raw.metadata["credential_id"] + if self.registry_kind == "api_key": + credential = ApiKeyCredential("api_key", key) + elif self.registry_kind == "oauth": + credential = OAuthCredential("oauth", key, ("reporting:write",)) + else: + credential = HttpSigCredential("http_sig", key, raw.caller_identity, 1.0) + # This is the adapter output of trusted verification, never an + # inbound body field. API/OAuth consumer resolution is registry-owned. + auth = AuthInfo(kind=self.registry_kind, credential=credential) + raw.metadata["adcp.auth_info"] = auth + agent = BuyerAgent(raw.caller_identity, "Fixture buyer", "active") + if self.hydrated: + context = RequestContext( + account=Account(id=raw.metadata.get("account_id", "unresolved")), + caller_identity="opaque:AccountStore:cache:key:identical-for-both-consumers", + tenant_id=raw.tenant_id, + metadata=raw.metadata, + auth_info=auth, + buyer_agent=agent, + auth_principal=raw.caller_identity, + ) + else: + context = raw + self.contexts.append(context) + return context + + async def middleware(self, name, params, context, call_next): + async def execute(params, context): + return await call_next() + + return await self.idempotency.wrap(execute)(params, context) + + @asynccontextmanager + async def client(self, **options): + app = _build_mcp_and_a2a_app( + self.handler, + name="receipt-ingress", + port=3001, + host="127.0.0.1", + instructions=None, + test_controller=None, + context_factory=self.context, + middleware=[self.middleware], + allowed_hosts=["localhost"], + auth=BearerTokenAuth(validate_token=lambda token: self.tokens.get(token)), + **options, + ) + async with LifespanManager(app): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://localhost" + ) as client: + yield client + + async def mcp( + self, client, request=None, *, token="token-one", inventory=False, mutate_wire=None + ): + headers = { + "accept": "application/json, text/event-stream", + "authorization": f"Bearer {token}", + } + if token not in self.sessions: + initial = await client.post( + "/mcp/", + headers=headers, + json={ + "jsonrpc": "2.0", + "id": 0, + "method": "initialize", + "params": { + "protocolVersion": "2025-06-18", + "capabilities": {}, + "clientInfo": {"name": "receipt-test", "version": "1"}, + }, + }, + ) + assert initial.status_code == 200, initial.text + self.sessions[token] = initial.headers.get("mcp-session-id") + if self.sessions[token] is not None: + headers["mcp-session-id"] = self.sessions[token] + self.counter += 1 + wire = json.dumps( + { + "jsonrpc": "2.0", + "id": self.counter, + "method": "tools/list" if inventory else "tools/call", + "params": ( + {} if inventory else {"name": "sync_reporting_receipts", "arguments": request} + ), + } + ) + response = await client.post( + "/mcp/", + headers={**headers, "content-type": "application/json"}, + content=mutate_wire(wire) if mutate_wire is not None else wire, + ) + if response.status_code != 200: + try: + payload = response.json() + except ValueError: + payload = {"unstructured_transport_error": True} + return response.status_code, payload + payload = next( + ( + json.loads(line[6:]) + for line in response.text.splitlines() + if line.startswith("data: ") + ), + None, + ) + if payload is None: + payload = response.json() + if inventory: + return 200, payload["result"] + result = payload.get("result", payload) + if "structuredContent" in result: + return 200, result["structuredContent"] + for part in result.get("content", []): + if part.get("type") == "text": + return 200, json.loads(part["text"]) + return 200, result + + async def a2a(self, client, request, *, token="token-one", mutate_wire=None, v1=False): + self.counter += 1 + envelope = { + "jsonrpc": "2.0", + "id": str(self.counter), + "method": "SendMessage" if v1 else "message/send", + "params": { + "message": { + "messageId": f"message-{self.counter}", + "role": "user", + "parts": [ + { + "kind": "data", + "data": {"skill": "sync_reporting_receipts", "parameters": request}, + } + ], + } + }, + } + if v1: + del envelope["params"]["message"]["parts"][0]["kind"] + envelope["params"]["message"]["role"] = "ROLE_USER" + wire = json.dumps(envelope) + response = await client.post( + "/", + headers={ + "authorization": f"Bearer {token}", + "content-type": "application/json", + "A2A-Version": "1.0" if v1 else "0.3", + }, + content=mutate_wire(wire) if mutate_wire is not None else wire, + ) + if response.status_code != 200: + try: + payload = response.json() + except ValueError: + payload = {"unstructured_transport_error": True} + return response.status_code, payload + payload = response.json() + result = payload.get("result", payload) + if "task" in result: + result = result["task"] + for artifact in result.get("artifacts", []): + for part in artifact.get("parts", []): + if "data" in part: + return 200, part["data"] + return 200, result + + +def error_code(payload): + if "adcp_error" in payload: + return payload["adcp_error"]["code"] + if "errors" in payload: + return payload["errors"][0]["code"] + raise AssertionError(payload) diff --git a/tests/conformance/reporting/test_reporting_generation_identity.py b/tests/conformance/reporting/test_reporting_generation_identity.py index 644bcffbe..d55480681 100644 --- a/tests/conformance/reporting/test_reporting_generation_identity.py +++ b/tests/conformance/reporting/test_reporting_generation_identity.py @@ -26,6 +26,7 @@ consumer_mismatch_issue_key, ) from adcp.reporting.ledger.pg import PgReportingLedgerStore +from adcp.reporting.outbox._schema import schema_objects from adcp.reporting.source import ReportingSourceSliceRequestV1 from tests.conformance.reporting._generation_support import ( END, @@ -144,7 +145,10 @@ async def test_concurrent_leases_and_releases_keep_accounts_separate( ) replacement = await store.lease_period_close(worker_id="next", now=NOW, lease_seconds=60) assert replacement is not None and replacement.generation_key == first.generation_key - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_1 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_1 is None await asyncio.gather( store.release_period_close(second, worker_id="shared"), store.release_period_close(replacement, worker_id="next"), @@ -169,11 +173,17 @@ async def test_an_expired_lease_cannot_release_a_replacement_with_the_same_worke replacement = await store.lease_period_close(worker_id="shared", now=later, lease_seconds=60) assert replacement is not None and replacement.generation_key == expired.generation_key await store.release_period_close(expired, worker_id="shared") - assert await store.lease_period_close(worker_id="extra", now=later, lease_seconds=60) is None + receipt_operation_2 = await store.lease_period_close( + worker_id="extra", now=later, lease_seconds=60 + ) + assert receipt_operation_2 is None await store.release_period_close(replacement, worker_id="shared") reclaimed = await store.lease_period_close(worker_id="new", now=later, lease_seconds=60) assert reclaimed is not None and reclaimed.generation_key == expired.generation_key - assert await store.lease_period_close(worker_id="extra", now=later, lease_seconds=60) is None + receipt_operation_3 = await store.lease_period_close( + worker_id="extra", now=later, lease_seconds=60 + ) + assert receipt_operation_3 is None async def test_a_worker_that_releases_each_turn_reaches_every_accounts_generation( @@ -185,6 +195,13 @@ async def test_a_worker_that_releases_each_turn_reaches_every_accounts_generatio always hands back the first leasable generation would close periods for one account forever and never reach the others -- invisible until two accounts share a ``delivery_config_id``, which is exactly what this change allows. + + Release clears ``lease_expires_at``, so ``ORDER BY lease_expires_at NULLS + FIRST`` alone is not a total order and the winner is whatever plan order + happens to apply; CI caught nine consecutive ``acct_a`` leases this way + while the in-memory store, which breaks the tie on turn, stayed fair. Both + stores now rank on a persisted turn advanced at acquisition, so reverting + that ordering fails this test deterministically rather than occasionally. """ accounts = ("acct_a", "acct_b", "acct_c") await asyncio.gather(*(store.put_configuration(configuration(name)) for name in accounts)) @@ -199,6 +216,278 @@ async def test_a_worker_that_releases_each_turn_reaches_every_accounts_generatio assert set(worked[: len(accounts)]) == set(accounts) +async def test_lease_turn_advances_on_acquisition_so_a_crashed_worker_cannot_starve_peers( + store: ReportingLedgerStore, +) -> None: + """Acquisition, not release, is where a generation loses its place in line. + + ``release_period_close`` only clears the lease, so the turn has to be + recorded when the lease is taken -- otherwise a worker that crashes + mid-close keeps the minimum turn forever and every expiry sweep hands the + same generation back. + + Ordering by expiry with ``NULLS FIRST`` hides that on its own: a never + worked generation always outranks an expired one, so the turn is only the + deciding term once two eligible generations share an expiry. This builds + exactly that state. ``acct_b`` is leased first and ``acct_a`` second, both + crash unreleased with the same expiry, so the generation that went longest + without a turn is ``acct_b`` even though ``acct_a`` sorts lower by key. A + store that advances the turn only on release ranks them equal and reaches + for ``acct_a``. + """ + # Created in this order so the later-acquired generation sorts lower by key. + await store.put_configuration(configuration("acct_b")) + crashed_first = await store.lease_period_close(worker_id="crash-1", now=NOW, lease_seconds=30) + assert crashed_first is not None and crashed_first.account_id == "acct_b" + await store.put_configuration(configuration("acct_a")) + crashed_second = await store.lease_period_close(worker_id="crash-2", now=NOW, lease_seconds=30) + assert crashed_second is not None and crashed_second.account_id == "acct_a" + assert crashed_first.lease_expires_at == crashed_second.lease_expires_at + # Never worked, so it must outrank both expired generations. + await store.put_configuration(configuration("acct_c")) + later = NOW + timedelta(seconds=31) + swept = [] + for _ in range(3): + lease = await store.lease_period_close(worker_id="solo", now=later, lease_seconds=30) + assert lease is not None + swept.append(lease.account_id) + assert swept == ["acct_c", "acct_b", "acct_a"] + + +async def test_lease_order_is_total_and_does_not_depend_on_acceptance_order( + store: ReportingLedgerStore, +) -> None: + """Both backends must choose the same generation, not this process's history. + + Never-leased generations all share the minimum turn, so something has to + break the tie. Ranking on "whichever generation this store accepted first" + is not reproducible: the SQL store cannot express it, it differs from the + in-memory store, and it does not survive a restart or a second worker. The + generation key is part of the rank instead, so these configurations are + handed out in key order even though they are accepted in the reverse. + """ + # Accepted in reverse key order, deliberately. + for name in ("acct_c", "acct_b", "acct_a"): + await store.put_configuration(configuration(name)) + worked = [] + for _ in range(3): + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + worked.append(lease.account_id) + await store.release_period_close(lease, worker_id="solo") + assert worked == ["acct_a", "acct_b", "acct_c"] + + +async def test_a_crashed_generation_is_not_starved_by_a_peer_that_keeps_releasing( + store: ReportingLedgerStore, +) -> None: + """A permanently unheld peer must not outrank a lower-turn expired generation. + + The acquisition filter already drops every live lease, so among the + survivors the expiry carries no fairness information. If unheld sorted + ahead of expired, a peer that is leased and released on every turn would be + NULL forever and win every comparison, while the generation whose worker + died would stay expired and never close another period -- starvation with + no expiry sweep able to clear it. + """ + await asyncio.gather( + *(store.put_configuration(configuration(name)) for name in ("acct_a", "acct_b")) + ) + crashed = await store.lease_period_close(worker_id="crashes", now=NOW, lease_seconds=30) + assert crashed is not None and crashed.account_id == "acct_a" + # acct_a is still live here, so this can only take acct_b; it releases. + released = await store.lease_period_close(worker_id="polite", now=NOW, lease_seconds=30) + assert released is not None and released.account_id == "acct_b" + await store.release_period_close(released, worker_id="polite") + # Past acct_a's expiry both are leasable: acct_b unheld, acct_a expired. + later = NOW + timedelta(seconds=31) + recovered = await store.lease_period_close(worker_id="solo", now=later, lease_seconds=30) + assert recovered is not None + assert recovered.account_id == "acct_a" + + +async def test_lease_fairness_migrates_onto_an_already_installed_older_schema() -> None: + """The additive upgrade reaches an existing install and stays idempotent. + + The rank lives in a private `adcp_`-prefixed table precisely so that + `schema_objects()` -- which enumerates every `reporting_*` table in the + schema -- keeps reporting the exact object set that older binaries + validate. So the upgrade has to be proven on an install that predates it. + """ + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + async with pool.connection() as connection: + # Reduce the install to the pre-fairness shape an older binary left. + await connection.execute( + "DROP TABLE IF EXISTS adcp_reporting_configuration_lease_turns" + ) + await connection.execute( + "DROP SEQUENCE IF EXISTS adcp_reporting_configuration_lease_turn_seq" + ) + for name in ("acct_a", "acct_b"): + await store.put_configuration(configuration(name)) + # Repeated migration is safe and restores the durable fairness rank. + await store.create_schema() + await store.create_schema() + async with pool.connection() as connection: + ranks = await ( + await connection.execute( + "SELECT count(*) FROM adcp_reporting_configuration_lease_turns" + ) + ).fetchone() + # Generations accepted before the upgrade have no rank row at all, which + # is the never-leased rank rather than a privileged one. + assert ranks is not None and ranks[0] == 0 + worked = [] + for _ in range(2): + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + worked.append(lease.account_id) + await store.release_period_close(lease, worker_id="solo") + assert worked == ["acct_a", "acct_b"] + # A generation accepted by an older writer that knows nothing about the + # private table still ranks as never leased, so it is served before the + # generations that already took a turn rather than starved behind them. + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO reporting_configurations" + " (delivery_config_id, delivery_config_version, account_id," + " report_definition_id, reporting_profile, feed_purpose, required_finality," + " account_timezone, schedule, media_buy_ids, activated_at," + " automated_recovery_seconds, status_retention_days, content_sha256)" + " SELECT delivery_config_id, delivery_config_version, 'acct_legacy'," + " report_definition_id, reporting_profile, feed_purpose, required_finality," + " account_timezone, schedule, media_buy_ids, activated_at," + " automated_recovery_seconds, status_retention_days, 'f' || content_sha256" + " FROM reporting_configurations WHERE account_id = %s", + ("acct_a",), + ) + legacy = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert legacy is not None + assert legacy.account_id == "acct_legacy" + + +async def test_lease_fairness_adds_no_enumerated_reporting_catalog_object() -> None: + """Exact `reporting_*` object identity is the A/B+C compatibility contract. + + `schema_objects()` enumerates every current-schema table whose name starts + with `reporting_`, plus that table's columns, constraints, indexes and + triggers, and the status suites compare the installed set to their + manifests exhaustively. The fairness rank must therefore add nothing to + that set -- not a column on `reporting_configurations`, and not a new + `reporting_*` table either. + """ + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + async with pool.connection() as connection: + installed = await schema_objects(connection) + await connection.execute( + "DROP TABLE IF EXISTS adcp_reporting_configuration_lease_turns" + ) + await connection.execute( + "DROP SEQUENCE IF EXISTS adcp_reporting_configuration_lease_turn_seq" + ) + without = await schema_objects(connection) + assert installed == without + assert not [k for k in installed if "lease_turn" in k or "fairness" in k] + # And the private objects really are the ones carrying the rank. + await store.create_schema() + for name in ("acct_a", "acct_b"): + await store.put_configuration(configuration(name)) + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + async with pool.connection() as connection: + assert await schema_objects(connection) == installed + rows = await ( + await connection.execute( + "SELECT account_id, lease_turn" " FROM adcp_reporting_configuration_lease_turns" + ) + ).fetchall() + assert [(r[0], r[1] > 0) for r in rows] == [("acct_a", True)] + + +async def test_lease_and_its_fairness_rank_commit_or_roll_back_together() -> None: + """The rank lives in another table, so it must share the lease transaction. + + If the two statements could commit separately, a crash between them would + either hand out a lease whose generation never lost its place in line, or + advance the rank for a lease nobody holds. The rank update is forced to fail + here; the acquisition must roll back with it. + + The pool is deliberately autocommit: that is the mode in which an implicit + per-block transaction does not exist, so it is the only mode that can + witness the explicit transaction actually doing the work. + """ + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + await store.put_configuration(configuration("acct_a")) + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE adcp_reporting_configuration_lease_turns" + " ADD CONSTRAINT reject_rank CHECK (lease_turn < 0)" + ) + with pytest.raises(Exception): # noqa: B017,PT011 - driver integrity error + await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + async with pool.connection() as connection: + held = await ( + await connection.execute( + "SELECT count(*) FROM reporting_configurations" + " WHERE lease_worker_id IS NOT NULL OR lease_expires_at IS NOT NULL" + ) + ).fetchone() + ranked = await ( + await connection.execute( + "SELECT count(*) FROM adcp_reporting_configuration_lease_turns" + ) + ).fetchone() + # Neither half survived. + assert held is not None and held[0] == 0 + assert ranked is not None and ranked[0] == 0 + # With the rank writable again the generation is still leasable. + async with pool.connection() as connection: + await connection.execute( + "ALTER TABLE adcp_reporting_configuration_lease_turns DROP CONSTRAINT reject_rank" + ) + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None and lease.account_id == "acct_a" + + +async def test_a_stale_fairness_rank_row_cannot_affect_another_generation() -> None: + """An orphan rank row is inert, and a re-put generation keeps its own rank. + + The rank table carries no foreign key, so a generation removed by an + operator can leave a row behind. The lazy join only matches on the exact + generation key, so such a row is never consulted for anyone else, and + ``put_configuration`` is immutable by key, so re-accepting the same + generation is the same generation and legitimately keeps its place in line. + """ + async with isolated_reporting_pool() as pool: + store = PgReportingLedgerStore(pool=pool, clock=lambda: NOW) + await store.create_schema() + for name in ("acct_a", "acct_b"): + await store.put_configuration(configuration(name)) + async with pool.connection() as connection: + await connection.execute( + "INSERT INTO adcp_reporting_configuration_lease_turns" + " (account_id, delivery_config_id, delivery_config_version, lease_turn)" + " VALUES ('acct_vanished', 'daily', 1, 999999)" + ) + worked = [] + for _ in range(2): + lease = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert lease is not None + worked.append(lease.account_id) + await store.release_period_close(lease, worker_id="solo") + assert worked == ["acct_a", "acct_b"] + # Re-accepting acct_a's exact generation does not reset its rank. + await store.put_configuration(configuration("acct_a")) + again = await store.lease_period_close(worker_id="solo", now=NOW, lease_seconds=60) + assert again is not None and again.account_id == "acct_a" + + async def test_concurrent_period_closes_converge_within_each_account( store: ReportingLedgerStore, ) -> None: @@ -312,7 +601,10 @@ async def fetch(request: ReportingSourceSliceRequestV1) -> None: assert finished.leased is not None and finished.leased.account_id == "acct_a" probe = await store.lease_period_close(worker_id="probe", now=NOW, lease_seconds=60) assert probe is not None and probe.generation_key == configs[0].generation_key - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_4 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_4 is None await store.release_period_close(probe, worker_id="probe") finish["acct_b"].set() turns = await asyncio.wait_for(asyncio.gather(*tasks), 10) diff --git a/tests/conformance/reporting/test_reporting_generation_migration.py b/tests/conformance/reporting/test_reporting_generation_migration.py index fe010a23c..0182a2419 100644 --- a/tests/conformance/reporting/test_reporting_generation_migration.py +++ b/tests/conformance/reporting/test_reporting_generation_migration.py @@ -74,6 +74,15 @@ async def _retained_rows(pool: AsyncConnectionPool) -> dict[str, list[Any]]: ) ).fetchall() result[table] = [row[0] for row in rows] + if table == "reporting_configurations": + # The durable period-close fairness turn is additive and + # defaulted. Dropping it only while it still holds the + # never-leased default keeps every byte of the pre-existing + # evidence under comparison: an upgrade that gave a retained + # generation a non-zero turn would still fail here. + for record in result[table]: + if record.get("lease_turn") == 0: + record.pop("lease_turn", None) if table == "reporting_obligations": # #1171 adds an explicit unknown currency. This test still # compares every byte of the pre-existing #1169 evidence. @@ -178,17 +187,22 @@ async def test_beta15_upgrade_preserves_all_evidence_and_survives_concurrent_boo account_id=old.account_id, reporting_revision_id=revision.reporting_revision_id ) assert rows.rows == ({"media_buy_id": "mb_acct_a", "impressions": 5},) - assert await store.commit_revision(revision, rows.rows) == revision + receipt_operation_1 = await store.commit_revision(revision, rows.rows) + assert receipt_operation_1 == revision statuses = await store.list_consumer_statuses( account_id=old.account_id, consumer_id="shared-buyer" ) assert len(statuses) == 1 and statuses[0].mismatch_code == "metric_missing" - assert await store.record_consumer_status(statuses[0]) == (statuses[0], False) + receipt_operation_2 = await store.record_consumer_status(statuses[0]) + assert receipt_operation_2 == (statuses[0], False) assert await _retained_rows(pool) == before # Replays append no new feed entries. # Existing leases survive, and releasing an old handle must not release # a new tenant's same-name generation even if the worker id is reused. - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_3 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_3 is None other = configuration("acct_b") await store.put_configuration(other) other_lease = await store.lease_period_close( @@ -201,7 +215,10 @@ async def test_beta15_upgrade_preserves_all_evidence_and_survives_concurrent_boo worker_id="new-worker", now=NOW, lease_seconds=60 ) assert reclaimed is not None and reclaimed.generation_key == old.generation_key - assert await store.lease_period_close(worker_id="extra", now=NOW, lease_seconds=60) is None + receipt_operation_4 = await store.lease_period_close( + worker_id="extra", now=NOW, lease_seconds=60 + ) + assert receipt_operation_4 is None # The retained feed and its sequence continue; no history is renumbered. checkpoint = await store.open_snapshot(account_id=old.account_id, filters_fingerprint="") diff --git a/tests/conformance/reporting/test_reporting_materializer_packaging.py b/tests/conformance/reporting/test_reporting_materializer_packaging.py index 36187037b..d01ca1374 100644 --- a/tests/conformance/reporting/test_reporting_materializer_packaging.py +++ b/tests/conformance/reporting/test_reporting_materializer_packaging.py @@ -47,6 +47,8 @@ def b1_wheels(built_distribution): "outbox/required_status_selector_schema.json", "ledger/reporting_materializer.sql", "materializer/required_schema.json", + "ledger/reporting_receipt_ingestion.sql", + "receipts/required_schema.json", ): assert ( vcs.read(f"adcp/reporting/{relative}") @@ -98,6 +100,10 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter durable_adopter = path / f"durable_adopter_{kind}.py" shutil.copy2(ROOT / "examples/reporting_durable_materializer.py", durable_example) shutil.copy2(ROOT / "tests/type_checks/reporting_durable_materializer.py", durable_adopter) + receipt_example = path / f"receipt_example_{kind}.py" + receipt_adopter = path / f"receipt_adopter_{kind}.py" + shutil.copy2(ROOT / "examples/reporting_receipt_ingress.py", receipt_example) + shutil.copy2(ROOT / "tests/type_checks/reporting_receipt_ingress.py", receipt_adopter) result = json.loads( run_step( [str(python), "-I", str(smoke)], @@ -113,6 +119,7 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter "installed": True, "assets": hashes, "durable": True, + "receipts": True, } config = path / "mypy.ini" config.write_text( @@ -133,6 +140,8 @@ def test_python310_installed_wheel_exports_verifier_reference_and_strict_adopter str(example), str(durable_adopter), str(durable_example), + str(receipt_adopter), + str(receipt_example), ], label=f"b1-{kind}-installed-adopter-types", cwd=path, diff --git a/tests/conformance/reporting/test_reporting_materializer_process.py b/tests/conformance/reporting/test_reporting_materializer_process.py index ceb451f94..e44d252b2 100644 --- a/tests/conformance/reporting/test_reporting_materializer_process.py +++ b/tests/conformance/reporting/test_reporting_materializer_process.py @@ -69,6 +69,7 @@ async def worker( "pause": pause, "notifications": notifications, "installed": installed, + "legacy_definition": not case.config.definition.monetary_metric_units, } ) yield child diff --git a/tests/conformance/reporting/test_reporting_materializer_rolling.py b/tests/conformance/reporting/test_reporting_materializer_rolling.py index 46eaf7956..518d17698 100644 --- a/tests/conformance/reporting/test_reporting_materializer_rolling.py +++ b/tests/conformance/reporting/test_reporting_materializer_rolling.py @@ -29,9 +29,9 @@ } -def build_frozen(artifact, tmp_path_factory, request): +def build_frozen(artifact, tmp_path_factory, request, *, sha=None): require_rolling_database() - sha = ARTIFACTS[artifact] + sha = sha or ARTIFACTS[artifact] root = tmp_path_factory.mktemp(f"materializer-{artifact}") request.addfinalizer(lambda: shutil.rmtree(root)) archive, source, dist, environment = ( @@ -80,6 +80,8 @@ def build_frozen(artifact, tmp_path_factory, request): "outbox/status_pg.py", "materializer/contracts.py", "materializer/verification.py", + "materializer/pg.py", + "materializer/capture.py", ): path = source / "src/adcp/reporting" / relative if path.exists(): diff --git a/tests/conformance/reporting/test_reporting_receipt_batches.py b/tests/conformance/reporting/test_reporting_receipt_batches.py new file mode 100644 index 000000000..46e359e56 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_batches.py @@ -0,0 +1,391 @@ +"""The same mixed-batch state machine runs in memory and on PostgreSQL 16.""" + +import asyncio +from copy import deepcopy +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingDeliveryPrincipal, ReportingMaterializationCheck +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.receipts import ReportingReceiptError +from adcp.reporting.receipts.wire import ReceiptBatch, validate_receipt_response + +from ._receipt_support import adjustment_for, batch_state, receipt_case, receipts, request_for + +__all__ = ["receipts"] + + +async def test_mixed_batch_order_durable_failure_and_exact_response_replay(receipts): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + adjustment = await adjustment_for(h, s) + bad_revision = { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "revision-missing-0002", + "reporting_revision_id": "missing", + } + bad_adjustment = { + **adjustment, + "reporting_receipt_id": "adjustment-missing-0002", + "reporting_adjustment_id": "missing", + } + request = request_for( + s, + receipts=[bad_revision, receipt_to_wire(s.receipt)], + adjustment_receipts=[adjustment, bad_adjustment], + context={"trace": "original"}, + ) + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + validate_receipt_response(result, ReceiptBatch.parse(request)) + assert [r["result"] for r in result["results"]] == ["failed", "recorded", "recorded", "failed"] + assert result["results"][0]["reporting_receipt_id"] == bad_revision["reporting_receipt_id"] + assert result["results"][3]["reporting_receipt_id"] == bad_adjustment["reporting_receipt_id"] + assert result["results"][0]["errors"] == result["results"][3]["errors"] + assert result["context"] == {"trace": "original"} + captures = await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) + assert len(captures) == 2 and captures[0].account_sequence < captures[1].account_sequence + assert all(b.core.consumer_ids == (s.binding.consumer_id,) for b in captures) + # A later authorization call can replay after readability/clock changes; + # no item is recomputed and original recorded outcomes/time remain intact. + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + h.clock.now += timedelta(days=450) + before = await h.image() + replay = await h.store.ingest_receipt_batch(deepcopy(request), caller=s.attempt.scope.principal) + assert replay == result and "replayed" not in replay + assert await h.image() == before + replay["results"].clear() + receipt_operation_1 = await h.store.ingest_receipt_batch( + request, caller=s.attempt.scope.principal + ) + assert receipt_operation_1 == result + + +@pytest.mark.parametrize( + "case", + [ + "absent", + "empty", + "empty_sibling", + "null", + "not_list", + "duplicate", + "cross_duplicate", + "too_many", + "received_at", + "adjustment_received_at", + "spoof", + "bad_late_shape", + ], +) +async def test_whole_shape_admission_has_zero_writes(receipts, case): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s) + if case == "absent": + request.pop("receipts") + if case == "empty": + request["receipts"] = [] + if case == "empty_sibling": + request["adjustment_receipts"] = [] + if case == "null": + request["adjustment_receipts"] = None + if case == "not_list": + request["receipts"] = {} + if case == "duplicate": + request["receipts"] *= 2 + if case == "cross_duplicate": + request["adjustment_receipts"] = [ + {**adjustment, "reporting_receipt_id": s.receipt.reporting_receipt_id} + ] + if case == "too_many": + request["receipts"] = [ + {**request["receipts"][0], "reporting_receipt_id": f"receipt-many-{i:06d}"} + for i in range(50) + ] + request["adjustment_receipts"] = [ + {**adjustment, "reporting_receipt_id": f"adjustment-many-{i:06d}"} for i in range(51) + ] + if case == "received_at": + request["receipts"][0]["received_at"] = s.receipt.observed_at.isoformat() + if case == "adjustment_received_at": + request["adjustment_receipts"] = [{**adjustment, "received_at": None}] + if case == "spoof": + request["consumer_id"] = "other" + if case == "bad_late_shape": + request["adjustment_receipts"] = [{"reporting_receipt_id": "invalid-late-0001"}] + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert error.value.code == "INVALID_REQUEST" + assert await h.image() == before + + +@pytest.mark.parametrize( + "change", ["order", "context", "extension", "target", "omitted_array", "time_spelling"] +) +async def test_whole_request_conflict_before_any_write(receipts, change): + h = receipts + s = await receipt_case(h) + request = request_for( + s, + receipts=[ + receipt_to_wire(s.receipt), + { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "receipt-second-0002", + "reporting_revision_id": "missing", + }, + ], + ) + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + changed = deepcopy(request) + if change == "order": + changed["receipts"].reverse() + if change == "context": + changed["context"] = {"trace": "changed"} + if change == "extension": + changed["ext"] = {"test": {"version": 1}} + if change == "target": + changed["receipts"][1]["reporting_revision_id"] = "another" + if change == "omitted_array": + changed["receipts"].pop() + if change == "time_spelling": + changed["receipts"][0]["observed_at"] = changed["receipts"][0]["observed_at"].replace( + "Z", "+00:00" + ) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(changed, caller=s.attempt.scope.principal) + assert error.value.code == "IDEMPOTENCY_CONFLICT" + assert await h.image() == before + + +async def test_concurrent_same_batch_and_changed_body_converge(receipts): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + responses = await asyncio.gather( + *( + h.store.ingest_receipt_batch(deepcopy(request), caller=s.attempt.scope.principal) + for _ in range(8) + ) + ) + assert all(response == responses[0] for response in responses) + assert [r["result"] for r in responses[0]["results"]] == ["recorded", "recorded"] + assert await batch_state(h) == ((2, True),) + assert len(await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal)) == 2 + other = {**request, "context": {"changed": True}} + responses = await asyncio.gather( + h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal), + h.store.ingest_receipt_batch(other, caller=s.attempt.scope.principal), + return_exceptions=True, + ) + assert responses[0]["results"][0]["result"] == "recorded" + assert ( + isinstance(responses[1], ReportingReceiptError) + and responses[1].code == "IDEMPOTENCY_CONFLICT" + ) + + +async def test_same_ids_keys_scoped_to_exact_accounts_and_consumers(receipts): + h = receipts + cases = [ + await receipt_case(h, account_id=a, consumer_id=c) + for a, c in ( + ("acct_a", "buyer"), + ("acct_a", "https://buyer.example.test/agent"), + ("acct_b", "buyer"), + ("acct:a", "b:c"), + ("acct:a:b", "c"), + ) + ] + for s in cases: + response = await h.store.ingest_receipt_batch( + request_for(s), caller=s.attempt.scope.principal + ) + assert response["results"][0]["result"] == "recorded" + assert len(await batch_state(h)) == len(cases) + s = cases[0] + unauthorized = await h.store.ingest_receipt_batch( + request_for(s, key="unavailable-key-0001"), + caller=ReportingDeliveryPrincipal("acct_a", "other"), + ) + unknown = await h.store.ingest_receipt_batch( + request_for( + s, + key="unavailable-key-0002", + receipts=[{**receipt_to_wire(s.receipt), "reporting_revision_id": "missing"}], + ), + caller=ReportingDeliveryPrincipal("acct_a", "other"), + ) + assert unauthorized == unknown + + +async def test_rejected_leaf_replacement_terminal_acceptance_and_no_retry_signal(receipts): + h = receipts + s = await receipt_case(h) + request = request_for( + s, + receipts=[ + {**receipt_to_wire(s.receipt), "status": "rejected", "rejection_codes": ["LOAD_FAILED"]} + ], + ) + work_before = await h.works() + first = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert first["results"][0]["receipt"]["status"] == "rejected" + assert await h.works() == work_before + corrected = { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "receipt-replacement-0002", + "supersedes_reporting_receipt_id": s.receipt.reporting_receipt_id, + } + accepted = await h.store.ingest_receipt_batch( + request_for(s, key="replacement-batch-0002", receipts=[corrected]), + caller=s.attempt.scope.principal, + ) + assert accepted["results"][0]["receipt"]["status"] == "accepted" + terminal = await h.store.ingest_receipt_batch( + request_for( + s, + key="terminal-batch-0003", + receipts=[ + { + **corrected, + "reporting_receipt_id": "receipt-after-accept-0003", + "supersedes_reporting_receipt_id": corrected["reporting_receipt_id"], + } + ], + ), + caller=s.attempt.scope.principal, + ) + assert terminal["results"][0]["errors"][0]["code"] == "ACCEPTED_RECEIPT_TERMINAL" + replay = await h.store.ingest_receipt_batch( + request_for(s, key="unchanged-batch-0004", receipts=[corrected]), + caller=s.attempt.scope.principal, + ) + assert replay["results"][0] == {**accepted["results"][0], "result": "unchanged"} + + +@pytest.mark.parametrize("later", ["success", "failure", "expiry", "corruption"]) +async def test_accepted_artifact_authority_survives_later_outcomes_separately_from_readability( + receipts, later +): + h = receipts + s = await receipt_case(h) + result = await h.store.ingest_receipt_batch(request_for(s), caller=s.attempt.scope.principal) + original = result["results"][0]["receipt"] + if later in {"success", "failure"}: + attempt = replace(s.attempt, reporting_materialization_id="materialization-2", attempt=2) + await h.store.commit_materialization_attempt(attempt) + outcome = replace( + s.outcome, reporting_materialization_id=attempt.reporting_materialization_id + ) + if later == "failure": + outcome = replace( + outcome, + status="failed", + resource=None, + verification=None, + failure_code="CONTENT_CORRUPT", + ) + await h.store.commit_materialization(outcome) + at = h.clock.now + if later == "expiry": + at = s.outcome.resource.expires_at + if later == "corruption": + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "check-corrupt", + "corrupt", + h.clock.now, + ) + ) + snapshot = await h.store.read_reconciliation_snapshot(caller=s.attempt.scope.principal) + assert len(snapshot.current_receipts) == 1 + assert receipt_to_wire(snapshot.current_receipts[0]) == original + view = snapshot.materialization(s.attempt.key) + assert view.outcome == s.outcome + assert view.readable_at(at) is (later not in {"expiry", "corruption"}) + receipt_operation_2 = await h.store.ingest_receipt_batch( + request_for(s), caller=s.attempt.scope.principal + ) + assert receipt_operation_2 == result + + +async def test_hundred_combined_results_and_adjustment_only_admission(receipts): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for( + s, + receipts=[ + {**receipt_to_wire(s.receipt), "reporting_receipt_id": f"revision-{i:010d}"} + for i in range(50) + ], + adjustment_receipts=[ + {**adjustment, "reporting_receipt_id": f"adjustment-{i:010d}"} for i in range(50) + ], + ) + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert len(result["results"]) == 100 + assert sum(r["result"] == "recorded" for r in result["results"]) == 2 + only = { + "account": request["account"], + "idempotency_key": "only-adjustment-0001", + "adjustment_receipts": [request["adjustment_receipts"][0]], + } + receipt_operation_3 = await h.store.ingest_receipt_batch(only, caller=s.attempt.scope.principal) + assert (receipt_operation_3)["results"][0]["result"] == "unchanged" + + +@pytest.mark.parametrize( + "context", + [ + {"field": "nul\x00value"}, + {"nul\x00key": "value"}, + {"field": "\ud800"}, + {"\udfff": "value"}, + {"field": "\ud83d\ude00"}, + ], +) +async def test_unrepresentable_jsonb_strings_fail_whole_shape_before_any_header(receipts, context): + h = receipts + s = await receipt_case(h) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch( + request_for(s, context=context), caller=s.binding.principal + ) + assert error.value.code == "INVALID_REQUEST" + assert await h.image() == before + + +async def test_whole_request_utf16_key_order_survives_storage_and_exact_replay(receipts): + h = receipts + s = await receipt_case(h) + context = { + "nested": {"\ue000": "BMP", "\U00010000": "supplementary"}, + "array": [{"\U0001f600": "face", "e\u0301": "combining", "x": "a\\b\n"}], + } + request = request_for(s, context=context) + response = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert response["results"][0]["result"] == "recorded" + assert response["context"] == context + receipt_operation_4 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_4 == response + before = await h.image() + changed = {**request, "context": {**context, "nested": {"\ue000": "changed"}}} + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(changed, caller=s.binding.principal) + assert error.value.code == "IDEMPOTENCY_CONFLICT" + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_receipt_graph.py b/tests/conformance/reporting/test_reporting_receipt_graph.py new file mode 100644 index 000000000..92637ac71 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_graph.py @@ -0,0 +1,676 @@ +"""Financial predicates are shared by ingress and literal PostgreSQL writes.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger._delivery_state import current_receipt, receipt_chain +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingReceiptKey, +) +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.receipts.records import receipt_record + +from ._generation_support import END +from ._receipt_support import ( + adjustment_for, + foreign_account, + foreign_targets, + receipt_case, + receipt_harness, + receipts, + request_for, +) +from .test_reporting_reconciliation_sql import raw_insert + +__all__ = ["receipts"] + + +@pytest.mark.parametrize( + "fault,expected", + [ + ("rows", "RECEIPT_TOTALS_MISMATCH"), + ("totals", "RECEIPT_TOTALS_MISMATCH"), + ("currency", "RECEIPT_TOTALS_MISMATCH"), + ("digest", "RECEIPT_EVIDENCE_MISMATCH"), + ("manifest", "RECEIPT_EVIDENCE_MISMATCH"), + ("profile", "RECEIPT_PROFILE_MISMATCH"), + ("materialization", "REPORTING_RECORD_UNAVAILABLE"), + ("revision", "REPORTING_RECORD_UNAVAILABLE"), + ("obligation", "REPORTING_RECORD_UNAVAILABLE"), + ("future", "REPORTING_TIME_INVALID"), + ("before_outcome", "REPORTING_TIME_INVALID"), + ("duplicate_totals", "INVALID_REPORTING_RECORD"), + ], +) +async def test_exact_artifact_predicates_produce_durable_item_failures(receipts, fault, expected): + h = receipts + s = await receipt_case(h) + request = request_for(s) + item = request["receipts"][0] + if fault == "rows": + item["observed_row_count"] += 1 + elif fault == "totals": + item["observed_control_totals"][0]["value"] = "1000" + elif fault == "currency": + next(t for t in item["observed_control_totals"] if t["name"] == "spend")["unit"] = "GBP" + elif fault == "digest": + item["observed_canonical_content_digest"]["value"] = "f" * 64 + elif fault == "manifest": + item["observed_manifest_sha256"] = "f" * 64 + elif fault == "profile": + item["verification_profile"] = "manifest_checksums" + item["observed_manifest_sha256"] = "a" * 64 + elif fault == "materialization": + item["reporting_materialization_id"] = "unknown" + elif fault == "revision": + item["reporting_revision_id"] = "unknown" + elif fault == "obligation": + item["reporting_obligation_id"] = "unknown" + elif fault == "future": + item["observed_at"] = "2099-01-01T00:00:00Z" + elif fault == "before_outcome": + item["observed_at"] = END.isoformat() + else: + item["observed_control_totals"].append({**item["observed_control_totals"][0], "value": "2"}) + result = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert result["results"][0]["errors"][0]["code"] == expected + assert not await h.store.get_receipt(s.receipt.key) + assert not await h.store.read_receipt_boundaries(caller=s.binding.principal) + receipt_operation_1 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_1 == result + + +@pytest.mark.parametrize( + "fault", + [ + "rows", + "digest", + "manifest", + "profile", + "materialization", + "revision", + "obligation", + "readability", + ], +) +async def test_literal_sql_enforces_exact_artifact_and_readability_without_python_validator(fault): + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + item = receipt_to_wire(s.receipt) + if fault == "rows": + item["observed_row_count"] += 1 + elif fault == "digest": + item["observed_canonical_content_digest"]["value"] = "f" * 64 + elif fault == "manifest": + item["observed_manifest_sha256"] = "f" * 64 + elif fault == "profile": + item["verification_profile"] = "manifest_checksums" + item["observed_manifest_sha256"] = "a" * 64 + elif fault == "materialization": + item["reporting_materialization_id"] = "unknown" + elif fault == "revision": + item["reporting_revision_id"] = "unknown" + elif fault == "obligation": + item["reporting_obligation_id"] = "unknown" + else: + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "corrupt-before-acceptance", + "corrupt", + s.receipt.observed_at - timedelta(microseconds=1), + ) + ) + candidate = receipt_record("revision_receipt", item, s.binding.principal, s.obligation) + if fault == "obligation": + candidate = replace( + candidate, scope=replace(candidate.scope, reporting_obligation_id="unknown") + ) + before = await h.image() + with pytest.raises(IntegrityError): + await raw_insert(h.pool, candidate) + assert await h.image() == before + + +@pytest.mark.parametrize( + "fault,expected", + [ + ("digest", "ADJUSTMENT_DIGEST_MISMATCH"), + ("revision", "REPORTING_RECORD_UNAVAILABLE"), + ("unknown", "REPORTING_RECORD_UNAVAILABLE"), + ("observed_before_adjustment", "ADJUSTMENT_ORDER_INVALID"), + ("correction_before_finality", "ADJUSTMENT_ORDER_INVALID"), + ], +) +async def test_official_adjustment_digest_ownership_and_order(receipts, fault, expected): + h = receipts + s = await receipt_case(h) + changes = ( + {"correction_observed_at": END - timedelta(seconds=1)} + if fault == "correction_before_finality" + else {} + ) + item = await adjustment_for(h, s, **changes) + if fault == "digest": + item["observed_adjustment_sha256"] = "f" * 64 + elif fault == "revision": + item["adjusts_reporting_revision_id"] = "unknown" + elif fault == "unknown": + item["reporting_adjustment_id"] = "unknown" + elif fault == "observed_before_adjustment": + item["observed_at"] = END.isoformat() + request = request_for(s, adjustment_receipts=[item]) + result = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert result["results"][0]["result"] == "recorded" + assert result["results"][1]["errors"][0]["code"] == expected + assert result["results"][1]["reporting_receipt_id"] == item["reporting_receipt_id"] + receipt_operation_2 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_2 == result + + +@pytest.mark.parametrize( + "fault", ["digest", "observed_before_adjustment", "correction_before_finality"] +) +async def test_literal_sql_adjustment_finality_digest_and_order(fault): + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + changes = ( + {"correction_observed_at": END - timedelta(seconds=1)} + if fault == "correction_before_finality" + else {} + ) + item = await adjustment_for(h, s, **changes) + if fault == "digest": + item["observed_adjustment_sha256"] = "f" * 64 + elif fault == "observed_before_adjustment": + item["observed_at"] = END.isoformat() + candidate = receipt_record("adjustment_receipt", item, s.binding.principal, s.obligation) + before = await h.image() + with pytest.raises(IntegrityError): + await raw_insert(h.pool, candidate) + assert await h.image() == before + + +async def test_adjustment_rejected_current_leaf_replacement_and_accepted_terminality(receipts): + h = receipts + s = await receipt_case(h) + item = await adjustment_for(h, s) + rejected = {**item, "status": "rejected", "rejection_codes": ["LOAD_FAILED"]} + first = request_for(s, adjustment_receipts=[rejected]) + receipt_operation_3 = await h.store.ingest_receipt_batch(first, caller=s.binding.principal) + assert (receipt_operation_3)["results"][1]["result"] == "recorded" + accepted = { + **item, + "reporting_receipt_id": "accepted-adjustment-0002", + "supersedes_reporting_receipt_id": item["reporting_receipt_id"], + } + second = request_for(s, key="receipt-batch-0002", adjustment_receipts=[accepted]) + receipt_operation_4 = await h.store.ingest_receipt_batch(second, caller=s.binding.principal) + assert (receipt_operation_4)["results"][1]["result"] == "recorded" + third = request_for( + s, + key="receipt-batch-0003", + adjustment_receipts=[ + { + **accepted, + "reporting_receipt_id": "terminal-adjustment-0003", + "supersedes_reporting_receipt_id": accepted["reporting_receipt_id"], + } + ], + ) + result = await h.store.ingest_receipt_batch(third, caller=s.binding.principal) + assert result["results"][1]["errors"][0]["code"] == "ACCEPTED_RECEIPT_TERMINAL" + leaves = ( + await h.store.read_reconciliation_snapshot(caller=s.binding.principal) + ).current_receipts + assert {r.reporting_receipt_id for r in leaves} == { + s.receipt.reporting_receipt_id, + accepted["reporting_receipt_id"], + } + + +def damaged_chain(s, fault, root=None): + root = replace( + root if root is not None else s.receipt, status="rejected", rejection_codes=("LOAD_FAILED",) + ) + leaf = replace( + root, + reporting_receipt_id="leaf-receipt-0002", + supersedes_reporting_receipt_id=root.reporting_receipt_id, + ) + records = [root, leaf] + if fault == "fork": + records.append(replace(leaf, reporting_receipt_id="fork-receipt-0003")) + elif fault == "cycle": + records[0] = replace(root, supersedes_reporting_receipt_id=leaf.reporting_receipt_id) + elif fault == "gap": + records.pop(0) + elif fault == "cross_target": + # An adjustment chain key binds only reporting_adjustment_id, so a + # divergent adjusts_reporting_revision_id stays inside the same chain + # and only the dedicated official-target predicate can reject it. + field = ( + "adjusts_reporting_revision_id" + if isinstance(root, ReportingAdjustmentReceiptRecord) + else "reporting_revision_id" + ) + records[0] = replace(root, **{field: "other-revision"}) + elif fault == "accepted_predecessor": + records[0] = replace(root, status="accepted", rejection_codes=()) + else: + cycle = replace( + root, + reporting_receipt_id="cycle-receipt-0003", + supersedes_reporting_receipt_id="cycle-receipt-0004", + ) + records += [ + cycle, + replace( + cycle, + reporting_receipt_id="cycle-receipt-0004", + supersedes_reporting_receipt_id=cycle.reporting_receipt_id, + ), + ] + return records, leaf + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_complete_receipt_chain_rejects_damage_with_one_apparent_leaf(fault): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + records, leaf = damaged_chain(s, fault) + with pytest.raises(LedgerConflictError) as error: + current_receipt(tuple(records), leaf) + assert error.value.code == "REPORTING_HISTORY_CORRUPT" + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_literal_sql_blocks_forks_and_detects_damaged_existing_chains(fault): + from adcp.reporting.ledger._delivery_state import ( + change_id, + fingerprint, + payload, + storage_identity, + ) + from adcp.reporting.ledger.delivery_pg import _IDENTITY_COLUMNS + + from ._generation_support import NOW + + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError, sql + from psycopg.types.json import Jsonb + + s = await receipt_case(h) + records, leaf = damaged_chain(s, fault) + # Deliberate owner-level corruption bypasses triggers only while seeding + # the invalid prior graph. The subsequent literal INSERT runs every + # production predicate, without a Python validator or head API. + from contextlib import nullcontext + + seed_before = await h.image() + # A fork cannot even be seeded with replica triggers: the reviewed + # unique successor index is an independent hard financial predicate. + with pytest.raises(IntegrityError) if fault == "fork" else nullcontext(): + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role = replica") + for record in records: + record = replace(record, received_at=NOW) + row = dict(zip(_IDENTITY_COLUMNS, storage_identity(record))) + row.update( + payload=Jsonb(payload(record)), + content_sha256=fingerprint(record), + change_id=change_id(record), + ) + await c.execute( + sql.SQL( + "INSERT INTO reporting_reconciliation_records ({}) VALUES ({})" + ).format( + sql.SQL(", ").join(map(sql.Identifier, row)), + sql.SQL(", ").join(sql.Placeholder() for _ in row), + ), + tuple(row.values()), + ) + if fault == "fork": + assert await h.image() == seed_before + return + new = replace( + leaf, + reporting_receipt_id="new-leaf-receipt-0099", + supersedes_reporting_receipt_id=leaf.reporting_receipt_id, + ) + before = await h.image() + with pytest.raises(IntegrityError): + await raw_insert(h.pool, new) + assert await h.image() == before + + +@pytest.mark.parametrize( + "method,profile,field,wrong", + [ + ("file_transfer", "canonical_digest", "observed_canonical_content_digest", None), + ("file_transfer", "manifest_checksums", "observed_manifest_sha256", "f" * 64), + ("file_transfer", "native_commit", "observed_native_version_ref", "wrong-version"), + ("dataset_share", "native_commit", "observed_native_version_ref", "wrong-version"), + ( + "warehouse_materialization", + "native_commit", + "observed_native_version_ref", + "wrong-version", + ), + ], +) +async def test_all_receipt_evidence_profiles_match_exact_immutable_artifact( + receipts, method, profile, field, wrong +): + from copy import deepcopy + + h = receipts + s = await receipt_case(h, method=method, profile=profile, billing=profile == "canonical_digest") + request = request_for(s) + bad = deepcopy(request) + if wrong is None: + bad["receipts"][0][field][ + "canonicalization_uri" + ] = "https://wrong.example.test/profile.json" + else: + bad["receipts"][0][field] = wrong + result = await h.store.ingest_receipt_batch(bad, caller=s.binding.principal) + assert result["results"][0]["errors"][0]["code"] == "RECEIPT_EVIDENCE_MISMATCH" + request["idempotency_key"] = "valid-artifact-receipt" + accepted = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert accepted["results"][0]["result"] == "recorded" + assert accepted["results"][0]["receipt"][field] == request["receipts"][0][field] + receipt_operation_5 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_5 == accepted + + +async def test_snapshot_adjustment_is_never_admitted_as_official_receipt_evidence(receipts): + h = receipts + s = await receipt_case(h, finality="snapshot", billing=False) + before = await h.image() + with pytest.raises(LedgerConflictError) as error: + await adjustment_for(h, s) + assert error.value.code == "ADJUSTMENT_REQUIRES_OFFICIAL" + assert await h.image() == before + item = { + "reporting_receipt_id": "snapshot-adjustment-0001", + "reporting_adjustment_id": "adjustment-1", + "adjusts_reporting_revision_id": s.revision.reporting_revision_id, + "status": "accepted", + "observed_at": s.receipt.observed_at.isoformat(), + "observed_adjustment_sha256": "0" * 64, + } + result = await h.store.ingest_receipt_batch( + request_for(s, adjustment_receipts=[item]), caller=s.binding.principal + ) + assert result["results"][0]["result"] == "recorded" + assert result["results"][1]["errors"][0]["code"] == "REPORTING_RECORD_UNAVAILABLE" + + +_FOREIGN_SQL_PREDICATE = { + "materialization_other_obligation": "receipt artifact is unavailable", + "materialization_other_consumer": "receipt artifact is unavailable", + "revision_other_obligation": "receipt target is unavailable", + "obligation_other_target": "receipt target is unavailable", + "adjustment_other_revision": "receipt adjustment order is invalid", + "adjustment_revision_other_target": "receipt target is unavailable", +} +_FOREIGN_TARGETS = list(_FOREIGN_SQL_PREDICATE) + + +def _foreign_case(s, f, fault): + """One existing, valid but foreign reference, plus its truly absent twin. + + ``adjustment_other_revision`` deliberately observes *after* the foreign + adjustment was created. Every ordering comparison then holds on its own, so + within this trigger ``adjusts_reporting_revision_id`` is the remaining + reason the adjustment tuple finds no row. A nearer observation would be + refused by the ordering rule instead and could not attribute the ownership + predicate. Inherited reconciliation guards still refuse the write if this + predicate is weakened, so this is attribution, not exclusive necessity. + """ + if fault == "materialization_other_obligation": + return "revision", {"reporting_materialization_id": f.materialization_id}, {} + if fault == "materialization_other_consumer": + return "revision", {"reporting_materialization_id": f.consumer_materialization_id}, {} + if fault == "revision_other_obligation": + return "revision", {"reporting_revision_id": f.revision.reporting_revision_id}, {} + if fault == "obligation_other_target": + return "revision", {"reporting_obligation_id": f.obligation.reporting_obligation_id}, {} + if fault == "adjustment_other_revision": + late = (f.adjustment.created_at + timedelta(seconds=1)).isoformat().replace("+00:00", "Z") + return ( + "adjustment", + {"reporting_adjustment_id": f.adjustment.reporting_adjustment_id, "observed_at": late}, + {"observed_at": late}, + ) + return "adjustment", {"adjusts_reporting_revision_id": f.revision.reporting_revision_id}, {} + + +@pytest.mark.parametrize("fault", _FOREIGN_TARGETS) +async def test_existing_foreign_targets_fail_exactly_like_absent_records(receipts, fault): + """A valid row owned by another target is not a usable receipt reference. + + Absence and misownership must be indistinguishable, so a caller cannot probe + another account's, consumer's, obligation's or revision's artifact history. + """ + h = receipts + s = await receipt_case(h) + f = await foreign_targets(h, s) + kind, foreign, shared = _foreign_case(s, f, fault) + base = await adjustment_for(h, s) if kind == "adjustment" else receipt_to_wire(s.receipt) + array = "adjustment_receipts" if kind == "adjustment" else "receipts" + # An adjustment batch keeps its ordinary revision receipt at ordinal zero. + index = 1 if kind == "adjustment" else 0 + + def build(overrides, key, receipt_id): + item = {**base, **shared, **overrides, "reporting_receipt_id": receipt_id} + return request_for(s, key=key, **{array: [item]}) + + failed_id = base["reporting_receipt_id"] + request = build(foreign, "receipt-batch-0001", failed_id) + absent_of = {name: f"absent-{name}" for name in foreign if name != "observed_at"} + control = build(absent_of, "receipt-batch-0002", "receipt-control-0001") + result = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + failure = result["results"][index] + assert failure["result"] == "failed" + assert failure["reporting_receipt_id"] == failed_id + assert failure["errors"][0]["code"] == "REPORTING_RECORD_UNAVAILABLE" + # The refused item is durably absent from evidence, the caller feed and capture. + assert not await h.store.get_receipt(ReportingReceiptKey(s.binding.principal, failed_id)) + boundaries = await h.store.read_receipt_boundaries(caller=s.binding.principal) + assert failed_id not in {b.reporting_receipt_id for b in boundaries} + assert len(boundaries) == index + # Identical durable replay, and an identical closed error for an absent row. + receipt_operation_6 = await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert receipt_operation_6 == result + absent = await h.store.ingest_receipt_batch(control, caller=s.binding.principal) + assert absent["results"][index]["result"] == "failed" + assert absent["results"][index]["errors"] == failure["errors"] + + +@pytest.mark.parametrize("fault", _FOREIGN_TARGETS) +async def test_literal_sql_rejects_existing_foreign_targets(fault): + """The exact tuple predicates run without any Python validator or head API.""" + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + f = await foreign_targets(h, s) + kind, foreign, shared = _foreign_case(s, f, fault) + obligation = s.obligation + if kind == "adjustment": + item = {**await adjustment_for(h, s), **shared, **foreign} + else: + item = {**receipt_to_wire(s.receipt), **shared, **foreign} + # The scope carries the obligation, so a foreign one replaces it. + obligation = f.obligation if "reporting_obligation_id" in foreign else s.obligation + item.pop("reporting_obligation_id", None) + item["reporting_obligation_id"] = obligation.reporting_obligation_id + candidate = receipt_record(f"{kind}_receipt", item, s.binding.principal, obligation) + before = await h.image() + with pytest.raises(IntegrityError) as error: + await raw_insert(h.pool, candidate) + # Attribution, not sole prevention: the inherited reconciliation guards + # independently refuse these same tuples, so a bare IntegrityError + # cannot show that this slice's predicates run at all. Asserting the + # exact message proves they reject first. Predicate *drift* is refused + # separately by the fingerprinted readiness manifest -- see + # test_reporting_receipt_migration.py's function-body damage cases. + assert error.value.diag.message_primary == _FOREIGN_SQL_PREDICATE[fault] + assert await h.image() == before + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_complete_adjustment_receipt_chain_rejects_damage(fault): + """Adjustment chains need their own topology and official-target checks.""" + async with receipt_harness("memory") as h: + s = await receipt_case(h) + root = receipt_record( + "adjustment_receipt", await adjustment_for(h, s), s.binding.principal, s.obligation + ) + records, leaf = damaged_chain(s, fault, root) + if fault == "cross_target": + # The divergent predecessor is inside the requested chain, so no + # fork/gap/leaf-count rule can stand in for the exact predicate. + assert receipt_chain(records[0]) == receipt_chain(leaf) + with pytest.raises(LedgerConflictError) as error: + current_receipt(tuple(records), leaf) + assert error.value.code == "REPORTING_HISTORY_CORRUPT" + + +@pytest.mark.parametrize( + "fault", ["fork", "cycle", "gap", "cross_target", "accepted_predecessor", "disconnected_cycle"] +) +async def test_literal_sql_blocks_damaged_adjustment_chains(fault): + from contextlib import nullcontext + + from adcp.reporting.ledger._delivery_state import ( + change_id, + fingerprint, + payload, + storage_identity, + ) + from adcp.reporting.ledger.delivery_pg import _IDENTITY_COLUMNS + + from ._generation_support import NOW + + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError, sql + from psycopg.types.json import Jsonb + + s = await receipt_case(h) + root = receipt_record( + "adjustment_receipt", await adjustment_for(h, s), s.binding.principal, s.obligation + ) + records, leaf = damaged_chain(s, fault, root) + seed_before = await h.image() + # Owner-level seeding only; the final literal INSERT runs every trigger. + # "fork" is the exception: the unique successor index refuses it during + # seeding, so that variant never reaches a trigger-enabled INSERT. + with pytest.raises(IntegrityError) if fault == "fork" else nullcontext(): + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role = replica") + for record in records: + record = replace(record, received_at=NOW) + row = dict(zip(_IDENTITY_COLUMNS, storage_identity(record))) + row.update( + payload=Jsonb(payload(record)), + content_sha256=fingerprint(record), + change_id=change_id(record), + ) + await c.execute( + sql.SQL( + "INSERT INTO reporting_reconciliation_records ({}) VALUES ({})" + ).format( + sql.SQL(", ").join(map(sql.Identifier, row)), + sql.SQL(", ").join(sql.Placeholder() for _ in row), + ), + tuple(row.values()), + ) + if fault == "fork": + assert await h.image() == seed_before + return + new = replace( + leaf, + reporting_receipt_id="new-adjustment-receipt-0099", + supersedes_reporting_receipt_id=leaf.reporting_receipt_id, + ) + before = await h.image() + with pytest.raises(IntegrityError) as error: + await raw_insert(h.pool, new) + assert error.value.diag.message_primary in { + "receipt replacement is unavailable", + "receipt history is inconsistent", + } + assert await h.image() == before + + +_FOREIGN_ACCOUNT_SQL = { + "revision_other_account": "receipt target is unavailable", + "obligation_other_account": "receipt target is unavailable", + "materialization_other_account": "receipt artifact is unavailable", + "adjustment_other_account": "receipt adjustment order is invalid", +} + + +@pytest.mark.parametrize("fault", list(_FOREIGN_ACCOUNT_SQL)) +async def test_literal_sql_rejects_valid_other_account_targets(fault): + """The account column of each financial tuple, at the write predicate itself. + + Store-level and mounted authorization denials are a different boundary and + cannot stand in for this: they never reach an INSERT. Obligations, revisions + and adjustments are globally keyed, so a foreign-account reference is the + only way to exercise the account column with a wholly valid target row. + """ + async with receipt_harness("postgres") as h: + from psycopg import IntegrityError + + s = await receipt_case(h) + other, adjustment, materialization = await foreign_account(h) + if fault == "adjustment_other_account": + candidate = receipt_record( + "adjustment_receipt", await adjustment_for(h, s), s.binding.principal, s.obligation + ) + candidate = replace( + candidate, reporting_adjustment_id=adjustment["reporting_adjustment_id"] + ) + else: + candidate = receipt_record( + "revision_receipt", receipt_to_wire(s.receipt), s.binding.principal, s.obligation + ) + if fault == "revision_other_account": + candidate = replace( + candidate, reporting_revision_id=other.revision.reporting_revision_id + ) + elif fault == "obligation_other_account": + candidate = replace( + candidate, + scope=replace( + candidate.scope, + reporting_obligation_id=other.obligation.reporting_obligation_id, + ), + ) + else: + candidate = replace(candidate, reporting_materialization_id=materialization) + before = await h.image() + with pytest.raises(IntegrityError) as error: + await raw_insert(h.pool, candidate) + assert error.value.diag.message_primary == _FOREIGN_ACCOUNT_SQL[fault] + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_receipt_installed_pg.py b/tests/conformance/reporting/test_reporting_receipt_installed_pg.py new file mode 100644 index 000000000..3d2488963 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_installed_pg.py @@ -0,0 +1,139 @@ +"""Installed VCS/sdist Python 3.10 receipts through real transports after SIGKILL.""" + +import asyncio +import hashlib +import json +import shutil +from pathlib import Path + +import pytest + +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.receipts import PgReportingReceiptStore + +from ._durable_materializer_support import DurableHarness +from ._generation_support import isolated_reporting_pool +from ._receipt_support import adjustment_for, receipt_case, request_for +from ._reconciliation_support import Clock +from .test_reporting_materializer_installed_pg import installed_materializer +from .test_reporting_materializer_packaging import ROOT, b1_wheels, built_distribution, run_step +from .test_reporting_receipt_process import worker + +__all__ = ["installed_materializer", "b1_wheels", "built_distribution"] + + +@pytest.fixture(scope="module") +def installed_receipts(installed_materializer): + root, python, _, original = installed_materializer + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, "asgi-lifespan==2.1.0"], + label="receipt-installed-transport-test-lifespan", + cwd=root, + timeout=60, + ) + modules = dict(original["modules"]) + for name in ( + "reporting/receipts/pg.py", + "reporting/receipts/handler.py", + "reporting/receipts/wire.py", + "reporting/receipts/transport.py", + "server/mcp_tools.py", + "server/a2a_server.py", + "server/serve.py", + ): + modules["adcp." + name.removesuffix(".py").replace("/", ".")] = hashlib.sha256( + (ROOT / "src/adcp" / name).read_bytes() + ).hexdigest() + scripts = {} + for name in ("_receipt_process.py", "_receipt_installed.py", "_receipt_transport.py"): + scripts[name] = root / name + shutil.copy2(Path(__file__).with_name(name), scripts[name]) + return root, python, scripts, {**original, "modules": modules} + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_installed_receipt_sql_crash_and_mounted_replay(installed_receipts, notifications): + root, python, scripts, installed = installed_receipts + async with isolated_reporting_pool(autocommit=True) as pool: + settings = { + "installed": installed, + "conninfo": pool.conninfo, + "kwargs": pool.kwargs, + "notifications": notifications, + "action": "install", + } + + async def invoke(settings): + result = json.loads( + await asyncio.to_thread( + run_step, + [str(python), "-I", str(scripts["_receipt_installed.py"])], + label=f"receipt-installed-{settings['action']}", + cwd=root, + value=settings, + timeout=90, + ) + ) + assert "failure" not in result, result + return result + + ready = await invoke(settings) + assert ready["result"] == {"installed": True} + h = DurableHarness( + PgReportingReceiptStore(pool=pool, notifications=notifications), Clock(), pool + ) + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + async with worker( + h, + s, + request, + pause="between_ordinals", + installed=installed, + python=python, + script=scripts["_receipt_process.py"], + ) as child: + await child.event("between_ordinals") + # The child owns the next ordinal's account lock. Kill it + # before using the public locked reader; ordinal zero is + # already durable and must survive that process death. + await child.kill() + original = await h.store.get_receipt(s.receipt.key) + assert original is not None + done = await invoke( + { + **settings, + "action": "replay", + "request": request, + "helper": str(scripts["_receipt_transport.py"]), + "caller": { + "account_id": s.obligation.account_id, + "consumer_id": s.binding.consumer_id, + }, + } + ) + assert done["result"]["results"][0]["receipt"] == receipt_to_wire(original) + assert done["origins"] == ready["origins"] + receipt_operation_1 = await h.store.ingest_receipt_batch( + request, caller=s.binding.principal + ) + assert receipt_operation_1 == done["result"] + assert len(await h.store.read_receipt_boundaries(caller=s.binding.principal)) == 2 + assert await h.queue() == ((), ()) + print( + json.dumps( + { + "installed_receipts": installed, + "notifications": notifications, + "origins": done["origins"], + "received_at": done["result"]["results"][0]["receipt"]["received_at"], + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_receipt_materializer.py b/tests/conformance/reporting/test_reporting_receipt_materializer.py new file mode 100644 index 000000000..42dd494fa --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_materializer.py @@ -0,0 +1,65 @@ +"""Receipt composition preserves the one reviewed materializer transaction.""" + +from datetime import datetime, timezone + +import pytest + +from adcp.reporting.materializer import ReportingWriterError +from adcp.reporting.materializer.capture import MaterializerNotificationState + +from ._durable_materializer_support import durable_case +from ._receipt_support import receipt_harness +from ._reliable_support import ManualClock +from .test_reporting_receipt_transactions import fail_after + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_receipt_store_atomic_finish_capture_ack_and_enabled_enqueue(backend, monkeypatch): + async with receipt_harness(backend, notifications=True) as h: + h.clock = ManualClock(datetime.now(timezone.utc)) + h.store._clock = h.clock if h.pool is None else None + case = await durable_case(h.store, reconciliation_mode="consumer_receipt") + lease = await case.claim() + prepared, verified = await case.verified(lease) + before = await h.image() + if h.pool is None: + fail_after(monkeypatch, MaterializerNotificationState, "enqueue") + else: + from psycopg import AsyncConnection + + fail_after( + monkeypatch, + AsyncConnection, + "execute", + asynchronous=True, + predicate=lambda args, _: isinstance(args[1], str) + and args[1].startswith("INSERT INTO reporting_materializer_notification_events"), + ) + with pytest.raises(ReportingWriterError): + await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + monkeypatch.undo() + assert await h.image() == before + receipt_operation_1 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified + ) + assert (receipt_operation_1).state == "verified" + captured = await h.store.read_materializer_boundaries(caller=case.scope.principal) + assert len(captured) == 1 and captured[0].account_sequence == 1 + assert (await h.queue())[1] == ("quarantined",) + assert (await h.works())[0][1] == "acked" + assert not await h.store.read_receipt_boundaries(caller=case.scope.principal) + assert "reporting.delivery_ready" not in await h.ordinary_events() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_receipt_composition_preserves_complete_parent_notification_advertisement_veto( + backend, monkeypatch +): + # Reuse the reviewed same-store vector, now instantiated with the additive + # receipt store. This covers Core ledger_changed too, not just Managed. + from . import test_reporting_materializer_durable as parent_vectors + + monkeypatch.setattr(parent_vectors, "durable_harness", receipt_harness) + await parent_vectors.test_materializer_store_closes_core_advertisement_until_b24_admits_it( + backend + ) diff --git a/tests/conformance/reporting/test_reporting_receipt_migration.py b/tests/conformance/reporting/test_reporting_receipt_migration.py new file mode 100644 index 000000000..71955facc --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_migration.py @@ -0,0 +1,150 @@ +"""Additive receipt manifests and migration preserve the approved materializer.""" + +import asyncio +import json +from importlib.resources import files + +import pytest + +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts import PgReportingReceiptStore, ReportingReceiptError + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import isolated_reporting_pool +from ._receipt_support import receipt_case, receipt_harness, request_for +from ._reconciliation_support import Clock + +SQL = files("adcp.reporting.ledger").joinpath("reporting_receipt_ingestion.sql").read_text() +MANIFEST = json.loads(files("adcp.reporting.receipts").joinpath("required_schema.json").read_text()) +PARENT_MANIFEST = json.loads( + files("adcp.reporting.materializer").joinpath("required_schema.json").read_text() +) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_populated_repeated_concurrent_migration_preserves_parent_objects_captures_and_queues( + notifications, autocommit +): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + old = PgReportingMaterializerStore(pool=pool, notifications=notifications) + await old.create_schema() + case = await durable_case(old) + receipt_operation_1 = await case.service().run_once() + assert (receipt_operation_1).state == "verified" + h = DurableHarness(old, Clock(), pool) + before = await h.image() + captures = await old.read_materializer_boundaries(caller=case.scope.principal) + queue = await h.queue() + async with pool.connection() as c: + original = await schema_objects(c) + new = PgReportingReceiptStore(pool=pool, notifications=notifications) + with pytest.raises(ReportingReceiptError) as error: + await new.receipt_ingestion_ready() + assert error.value.code == "RECEIPT_SCHEMA_UNREADY" + await asyncio.gather(*(new.create_schema() for _ in range(3))) + await new.create_schema() + assert await new.receipt_ingestion_ready() + async with pool.connection() as c: + actual = await schema_objects(c) + assert {key: actual[key] for key in original} == original + assert len(PARENT_MANIFEST) == 187 + assert {key: actual[key] for key in PARENT_MANIFEST} == PARENT_MANIFEST + assert { + key: value for key, value in actual.items() if "reporting_receipt_ingestion_" in key + } == MANIFEST + after = await h.image() + assert {key: after[key] for key in before} == before + assert await new.read_materializer_boundaries(caller=case.scope.principal) == captures + assert await h.queue() == queue + assert queue[1] == (("quarantined",) if notifications else ()) + + +async def test_interrupted_receipt_migration_is_invisible_and_restart_converges(): + async with isolated_reporting_pool(autocommit=True) as pool: + await PgReportingMaterializerStore(pool=pool).create_schema() + entered, release = asyncio.Event(), asyncio.Event() + + async def migrate(): + async with pool.connection() as c, c.transaction(): + await c.execute(SQL) + entered.set() + await release.wait() + + task = asyncio.create_task(migrate()) + await asyncio.wait_for(entered.wait(), 10) + async with pool.connection() as c: + assert ( + await ( + await c.execute("SELECT to_regclass('reporting_receipt_ingestion_batches')") + ).fetchone() + )[0] is None + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + new = PgReportingReceiptStore(pool=pool) + with pytest.raises(ReportingReceiptError): + await new.receipt_ingestion_ready() + await new.create_schema() + assert await new.receipt_ingestion_ready() + + +@pytest.mark.parametrize( + "damage", + [ + "DROP INDEX reporting_receipt_ingestion_chain", + ( + "ALTER TABLE reporting_receipt_ingestion_results " + "DISABLE TRIGGER reporting_receipt_ingestion_result" + ), + "ALTER TABLE reporting_receipt_ingestion_batches ALTER COLUMN expected_count DROP NOT NULL", + "DROP TABLE reporting_receipt_ingestion_boundaries CASCADE", + "ALTER TABLE reporting_materializer_work DISABLE TRIGGER reporting_materializer_guard", + ( + "ALTER TABLE reporting_reconciliation_records " + "DISABLE TRIGGER reporting_reconciliation_guard" + ), + # Every object still present and enabled, but a financial predicate body + # silently replaced. Dropped objects and disabled triggers cannot stand + # in for this: a weakened tuple predicate is exactly how a deployment + # would lose graph enforcement without any visible schema difference. + ( + "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_graph()" + " RETURNS TRIGGER LANGUAGE plpgsql AS $damage$ BEGIN RETURN NEW; END $damage$" + ), + ( + "CREATE OR REPLACE FUNCTION reporting_receipt_ingestion_sha256(document JSONB)" + " RETURNS TEXT LANGUAGE SQL IMMUTABLE STRICT AS $damage$ SELECT repeat('0', 64) $damage$" + ), + ], +) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_old_partial_mismatched_schemas_fail_closed_even_for_completed_replay( + damage, notifications +): + async with receipt_harness("postgres", notifications=notifications) as h: + s = await receipt_case(h) + request = request_for(s) + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + async with h.pool.connection() as c: + await c.execute(damage) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert error.value.code == "RECEIPT_SCHEMA_UNREADY" + assert await h.image() == before + + +async def test_receipt_store_does_not_claim_readiness_on_old_parent_schema(): + async with isolated_reporting_pool(autocommit=True) as pool: + old = PgReportingMaterializerStore(pool=pool, clock=Clock()) + await old.create_schema() + h = DurableHarness(old, Clock(), pool) + s = await receipt_case(h) + before = await h.image() + new = PgReportingReceiptStore(pool=pool) + with pytest.raises(ReportingReceiptError) as error: + await new.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + assert error.value.code == "RECEIPT_SCHEMA_UNREADY" + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_receipt_process.py b/tests/conformance/reporting/test_reporting_receipt_process.py new file mode 100644 index 000000000..1c68447a5 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_process.py @@ -0,0 +1,123 @@ +"""Actual process death and independent size-one pools resume durable ordinals.""" + +import asyncio +import sys +from contextlib import asynccontextmanager +from pathlib import Path + +import pytest + +from ._receipt_support import ( + adjustment_for, + batch_state, + receipt_case, + receipt_harness, + request_for, +) +from .test_reporting_materializer_process import Child + + +@asynccontextmanager +async def worker(h, s, request, *, pause=None, installed=None, python=None, script=None): + process = await asyncio.create_subprocess_exec( + str(python or sys.executable), + *(["-I"] if installed else []), + str(script or Path(__file__).with_name("_receipt_process.py")), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "caller": { + "account_id": s.obligation.account_id, + "consumer_id": s.binding.consumer_id, + }, + "request": request, + "pause": pause, + "notifications": h.store._notifications_enabled, + "installed": installed, + } + ) + yield child + finally: + await child.kill() + diagnostic = await process.stderr.read() + if process.returncode not in {0, -9}: + pytest.fail(diagnostic.decode()) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize( + "point", + ["header", "receipt", "feed", "capture", "ordinal", "between_ordinals", "final", "committed"], +) +async def test_process_crash_resumes_original_ordinals_and_final_response(point, notifications): + async with receipt_harness("postgres", notifications=notifications) as h: + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + async with worker(h, s, request, pause=point) as child: + await child.event(point) + expected = ( + 2 if point in {"final", "committed"} else (1 if point == "between_ordinals" else 0) + ) + if expected: + assert await batch_state(h) == ((expected, point == "committed"),) + else: + assert await batch_state(h) == () + # These are MVCC reads from another connection while the process + # owns the account lock. No partial receipt/feed/capture is visible. + async with h.pool.connection() as c: + row = await ( + await c.execute( + "SELECT (SELECT count(*) FROM reporting_reconciliation_records " + "WHERE namespace='receipt')," + " (SELECT count(*) FROM reporting_reconciliation_changes " + "WHERE namespace='receipt')," + " (SELECT count(*) FROM reporting_receipt_ingestion_boundaries)" + ) + ).fetchone() + assert row == (expected, expected, expected) + await child.kill() + original = await h.store.get_receipt(s.receipt.key) + async with worker(h, s, request) as resumed: + response = (await resumed.event("done"))["result"] + receipt_operation_2 = await asyncio.wait_for(resumed.process.wait(), 5) + assert receipt_operation_2 == 0 + assert [r["result"] for r in response["results"]] == ["recorded", "recorded"] + if original is not None: + from adcp.reporting.ledger.delivery import receipt_to_wire + + assert response["results"][0]["receipt"] == receipt_to_wire(original) + receipt_operation_1 = await h.store.ingest_receipt_batch( + request, caller=s.binding.principal + ) + assert receipt_operation_1 == response + assert await batch_state(h) == ((2, True),) + assert len(await h.store.read_receipt_boundaries(caller=s.binding.principal)) == 2 + assert await h.queue() == ((), ()) + + +async def test_two_processes_serialize_with_size_one_pools_and_keep_original_recorded_outcomes(): + async with receipt_harness("postgres") as h: + s = await receipt_case(h) + request = request_for(s) + async with worker(h, s, request, pause="receipt") as first: + await first.event("receipt") + async with worker(h, s, request) as second: + pending = asyncio.create_task(second.event("done")) + await asyncio.sleep(0.1) + assert not pending.done() + await first.send({"continue": True}) + a = (await first.event("done"))["result"] + b = (await asyncio.wait_for(pending, 30))["result"] + assert a == b and a["results"][0]["result"] == "recorded" + receipt_operation_4 = await asyncio.wait_for(second.process.wait(), 5) + assert receipt_operation_4 == 0 + receipt_operation_3 = await asyncio.wait_for(first.process.wait(), 5) + assert receipt_operation_3 == 0 diff --git a/tests/conformance/reporting/test_reporting_receipt_rolling.py b/tests/conformance/reporting/test_reporting_receipt_rolling.py new file mode 100644 index 000000000..0d8576b2a --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_rolling.py @@ -0,0 +1,237 @@ +"""Eight actual historical installations on the approved B2.1 and B2.2 schemas.""" + +import asyncio +import json +import shutil +import sys +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import ( + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.delivery import adjustment_to_wire, receipt_to_wire +from adcp.reporting.materializer import PgReportingMaterializerStore +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.receipts import PgReportingReceiptStore + +from ._durable_materializer_support import DurableHarness, durable_case +from ._generation_support import END, isolated_reporting_pool +from .test_reporting_materializer_process import worker +from .test_reporting_materializer_rolling import ARTIFACTS, build_frozen, frozen_call + +B21 = "3fd62121c96a074e3ea458c30c5224d6a586f169" + + +def receipt_probe(artifact): + root, python, _, settings = artifact + script = root / "receipt_frozen.py" + shutil.copy2(Path(__file__).with_name("_receipt_frozen.py"), script) + return root, python, script, settings + + +@pytest.fixture(scope="module") +def installed_b21(tmp_path_factory, request): + return receipt_probe(build_frozen("b21", tmp_path_factory, request, sha=B21)) + + +@pytest.fixture(scope="module", params=(*ARTIFACTS, "b21")) +def installed_receipt_history(request, tmp_path_factory, installed_b21): + if request.param == "b21": + return installed_b21 + return receipt_probe(build_frozen(request.param, tmp_path_factory, request)) + + +async def immutable_parent_rows(pool): + async with pool.connection() as connection: + return [ + await (await connection.execute(f"SELECT * FROM {table} ORDER BY 1,2,3")).fetchall() + for table in ( + "reporting_materializer_work", + "reporting_materializer_status_boundaries", + "reporting_materializer_status_heads", + "reporting_materializer_notification_events", + "reporting_materializer_notification_expansions", + ) + ] + + +async def test_actual_old_readers_and_writers_before_and_after_receipt_migration( + installed_receipt_history, installed_b21, tmp_path +): + artifact = installed_receipt_history[3]["artifact"] + # All eight default-off ordinary writers are required. A's inherited + # notification-readiness closure after C is compared separately on both + # sides. The four compatible notification writers also exercise enabled + # writes; no applicable case is hidden behind a parameter skip. + for notifications in [False, True] if artifact in {"b", "c", "b1", "b21"} else [False]: + async with isolated_reporting_pool(autocommit=True) as pool: + installer = await frozen_call( + installed_b21, pool, "install", notifications=notifications + ) + assert installer["manifest_objects"] == 187 + parent = PgReportingMaterializerStore(pool=pool, notifications=notifications) + case = await durable_case( + parent, + count=3, + finality="official", + required="official", + reconciliation_mode="consumer_receipt", + consumer=( + "frozen-buyer" + if artifact in {"beta15", "records", "integration", "a", "b"} + else "https://buyer.example.test/historical" + ), + legacy_definition=artifact == "beta15", + ) + # This is the actual approved B2.1 installation executing reserve / + # write / verification / finish / captured boundary / quarantine. + root, python, _, settings = installed_b21 + process_script = root / "parent_process.py" + shutil.copy2(Path(__file__).with_name("_materializer_process.py"), process_script) + destination = tmp_path / f"destination-{notifications}" + destination.mkdir(mode=0o700) + installed = {**settings, "python": list(sys.version_info[:2])} + h = DurableHarness(parent, None, pool) + async with worker( + h, + case, + destination, + python=python, + script=process_script, + installed=installed, + notifications=notifications, + ) as child: + produced = await child.event("done") + assert produced["state"] == "verified", produced + receipt_operation_2 = await asyncio.wait_for(child.process.wait(), 5) + assert receipt_operation_2 == 0 + outcome = (await case.outcomes())[0] + assert outcome.verification is not None + kwargs = { + "account": case.scope.principal.account_id, + "consumer": case.scope.principal.consumer_id, + "obligation": case.obligation.reporting_obligation_id, + "notifications": notifications, + } + before = await frozen_call( + installed_receipt_history, + pool, + "exercise", + phase="before", + receipt_count=0, + **kwargs, + ) + saved = await immutable_parent_rows(pool) + queue = await h.queue() + assert queue[1] == (("quarantined",) if notifications else ()) + async with pool.connection() as c: + old_objects = await schema_objects(c) + store = PgReportingReceiptStore(pool=pool, notifications=notifications) + await store.create_schema() + assert await store.receipt_ingestion_ready() + async with pool.connection() as c: + new_objects = await schema_objects(c) + assert {key: new_objects[key] for key in old_objects} == old_objects + assert all( + "reporting_receipt_ingestion_" in key + for key in new_objects.keys() - old_objects.keys() + ) + assert await immutable_parent_rows(pool) == saved + adjustment = ReportingAdjustmentRecord( + "frozen-adjustment", + case.config.account_id, + case.revision.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-1.50"),), + END + timedelta(seconds=5), + END + timedelta(seconds=6), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-1.50", "decimal", "USD"), + ), + ) + await store.commit_adjustment(adjustment) + verification = outcome.verification + observed = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + receipt = ReportingRevisionReceiptRecord( + case.scope, + "historical-revision-0001", + case.revision.reporting_revision_id, + outcome.reporting_materialization_id, + "accepted", + verification.verification_profile, + verification.row_count, + verification.control_totals, + datetime.fromisoformat(observed.replace("Z", "+00:00")), + observed_canonical_content_digest=verification.canonical_content_digest, + ) + request = { + "adcp_version": "3.2-rc.6", + "account": {"account_id": case.config.account_id}, + "idempotency_key": "historical-mixed-receipts", + "receipts": [receipt_to_wire(receipt)], + "adjustment_receipts": [ + { + "reporting_receipt_id": "historical-adjustment-0001", + "reporting_adjustment_id": adjustment.reporting_adjustment_id, + "adjusts_reporting_revision_id": case.revision.reporting_revision_id, + "observed_at": observed, + "status": "accepted", + "observed_adjustment_sha256": adjustment_to_wire(adjustment)[ + "canonical_adjustment_sha256" + ], + } + ], + } + response = await store.ingest_receipt_batch(request, caller=case.scope.principal) + assert [r["result"] for r in response["results"]] == ["recorded", "recorded"], response + captured = await store.read_receipt_boundaries(caller=case.scope.principal) + after = await frozen_call( + installed_receipt_history, + pool, + "exercise", + phase="after", + receipt_count=2, + **kwargs, + ) + assert before["ordinary_core"] and after["ordinary_core"] + assert before["notification_readiness"] == after["notification_readiness"] + assert ( + before["ordinary_materializer"] + == after["ordinary_materializer"] + == (artifact != "beta15") + ) + receipt_operation_1 = await store.ingest_receipt_batch( + request, caller=case.scope.principal + ) + assert receipt_operation_1 == response + assert await store.read_receipt_boundaries(caller=case.scope.principal) == captured + assert len(captured) == 2 and captured[0].account_sequence > 1 + assert await h.queue() == queue + assert await immutable_parent_rows(pool) == saved + assert await parent.materializer_ready() + print( + json.dumps( + { + "receipt_rolling": artifact, + "notifications": notifications, + "parent": B21, + "parent_process_origins": produced["origins"], + "before": before, + "after": after, + "wheel_sha256": installed_receipt_history[3]["wheel_sha256"], + "additive_objects": len(new_objects.keys() - old_objects.keys()), + "parent_manifest": 187, + "receipt_count": 2, + "capture_count": len(captured), + "quarantine_preserved": True, + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_receipt_transactions.py b/tests/conformance/reporting/test_reporting_receipt_transactions.py new file mode 100644 index 000000000..9a80deb61 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_transactions.py @@ -0,0 +1,340 @@ +"""Receipt, feed, captured dirty input, and ordinal share one rollback boundary.""" + +from copy import deepcopy + +import pytest + +from adcp.reporting.ledger.delivery import receipt_to_wire +from adcp.reporting.receipts import ReportingReceiptError + +from ._receipt_support import adjustment_for, batch_state, receipt_case, receipts, request_for + +__all__ = ["receipts"] + + +def fail_after(monkeypatch, owner, name, *, predicate=None, asynchronous=False): + original = getattr(owner, name) + if asynchronous: + + async def injected(*args, **kwargs): + result = await original(*args, **kwargs) + if predicate is None or predicate(args, kwargs): + raise RuntimeError("receipt fault") + return result + + else: + + def injected(*args, **kwargs): + result = original(*args, **kwargs) + if predicate is None or predicate(args, kwargs): + raise RuntimeError("receipt fault") + return result + + monkeypatch.setattr(owner, name, injected) + + +@pytest.mark.parametrize( + "point", + [ + "header", + "receipt", + "receipt_head", + "feed_head", + "feed", + "dirty", + "capture_head", + "account_capture_head", + "capture", + "ordinal", + ], +) +async def test_every_insertion_rolls_back_first_ordinal_in_both_notification_modes( + receipts, point, monkeypatch +): + h = receipts + s = await receipt_case(h) + request = request_for(s) + before = await h.image() + store_class = type(h.store) + if h.pool is None: + if point in {"receipt", "receipt_head", "feed_head", "feed"}: + # Memory publishes the evidence, caller feed and sequence in one + # list assignment; there are no independently mutable receipt heads. + method = "_append_reconciliation_change" + elif point in {"capture_head", "account_capture_head"}: + # Construction occurs after both newly allocated heads changed but + # before the boundary is appended. This catches incomplete rollback + # snapshots that forget first-use collections or sequence counters. + import adcp.reporting.receipts.memory as memory + + fail_after(monkeypatch, memory, "ReportingReceiptBoundary") + method = None + else: + method = { + "header": "_receipt_batch", + "dirty": "_capture_receipt", + "capture": "_capture_receipt", + "ordinal": "_append_receipt_result", + }[point] + if method is not None: + fail_after(monkeypatch, store_class, method) + else: + if point == "receipt_head": + async with h.pool.connection() as c: + await c.execute( + "CREATE FUNCTION receipt_test_fault() RETURNS trigger LANGUAGE plpgsql " + "AS $$ BEGIN RAISE EXCEPTION 'receipt fault'; END $$" + ) + await c.execute( + "CREATE TRIGGER receipt_test_fault AFTER INSERT OR UPDATE ON " + "reporting_receipt_heads FOR EACH ROW EXECUTE FUNCTION receipt_test_fault()" + ) + elif point in {"feed_head", "capture_head", "account_capture_head"}: + from psycopg import AsyncConnection + + needle = { + "feed_head": "INSERT INTO reporting_reconciliation_heads", + "capture_head": "INSERT INTO reporting_receipt_ingestion_heads", + "account_capture_head": ( + "UPDATE reporting_materializer_accounts SET captured_sequence" + ), + }[point] + fail_after( + monkeypatch, + AsyncConnection, + "execute", + predicate=lambda args, _: isinstance(args[1], str) and needle in args[1], + asynchronous=True, + ) + else: + method = { + "header": "_receipt_batch_on", + "receipt": "_insert", + "feed": "_append_reconciliation_change", + "dirty": "_capture_receipt_on", + "capture": "_capture_receipt_on", + "ordinal": "_insert_receipt_result_on", + }[point] + fail_after(monkeypatch, store_class, method, asynchronous=True) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + monkeypatch.undo() + assert await h.image() == before + if h.pool is not None and point == "receipt_head": + async with h.pool.connection() as c: + await c.execute("DROP TRIGGER receipt_test_fault ON reporting_receipt_heads") + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert result["results"][0]["result"] == "recorded" + + +@pytest.mark.parametrize("point", ["persistence", "assembly"]) +async def test_final_response_fault_preserves_prior_ordinals_and_original_recorded_results( + receipts, point, monkeypatch +): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + method = ( + "_assemble_receipt_response" + if point == "assembly" + else ("_save_receipt_response" if h.pool is None else "_save_receipt_response_on") + ) + fail_after( + monkeypatch, + type(h.store), + method, + asynchronous=h.pool is not None and point == "persistence", + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + monkeypatch.undo() + assert await batch_state(h) == ((2, False),) + original = await h.store.get_receipt(s.receipt.key) + before = await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) + assert len(before) == 2 + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert [r["result"] for r in result["results"]] == ["recorded", "recorded"] + assert result["results"][0]["receipt"] == receipt_to_wire(original) + assert await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) == before + assert await batch_state(h) == ((2, True),) + + +async def test_second_ordinal_failure_preserves_successful_sibling_and_durable_failure( + receipts, monkeypatch +): + h = receipts + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + request = request_for( + s, + receipts=[ + { + **receipt_to_wire(s.receipt), + "reporting_receipt_id": "missing-first-0001", + "reporting_revision_id": "missing", + }, + receipt_to_wire(s.receipt), + ], + adjustment_receipts=[adjustment], + ) + if h.pool is None: + # Fail after appending the second ordinal: its newly recorded receipt, + # private capture and result must roll back; ordinal-zero failure stays. + fail_after( + monkeypatch, + type(h.store), + "_append_receipt_result", + predicate=lambda args, _: len(args[1].results) == 2, + ) + else: + fail_after( + monkeypatch, + type(h.store), + "_insert_receipt_result_on", + predicate=lambda args, _: args[4] == 1, + asynchronous=True, + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + monkeypatch.undo() + assert await batch_state(h) == ((1, False),) + assert await h.store.get_receipt(s.receipt.key) is None + assert await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) == () + changed = deepcopy(request) + changed["receipts"][0]["reporting_revision_id"] = s.revision.reporting_revision_id + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(changed, caller=s.attempt.scope.principal) + assert error.value.code == "IDEMPOTENCY_CONFLICT" + assert await h.image() == before + result = await h.store.ingest_receipt_batch(request, caller=s.attempt.scope.principal) + assert [r["result"] for r in result["results"]] == ["failed", "recorded", "recorded"] + + +async def test_captured_inputs_remain_private_and_frozen_after_current_state_changes(receipts): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/one") + sibling = await receipt_case(h, consumer_id="https://buyer.example.test/two") + await h.store.ingest_receipt_batch(request_for(s), caller=s.attempt.scope.principal) + frozen = await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) + assert len(frozen) == 1 + assert not await h.store.read_receipt_boundaries(caller=sibling.attempt.scope.principal) + await h.store.ingest_receipt_batch(request_for(sibling), caller=sibling.attempt.scope.principal) + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + assert await h.store.read_receipt_boundaries(caller=s.attempt.scope.principal) == frozen + assert all( + ( + r.scope.consumer_id == s.binding.consumer_id + if hasattr(r, "scope") + else r.consumer_id == s.binding.consumer_id + ) + for r in frozen[0].reconciliation + ) + assert frozen[0].core.consumer_ids == (s.binding.consumer_id,) + assert all(r.readable for r in frozen[0].core.revisions) + other = await h.store.read_receipt_boundaries(caller=sibling.attempt.scope.principal) + assert frozen[0].account_sequence < other[0].account_sequence + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_ordinary_status_dirty_insert_failure_rolls_back_receipt_and_ordinal( + backend, monkeypatch +): + from ._receipt_support import receipt_harness + + async with receipt_harness(backend, notifications=True) as h: + s = await receipt_case(h) + before = await h.image() + if h.pool is None: + fail_after(monkeypatch, type(h.store), "_dirty_status") + else: + from psycopg import AsyncConnection + + fail_after( + monkeypatch, + AsyncConnection, + "execute", + asynchronous=True, + predicate=lambda args, _: isinstance(args[1], str) + and args[1].startswith("INSERT INTO reporting_status_dirty"), + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + monkeypatch.undo() + assert await h.image() == before + receipt_operation_1 = await h.store.ingest_receipt_batch( + request_for(s), caller=s.binding.principal + ) + assert (receipt_operation_1)["results"][0]["result"] == "recorded" + + +@pytest.mark.parametrize("damage", ["id", "kind", "outcome", "received_at", "ordinal"]) +async def test_corrupt_durable_prefix_blocks_resume_before_sibling_mutation( + receipts, monkeypatch, damage +): + from adcp.reporting.canonical_json import canonical_json_sha256_v1 + from adcp.reporting.receipts.wire import ReceiptBatch + + h = receipts + s = await receipt_case(h) + request = request_for(s, adjustment_receipts=[await adjustment_for(h, s)]) + # Retain the first committed ordinal, then corrupt only that immutable + # prefix through a deliberately privileged fixture. Restore guards before + # asking the actual store to resume. + method = "_append_receipt_result" if h.pool is None else "_insert_receipt_result_on" + fail_after( + monkeypatch, + type(h.store), + method, + asynchronous=h.pool is not None, + predicate=( + (lambda args, _: len(args[1].results) == 2) + if h.pool is None + else (lambda args, _: args[4] == 1) + ), + ) + with pytest.raises((RuntimeError, ReportingReceiptError)): + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + monkeypatch.undo() + if h.pool is None: + state = next(iter(h.store._receipt_batches.values())) + value = deepcopy(state.results[0]) + else: + async with h.pool.connection() as c: + value = ( + await ( + await c.execute("SELECT result FROM reporting_receipt_ingestion_results") + ).fetchone() + )[0] + if damage == "id": + value["receipt"]["reporting_receipt_id"] = "unrelated-receipt-0001" + elif damage == "kind": + value["adjustment_receipt"] = value.pop("receipt") + elif damage == "outcome": + value["receipt"]["status"] = "unrecognized" + elif damage == "received_at": + del value["receipt"]["received_at"] + else: + value["reporting_receipt_id"] = "extra-ordinal" + if h.pool is None: + state.results = (value,) + else: + from psycopg.types.json import Jsonb + + async with h.pool.connection() as c, c.transaction(): + await c.execute("SET LOCAL session_replication_role = replica") + await c.execute( + "UPDATE reporting_receipt_ingestion_results SET result=%s,content_sha256=%s", + (Jsonb(value), canonical_json_sha256_v1(value)), + ) + before = await h.image() + with pytest.raises(ReportingReceiptError) as error: + await h.store.ingest_receipt_batch(request, caller=s.binding.principal) + assert error.value.code == "RECEIPT_HISTORY_CORRUPT" + assert await h.image() == before + assert len(ReceiptBatch.parse(request).items) == 2 diff --git a/tests/conformance/reporting/test_reporting_receipt_transports.py b/tests/conformance/reporting/test_reporting_receipt_transports.py new file mode 100644 index 000000000..1686a2c45 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_receipt_transports.py @@ -0,0 +1,612 @@ +"""Mounted MCP/A2A auth, canonical identity, raw wire shape and durable replay.""" + +from copy import deepcopy +from dataclasses import replace + +import pytest +from google.protobuf.json_format import MessageToDict, ParseDict +from google.protobuf.struct_pb2 import Value +from jsonschema import Draft7Validator + +from adcp.decisioning import Account, AuthInfo, RequestContext +from adcp.decisioning.registry import BuyerAgent, HttpSigCredential +from adcp.exceptions import ADCPTaskError +from adcp.reporting.receipts.wire import receipt_schema +from adcp.server import ToolContext + +from ._receipt_support import adjustment_for, receipt_case, receipt_harness, receipts, request_for +from ._receipt_transport import MountedReceipts, error_code + +__all__ = ["receipts"] + + +@pytest.mark.parametrize("hydrated", [False, True]) +async def test_actual_mounts_reauthorize_exact_replay_and_bypass_both_generic_caches( + receipts, hydrated +): + h = receipts + one = await receipt_case(h, consumer_id="https://buyer.example.test/one") + two = await receipt_case(h, consumer_id="https://buyer.example.test/two") + mount = MountedReceipts(h, hydrated=hydrated) + mount.authorize(one) + mount.authorize(two, token="token-two") + adjustment = await adjustment_for(h, one) + request = request_for( + one, + adjustment_receipts=[adjustment], + context={"buyer": "original context", "\ue000": "BMP", "\U00010000": "supplementary"}, + ) + async with mount.client() as client: + code, first = await mount.mcp(client, request) + assert code == 200 and first.get("status") == "completed", first + assert [r["result"] for r in first["results"]] == ["recorded", "recorded"] + _, replay = await mount.a2a(client, deepcopy(request)) + assert replay == first + assert "replayed" not in replay + _, sibling = await mount.mcp(client, request, token="token-two") + assert [r["result"] for r in sibling["results"]] == ["recorded", "recorded"] + assert await h.store.get_receipt(one.receipt.key) + assert await h.store.get_receipt(two.receipt.key) + assert len(mount.auth_calls) == 3 + if hydrated: + assert len({c.caller_identity for c in mount.contexts}) == 1 + mount.grants.remove((one.obligation.account_id, one.binding.consumer_id)) + before = await h.image() + for call in (mount.mcp, mount.a2a): + _, denied = await call(client, request) + assert error_code(denied) == "UNAUTHORIZED" + assert await h.image() == before + assert len(mount.auth_calls) == 5 + del mount.tokens["token-one"] + for call in (mount.mcp, mount.a2a): + code, _ = await call(client, request) + assert code == 401 + + +async def test_same_consumer_key_across_two_accounts_is_not_a_generic_cache_collision(receipts): + h = receipts + first = await receipt_case(h, account_id="account:a", consumer_id="b:c") + second = await receipt_case(h, account_id="account:a:b", consumer_id="b:c") + mount = MountedReceipts(h) + mount.authorize(first) + mount.grants.add((second.obligation.account_id, second.binding.consumer_id)) + mount.accounts[second.obligation.account_id] = second.obligation.account_id + async with mount.client() as client: + _, a = await mount.mcp(client, request_for(first)) + _, b = await mount.a2a(client, request_for(second)) + assert a["results"][0]["result"] == b["results"][0]["result"] == "recorded" + _, replay = await mount.mcp(client, request_for(second)) + assert replay == b + changed = request_for(second, context={"changed": True}) + _, conflict = await mount.a2a(client, changed) + assert error_code(conflict) == "IDEMPOTENCY_CONFLICT" + + +@pytest.mark.parametrize("first_route", ["mcp", "a2a"]) +@pytest.mark.parametrize("v1", [False, True]) +async def test_protobuf_integer_spellings_preserve_whole_request_identity_and_timestamps( + receipts, first_route, v1 +): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, hydrated=True, version="3.2-rc.6") + mount.authorize(s) + request = request_for( + s, + context={"integer": 42, "nested": [100], "bounds": [-9007199254740991, 9007199254740991]}, + ) + # Use the actual protobuf conversion that an A2A client performs. Python's + # JSON printer then emits 1.0, so reading the original HTTP bytes must accept + # that exact integral spelling without rounding arbitrary fractional input. + proto = Value() + ParseDict(request, proto) + protobuf_request = MessageToDict(proto) + assert type(protobuf_request["receipts"][0]["observed_row_count"]) is float + assert type(protobuf_request["context"]["integer"]) is float + assert Draft7Validator(receipt_schema("request", version="3.2-rc.6")).is_valid(protobuf_request) + requests = {"mcp": request, "a2a": protobuf_request} + other = "a2a" if first_route == "mcp" else "mcp" + async with mount.client() as client: + _, first = await getattr(mount, first_route)( + client, requests[first_route], **({"v1": v1} if first_route == "a2a" else {}) + ) + assert first["results"][0]["result"] == "recorded", first + _, replay = await getattr(mount, other)( + client, requests[other], **({"v1": v1} if other == "a2a" else {}) + ) + assert replay == first + assert replay["context"] == request["context"] + _, exponent = await mount.mcp( + client, + request, + mutate_wire=lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1.0e+0' + ), + ) + assert exponent == first + assert len(mount.auth_calls) == 3 + + +@pytest.mark.parametrize("chunk_size", [1, 257]) +async def test_request_bound_capture_preserves_chunked_bytes_and_read_ahead_overflow( + receipts, monkeypatch, chunk_size +): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + # Shorten only the read-ahead window; the raw capture still has its real + # 10 MiB bound. The rest of this valid request must reach the normal decoder. + monkeypatch.setattr("adcp.reporting.receipts.transport.MAX_RECEIPT_BODY_BYTES", 256) + request = request_for(s, context={"original": "same request bytes"}) + + def chunked(wire): + async def chunks(): + raw = wire.encode() + for i in range(0, len(raw), chunk_size): + yield b"" + yield raw[i : i + chunk_size] + + return chunks() + + async with mount.client() as client: + _, first = await mount.a2a(client, request, mutate_wire=chunked) + assert first["results"][0]["result"] == "recorded", first + receipt_operation_1 = await mount.mcp(client, request) + assert (receipt_operation_1)[1] == first + assert len(mount.auth_calls) == 2 + + +@pytest.mark.parametrize("registry_kind", ["api_key", "oauth", "http_sig"]) +async def test_api_oauth_signed_registry_identity_is_refreshed_on_mounted_replay( + receipts, registry_kind +): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + mount = MountedReceipts(h, hydrated=True, registry_kind=registry_kind) + mount.authorize(s) + async with mount.client() as client: + _, first = await mount.mcp(client, request_for(s)) + assert first["results"][0]["result"] == "recorded", first + _, replay = await mount.a2a(client, request_for(s)) + assert replay == first + assert len(mount.registry.calls) >= 2 + for key, agent in mount.registry.agents.items(): + mount.registry.agents[key] = replace(agent, status="blocked") + before = await h.image() + _, denied = await mount.a2a(client, request_for(s)) + assert error_code(denied) == "UNAUTHORIZED" + assert await h.image() == before + + +@pytest.mark.parametrize( + "bad", + [ + "absent", + "anonymous", + "tenant_only", + "cache_only", + "auth_conflict", + "signed_conflict", + "agent_conflict", + "metadata_conflict", + "account_conflict", + ], +) +async def test_all_trusted_identities_must_agree_and_cache_and_tenant_never_identify_consumer(bad): + async with receipt_harness("memory") as h: + s = await receipt_case(h, consumer_id="https://buyer.example.test/agent") + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + consumer = s.binding.consumer_id + context = RequestContext( + caller_identity="cache-only", + account=Account(id=s.obligation.account_id), + auth_info=AuthInfo(kind="bearer", principal=consumer, credential=None), + auth_principal=consumer, + buyer_agent=BuyerAgent(consumer, "Buyer", "active"), + ) + if bad == "absent": + context = ToolContext() + elif bad == "anonymous": + context = ToolContext(caller_identity="anonymous") + elif bad == "tenant_only": + context = ToolContext(tenant_id=consumer) + elif bad == "cache_only": + context = replace( + context, + auth_info=None, + auth_principal=None, + buyer_agent=None, + caller_identity=consumer, + ) + elif bad == "auth_conflict": + context = replace( + context, auth_info=AuthInfo(kind="bearer", principal="other", credential=None) + ) + elif bad == "signed_conflict": + context = replace( + context, + auth_info=AuthInfo( + kind="http_sig", + principal=consumer, + credential=HttpSigCredential( + "http_sig", "key", "https://other.example.test/agent", 1.0 + ), + ), + ) + elif bad == "agent_conflict": + context = replace( + context, + buyer_agent=BuyerAgent("https://other.example.test/agent", "Other", "active"), + ) + elif bad == "metadata_conflict": + context.metadata["adcp.auth_info"] = AuthInfo( + kind="bearer", principal="other", credential=None + ) + else: + context = replace(context, account=replace(context.account, id="other-account")) + before = await h.image() + with pytest.raises(ADCPTaskError) as error: + await mount.handler.sync_reporting_receipts(request_for(s), context) + assert error.value.errors[0].code == "UNAUTHORIZED" + assert await h.image() == before + + +@pytest.mark.parametrize( + "bad", ["empty", "empty_adjustments", "received_at", "duplicate", "combined101", "spoof"] +) +async def test_actual_mount_shape_failure_is_task_invalid_request_with_no_writes(bad): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + request = request_for(s) + if bad == "empty": + request["receipts"] = [] + elif bad == "empty_adjustments": + request["adjustment_receipts"] = [] + elif bad == "received_at": + request["receipts"][0]["received_at"] = "2026-09-01T02:00:00Z" + elif bad == "duplicate": + request["receipts"] *= 2 + elif bad == "spoof": + request["consumer_id"] = "someone-else" + else: + request["receipts"] = [ + {**request["receipts"][0], "reporting_receipt_id": f"receipt-{i:016d}"} + for i in range(101) + ] + before = await h.image() + async with mount.client(validation=None) as client: + for call in (mount.mcp, mount.a2a): + _, response = await call(client, request) + assert error_code(response) == "INVALID_REQUEST" + assert await h.image() == before + assert mount.auth_calls == [] + + +@pytest.mark.parametrize("version", [None, "3.2-rc.6"]) +@pytest.mark.parametrize("fallback", [False, True]) +async def test_pinned_unpinned_and_fallback_mcp_schemas_keep_combined_and_received_rules( + version, fallback, monkeypatch +): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + adjustment = await adjustment_for(h, s) + mount = MountedReceipts(h, version=version) + mount.authorize(s) + if fallback: + monkeypatch.setattr( + "adcp.server.mcp_tools._ensure_pydantic_schemas_applied", lambda _: None + ) + async with mount.client() as client: + _, inventory = await mount.mcp(client, inventory=True) + definition = next( + t for t in inventory["tools"] if t["name"] == "sync_reporting_receipts" + ) + request = request_for(s, adjustment_receipts=[adjustment]) + validator = Draft7Validator(definition["inputSchema"]) + assert validator.is_valid(request) + assert validator.is_valid({k: v for k, v in request.items() if k != "receipts"}) + for change in ( + {"receipts": []}, + {"adjustment_receipts": []}, + {"receipts": [{**request["receipts"][0], "received_at": "2026-09-01T02:00:00Z"}]}, + {"receipts": request["receipts"] * 51, "adjustment_receipts": [adjustment] * 50}, + ): + assert not validator.is_valid({**request, **change}) + absent = { + k: v for k, v in request.items() if k not in {"receipts", "adjustment_receipts"} + } + assert not validator.is_valid(absent) + _, result = await mount.mcp(client, request) + assert Draft7Validator(definition["outputSchema"]).is_valid(result), result + del result["results"][0]["receipt"]["received_at"] + assert not Draft7Validator(definition["outputSchema"]).is_valid(result) + # Per-mount overlays never mutate another caller's cached schema. + definition["inputSchema"]["anyOf"] = [] + assert receipt_schema("request")["anyOf"] + + +async def test_receipt_hook_rewrite_rejected_and_response_enhancer_cannot_mutate_replay(): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + request = request_for(s) + original = deepcopy(request) + + def rewrite(task, params): + params["receipts"][0]["consumer_commit_ref"] = "changed" + return params + + async with mount.client( + pre_validation_hooks={"sync_reporting_receipts": [rewrite]} + ) as client: + _, error = await mount.mcp(client, request) + assert error_code(error) == "INVALID_REQUEST" + assert request == original + assert not await h.store.get_receipt(s.receipt.key) + mount.sessions.clear() + + def enhancer(*args): + raise AssertionError("a receipt response must not be enhanced") + + async with mount.client(response_enhancer=enhancer) as client: + _, first = await mount.mcp(client, request) + _, replay = await mount.a2a(client, request) + assert first == replay and first["results"][0]["result"] == "recorded" + + +@pytest.mark.parametrize("v1", [False, True]) +async def test_a2a_to_mcp_replay_after_mount_restart_keeps_every_original_ordinal_and_timestamp( + receipts, v1 +): + h = receipts + s = await receipt_case(h, consumer_id="https://buyer.example.test/restarted") + adjustment = await adjustment_for(h, s) + request = request_for(s, adjustment_receipts=[adjustment]) + request["receipts"].insert( + 0, + { + **request["receipts"][0], + "reporting_receipt_id": "failed-ordinal-0000", + "reporting_revision_id": "unknown", + }, + ) + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + async with mount.client() as client: + _, first = await mount.a2a(client, request, v1=v1) + assert [r["result"] for r in first["results"]] == ["failed", "recorded", "recorded"], first + if h.pool is not None: + from adcp.reporting.receipts import PgReportingReceiptStore + + h.store = PgReportingReceiptStore(pool=h.pool, notifications=h.store._notifications_enabled) + restarted = MountedReceipts(h, hydrated=True) + restarted.authorize(s) + async with restarted.client() as client: + _, replay = await restarted.mcp(client, request) + assert replay == first + _, again = await restarted.a2a(client, request, v1=v1) + assert again == first + assert len(restarted.auth_calls) == 2 + assert "replayed" not in again + + +@pytest.mark.parametrize( + "fault", + [ + "fractional", + "rounded_fraction", + "deep_rounded_fraction", + "unsafe_integer", + "unsafe_exponent", + "unrepresentable_exponent", + "negative_integer", + "nonfinite", + "boolean_count", + "numeric_string_field", + "duplicate_key", + "duplicate_nested", + "empty_array", + "metadata_shortcut", + "unpaired_surrogate", + ], +) +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_real_raw_strict_json_and_numeric_admission_fail_before_batch_mutation( + receipts, fault, route +): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, version="3.2-rc.6") + mount.authorize(s) + request = request_for(s) + mutations = { + "fractional": lambda w: w.replace('"observed_row_count": 1', '"observed_row_count": 1.5'), + "rounded_fraction": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1.0000000000000001' + ), + "deep_rounded_fraction": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1.000000000000000000001' + ), + "unsafe_integer": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 9007199254740993' + ), + "unsafe_exponent": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 9.007199254740993e15' + ), + "unrepresentable_exponent": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 1e99999999999999999999999' + ), + "negative_integer": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": -1.0' + ), + "nonfinite": lambda w: w.replace('"observed_row_count": 1', '"observed_row_count": NaN'), + "boolean_count": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": true' + ), + "duplicate_key": lambda w: w.replace( + '"account": {', '"account": {"account_id":"other"}, "account": {' + ), + "duplicate_nested": lambda w: w.replace( + '"observed_row_count": 1', '"observed_row_count": 999, "observed_row_count": 1' + ), + } + if fault == "empty_array": + request["receipts"] = [] + if fault == "unpaired_surrogate": + request["context"] = {"unicode": "\ud800"} + if fault == "numeric_string_field": + request["receipts"][0]["observed_control_totals"][0]["value"] = 1.0 + if fault == "metadata_shortcut": + request["receipts"] = [] + request["context"] = { + "adcp.receipt_ingress.raw_body": request_for(s), + "adcp.a2a_parsed_request": request_for(s), + } + before = await h.image() + async with mount.client(validation=None) as client: + _, result = await getattr(mount, route)(client, request, mutate_wire=mutations.get(fault)) + if fault == "unpaired_surrogate": + assert result["error"]["code"] == -32700, result + else: + assert error_code(result) == "INVALID_REQUEST", result + assert await h.image() == before + assert mount.auth_calls == [] + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_malformed_numeric_token_is_rejected_by_the_actual_json_decoder(receipts, route): + h = receipts + s = await receipt_case(h) + mount = MountedReceipts(h, version="3.2-rc.6") + mount.authorize(s) + request = request_for(s) + before = await h.image() + async with mount.client() as client: + status, response = await getattr(mount, route)( + client, + request, + mutate_wire=lambda w: w.replace('"observed_row_count": 1', '"observed_row_count": 1e'), + ) + # This invalid JSON cannot reach task admission. Preserve each real + # transport's parse-error envelope instead of inventing a task result. + assert status in {200, 400, 422}, response + if status == 200: + assert response["error"]["code"] in {-32700, -32600, -32602}, response + assert await h.image() == before + assert mount.auth_calls == [] + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_raw_capture_cannot_supply_auth_or_cross_request_body_or_override_actual_account( + route, +): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + request = request_for(s) + async with mount.client() as client: + call = getattr(mount, route) + _, first = await call(client, request) + assert first["results"][0]["result"] == "recorded" + before = await h.image() + _, invalid = await call(client, {**request, "receipts": []}) + assert error_code(invalid) == "INVALID_REQUEST" + changed = { + **request, + "account": {"account_id": "other"}, + "context": {"adcp.receipt_ingress.raw_body": request}, + } + _, denied = await call(client, changed) + assert error_code(denied) == "UNAUTHORIZED" + if route == "mcp": + # Avoid a fresh initialize in this helper: the authenticated + # tool request itself must reject a forged token. + mount.sessions["forged"] = mount.sessions["token-one"] + receipt_operation_6 = await call(client, request, token="forged") + assert (receipt_operation_6)[0] == 401 + assert await h.image() == before + + +async def test_unsupported_version_rejected_on_both_routes_before_ingress(): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + before = await h.image() + async with mount.client() as client: + for call in (mount.mcp, mount.a2a): + _, denied = await call(client, request_for(s, adcp_version="99.0")) + assert error_code(denied) == "VERSION_UNSUPPORTED" + assert await h.image() == before + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_raw_capture_size_bound_and_fallback_are_closed(monkeypatch, route): + from adcp.reporting.receipts.transport import receipt_body_receive + + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h) + mount.authorize(s) + + def bounded(scope, receive, **kwargs): + return receipt_body_receive(scope, receive, limit=128) + + monkeypatch.setattr("adcp.reporting.receipts.transport.receipt_body_receive", bounded) + before = await h.image() + async with mount.client() as client: + _, result = await getattr(mount, route)(client, request_for(s)) + assert error_code(result) == "INVALID_REQUEST" + assert await h.image() == before + + +@pytest.mark.parametrize("route", ["mcp", "a2a"]) +async def test_generic_middleware_cannot_replace_the_authenticated_whole_request(route): + async with receipt_harness("memory") as h: + s = await receipt_case(h) + mount = MountedReceipts(h, hydrated=True) + mount.authorize(s) + request = request_for(s, context={"attempt": 1}) + + async def rewrite(name, params, context, call_next): + params["context"]["attempt"] = 2 + return await call_next() + + mount.middleware = rewrite + before = await h.image() + async with mount.client() as client: + _, result = await getattr(mount, route)(client, request) + assert error_code(result) == "INVALID_REQUEST" + assert await h.image() == before + assert mount.auth_calls == [] + + +async def test_raw_mounted_requests_do_not_alias_account_consumer_pairs_or_cached_context(receipts): + h = receipts + first = await receipt_case(h, account_id="acct:a", consumer_id="b:c") + second = await receipt_case(h, account_id="acct:a:b", consumer_id="c") + mount = MountedReceipts(h, hydrated=True) + mount.authorize(first) + mount.authorize(second, token="token-two") + async with mount.client() as client: + _, a = await mount.mcp(client, request_for(first)) + _, b = await mount.a2a(client, request_for(second), token="token-two") + assert a["results"][0]["result"] == b["results"][0]["result"] == "recorded" + assert a != b + receipt_operation_2 = await mount.a2a(client, request_for(first)) + assert (receipt_operation_2)[1] == a + receipt_operation_3 = await mount.mcp(client, request_for(second), token="token-two") + assert (receipt_operation_3)[1] == b + before = await h.image() + receipt_operation_4 = await mount.a2a(client, request_for(second)) + assert error_code((receipt_operation_4)[1]) == "UNAUTHORIZED" + receipt_operation_5 = await mount.mcp(client, request_for(first), token="token-two") + assert error_code((receipt_operation_5)[1]) == "UNAUTHORIZED" + assert await h.image() == before + assert len({c.caller_identity for c in mount.contexts}) == 1 diff --git a/tests/type_checks/reporting_receipt_ingress.py b/tests/type_checks/reporting_receipt_ingress.py new file mode 100644 index 000000000..c4f65756c --- /dev/null +++ b/tests/type_checks/reporting_receipt_ingress.py @@ -0,0 +1,40 @@ +"""Additive receipt composition keeps existing stores and structural protocols.""" + +from typing import Any + +from typing_extensions import assert_type + +from adcp.reporting.ledger import ReportingDeliveryPrincipal +from adcp.reporting.materializer import ReportingMaterializerStore +from adcp.reporting.receipts import ( + InMemoryReportingReceiptStore, + PgReportingReceiptStore, + ReceiptAccountResolver, + ReportingReceiptBatchStore, + ReportingReceiptBoundary, + ReportingReceiptCaptureStore, + ReportingReceiptHandler, +) + + +async def adopter( + postgres: PgReportingReceiptStore, + memory: InMemoryReportingReceiptStore, + caller: ReportingDeliveryPrincipal, + request: dict[str, Any], + resolve_account: ReceiptAccountResolver, +) -> ReportingMaterializerStore: + store: ReportingReceiptBatchStore = postgres + store = memory + capture: ReportingReceiptCaptureStore = postgres + capture = memory + materializer: ReportingMaterializerStore = postgres + materializer = memory + assert_type(await store.ingest_receipt_batch(request, caller=caller), dict[str, Any]) + assert_type( + await capture.read_receipt_boundaries(caller=caller), tuple[ReportingReceiptBoundary, ...] + ) + handler = ReportingReceiptHandler(store, resolve_account=resolve_account) + assert_type(handler.receipt_store, ReportingReceiptBatchStore) + assert_type(await postgres.receipt_ingestion_ready(), bool) + return materializer