From 614d513caf2512b7e2bb56942cbef4251da3f0eb Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 18 Sep 2026 21:37:16 +0000 Subject: [PATCH 1/6] feat(reporting): enable production tier status and ownership Refs #1167, #1179, #1180. B2.4 of 4 completing B2 of B1/B2: versioned captured status, private ownership, production component admission, and atomic notification delivery. Preserve historical artifacts and separate final-head validation from development controls. Independent review and downstream gates remain open. --- .github/workflows/ci.yml | 167 ++- .github/workflows/pr-title-check.yml | 2 +- docs/reporting-production.md | 301 ++++ examples/reporting_production.py | 121 ++ pyproject.toml | 2 + scripts/post_generate_fixes.py | 119 ++ src/adcp/reporting/_reconcile.py | 208 ++- src/adcp/reporting/_timestamp.py | 43 + src/adcp/reporting/feed/memory.py | 4 + src/adcp/reporting/feed/pg.py | 8 +- src/adcp/reporting/feed/projection.py | 78 +- src/adcp/reporting/feed/snapshot.py | 49 +- src/adcp/reporting/ledger/models.py | 73 +- src/adcp/reporting/ledger/producer.py | 161 +- .../reporting/ledger/producer_progress.py | 93 ++ .../ledger/reconciliation_projection.py | 349 +++++ .../reporting/ledger/reporting_production.sql | 669 +++++++++ .../reporting/ledger/reporting_projection.sql | 242 +++ .../ledger/reporting_projection_feed.sql | 43 + .../reporting_projection_notifications.sql | 296 ++++ src/adcp/reporting/ledger/schedule.py | 95 ++ src/adcp/reporting/ledger/status.py | 97 +- .../reporting/ledger/status_projection.py | 92 +- src/adcp/reporting/ledger/status_snapshot.py | 9 +- src/adcp/reporting/ledger/store.py | 5 + src/adcp/reporting/materializer/memory.py | 63 +- src/adcp/reporting/materializer/pg.py | 2 + .../reporting/materializer/publication.py | 63 + src/adcp/reporting/materializer/work.py | 3 + src/adcp/reporting/outbox/_activity_pg.py | 83 +- src/adcp/reporting/outbox/memory.py | 51 +- src/adcp/reporting/outbox/status_memory.py | 48 +- src/adcp/reporting/outbox/status_pg.py | 21 +- src/adcp/reporting/outbox/worker.py | 66 +- src/adcp/reporting/ownership.py | 129 ++ src/adcp/reporting/production/__init__.py | 67 + .../reporting/production/configuration.py | 335 +++++ src/adcp/reporting/production/contracts.py | 221 +++ .../reporting/production/delivery_window.py | 160 ++ src/adcp/reporting/production/handler.py | 316 ++++ src/adcp/reporting/production/memory.py | 380 +++++ .../reporting/production/notifications.py | 364 +++++ src/adcp/reporting/production/offerings.py | 352 +++++ src/adcp/reporting/production/pg.py | 755 ++++++++++ .../reporting/production/required_schema.json | 1326 +++++++++++++++++ src/adcp/reporting/production/schema.py | 24 + src/adcp/reporting/production/service.py | 689 +++++++++ src/adcp/reporting/projection/__init__.py | 36 + src/adcp/reporting/projection/capture.py | 191 +++ src/adcp/reporting/projection/history.py | 119 ++ src/adcp/reporting/projection/memory.py | 411 +++++ .../reporting/projection/notifications.py | 41 + src/adcp/reporting/projection/pg.py | 632 ++++++++ .../reporting/projection/required_schema.json | 1270 ++++++++++++++++ src/adcp/reporting/projection/schema.py | 26 + src/adcp/reporting/projection/wire.py | 161 ++ src/adcp/reporting/receipts/handler.py | 37 +- src/adcp/server/a2a_server.py | 5 + src/adcp/server/serve.py | 4 + src/adcp/types/__init__.py | 4 + src/adcp/types/_eager.py | 4 + src/adcp/types/base.py | 30 +- .../get_adcp_capabilities_response.py | 59 +- .../core/reporting_delivery_capabilities.py | 59 +- src/adcp/validation/schema_loader.py | 81 +- .../reporting/_production_delivery_process.py | 133 ++ .../reporting/_production_installed.py | 142 ++ .../_production_installed_process.py | 283 ++++ .../reporting/_production_legacy_process.py | 137 ++ .../reporting/_production_packaging.py | 270 ++++ .../reporting/_production_progress_process.py | 57 + .../reporting/_production_support.py | 637 ++++++++ .../reporting/_production_transport.py | 35 + .../reporting/_projection_support.py | 75 + .../reporting/test_reporting_feed_rolling.py | 15 + .../reporting/test_reporting_production.py | 124 ++ .../test_reporting_production_bindings.py | 228 +++ ...test_reporting_production_configuration.py | 277 ++++ .../test_reporting_production_lifecycle.py | 297 ++++ .../test_reporting_production_lock_order.py | 265 ++++ .../test_reporting_production_migration.py | 252 ++++ ...test_reporting_production_notifications.py | 688 +++++++++ .../test_reporting_production_packaging.py | 56 + .../test_reporting_production_progress.py | 603 ++++++++ .../test_reporting_production_readiness.py | 381 +++++ .../test_reporting_production_restart.py | 108 ++ .../test_reporting_production_rolling.py | 409 +++++ ...est_reporting_production_signing_schema.py | 118 ++ .../test_reporting_production_transactions.py | 406 +++++ .../test_reporting_projection_capture.py | 143 ++ .../test_reporting_projection_history.py | 261 ++++ ...st_reporting_projection_memory_rollback.py | 160 ++ ...test_reporting_projection_notifications.py | 72 + .../test_reporting_projection_schedule.py | 174 +++ .../test_reporting_projection_timestamps.py | 114 ++ .../test_reporting_projection_wire.py | 77 + .../test_reporting_schedule_schema.py | 251 ++++ .../test_reporting_tier_projection.py | 152 ++ tests/fixtures/public_api_snapshot.json | 2 + tests/test_reporting_capability_models.py | 174 +++ tests/test_reporting_production_public.py | 50 + tests/test_reporting_revision_ownership.py | 243 +++ tests/test_schema_datetime_formats.py | 136 ++ tests/test_schema_loader_per_version.py | 75 + 104 files changed, 19781 insertions(+), 313 deletions(-) create mode 100644 docs/reporting-production.md create mode 100644 examples/reporting_production.py create mode 100644 src/adcp/reporting/_timestamp.py create mode 100644 src/adcp/reporting/ledger/producer_progress.py create mode 100644 src/adcp/reporting/ledger/reconciliation_projection.py create mode 100644 src/adcp/reporting/ledger/reporting_production.sql create mode 100644 src/adcp/reporting/ledger/reporting_projection.sql create mode 100644 src/adcp/reporting/ledger/reporting_projection_feed.sql create mode 100644 src/adcp/reporting/ledger/reporting_projection_notifications.sql create mode 100644 src/adcp/reporting/ledger/schedule.py create mode 100644 src/adcp/reporting/materializer/publication.py create mode 100644 src/adcp/reporting/ownership.py create mode 100644 src/adcp/reporting/production/__init__.py create mode 100644 src/adcp/reporting/production/configuration.py create mode 100644 src/adcp/reporting/production/contracts.py create mode 100644 src/adcp/reporting/production/delivery_window.py create mode 100644 src/adcp/reporting/production/handler.py create mode 100644 src/adcp/reporting/production/memory.py create mode 100644 src/adcp/reporting/production/notifications.py create mode 100644 src/adcp/reporting/production/offerings.py create mode 100644 src/adcp/reporting/production/pg.py create mode 100644 src/adcp/reporting/production/required_schema.json create mode 100644 src/adcp/reporting/production/schema.py create mode 100644 src/adcp/reporting/production/service.py create mode 100644 src/adcp/reporting/projection/__init__.py create mode 100644 src/adcp/reporting/projection/capture.py create mode 100644 src/adcp/reporting/projection/history.py create mode 100644 src/adcp/reporting/projection/memory.py create mode 100644 src/adcp/reporting/projection/notifications.py create mode 100644 src/adcp/reporting/projection/pg.py create mode 100644 src/adcp/reporting/projection/required_schema.json create mode 100644 src/adcp/reporting/projection/schema.py create mode 100644 src/adcp/reporting/projection/wire.py create mode 100644 tests/conformance/reporting/_production_delivery_process.py create mode 100644 tests/conformance/reporting/_production_installed.py create mode 100644 tests/conformance/reporting/_production_installed_process.py create mode 100644 tests/conformance/reporting/_production_legacy_process.py create mode 100644 tests/conformance/reporting/_production_packaging.py create mode 100644 tests/conformance/reporting/_production_progress_process.py create mode 100644 tests/conformance/reporting/_production_support.py create mode 100644 tests/conformance/reporting/_production_transport.py create mode 100644 tests/conformance/reporting/_projection_support.py create mode 100644 tests/conformance/reporting/test_reporting_production.py create mode 100644 tests/conformance/reporting/test_reporting_production_bindings.py create mode 100644 tests/conformance/reporting/test_reporting_production_configuration.py create mode 100644 tests/conformance/reporting/test_reporting_production_lifecycle.py create mode 100644 tests/conformance/reporting/test_reporting_production_lock_order.py create mode 100644 tests/conformance/reporting/test_reporting_production_migration.py create mode 100644 tests/conformance/reporting/test_reporting_production_notifications.py create mode 100644 tests/conformance/reporting/test_reporting_production_packaging.py create mode 100644 tests/conformance/reporting/test_reporting_production_progress.py create mode 100644 tests/conformance/reporting/test_reporting_production_readiness.py create mode 100644 tests/conformance/reporting/test_reporting_production_restart.py create mode 100644 tests/conformance/reporting/test_reporting_production_rolling.py create mode 100644 tests/conformance/reporting/test_reporting_production_signing_schema.py create mode 100644 tests/conformance/reporting/test_reporting_production_transactions.py create mode 100644 tests/conformance/reporting/test_reporting_projection_capture.py create mode 100644 tests/conformance/reporting/test_reporting_projection_history.py create mode 100644 tests/conformance/reporting/test_reporting_projection_memory_rollback.py create mode 100644 tests/conformance/reporting/test_reporting_projection_notifications.py create mode 100644 tests/conformance/reporting/test_reporting_projection_schedule.py create mode 100644 tests/conformance/reporting/test_reporting_projection_timestamps.py create mode 100644 tests/conformance/reporting/test_reporting_projection_wire.py create mode 100644 tests/conformance/reporting/test_reporting_schedule_schema.py create mode 100644 tests/conformance/reporting/test_reporting_tier_projection.py create mode 100644 tests/test_reporting_capability_models.py create mode 100644 tests/test_reporting_production_public.py create mode 100644 tests/test_reporting_revision_ownership.py create mode 100644 tests/test_schema_datetime_formats.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a501fbac..deb230d57 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,6 +12,7 @@ on: - conductor/1167b2-durable-managed-reporting - conductor/reporting-receipt-ingress-b22 - conductor/reporting-frozen-account-feed-b23 + - conductor/reporting-schema-proof-receipt-diagnostics-hardening # Default @adcp/sdk runner alias for storyboard jobs. Tracks the current # stable @adcp/sdk release via the ``latest`` npm dist-tag. @@ -106,7 +107,7 @@ jobs: - name: Run adopter type-check suite if: matrix.python-version == '3.12' - run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py examples/reporting_receipt_ingress.py + run: mypy --strict tests/type_checks/ examples/reporting_webhook_activity.py examples/reporting_status_notifications.py examples/reporting_destination_writer.py examples/reporting_durable_materializer.py examples/reporting_receipt_ingress.py examples/reporting_production.py - name: Enforce adopter type-check fixture contract if: matrix.python-version == '3.12' @@ -197,6 +198,10 @@ jobs: --ignore-glob='tests/conformance/reporting/test_reporting_status_*.py' \ --ignore-glob='tests/conformance/reporting/test_reporting_receipt_*.py' \ --ignore-glob='tests/conformance/reporting/test_reporting_feed_*.py' \ + --ignore-glob='tests/conformance/reporting/test_reporting_production*.py' \ + --ignore-glob='tests/conformance/reporting/test_reporting_projection*.py' \ + --ignore=tests/conformance/reporting/test_reporting_tier_projection.py \ + --ignore=tests/conformance/reporting/test_reporting_schedule_schema.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_rolling.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_process.py \ --ignore=tests/conformance/reporting/test_reporting_materializer_migration.py \ @@ -610,6 +615,166 @@ jobs: ${{ runner.temp }}/hardening-installed-evidence if-no-files-found: error + pg-reporting-production: + name: Production reporting status, ownership and notification contracts + runs-on: ubuntu-latest + timeout-minutes: 35 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_production_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run complete production and projection conformance + shell: bash + timeout-minutes: 30 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_production_test + run: | + source_tests=() + for test in tests/conformance/reporting/test_reporting_production*.py; do + case "$test" in + *test_reporting_production_packaging.py|*test_reporting_production_rolling.py) continue ;; + esac + source_tests+=("$test") + done + python scripts/reporting_test_harness.py pytest \ + "${source_tests[@]}" \ + tests/conformance/reporting/test_reporting_projection*.py \ + tests/conformance/reporting/test_reporting_tier_projection.py \ + tests/conformance/reporting/test_reporting_schedule_schema.py \ + tests/test_reporting_capability_models.py \ + tests/test_reporting_revision_ownership.py \ + tests/test_reporting_production_public.py \ + tests/test_schema_datetime_formats.py \ + -v -s -ra | tee pg-reporting-production-evidence.log + - name: Preserve production contract evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-production-evidence-${{ github.run_attempt }} + path: pg-reporting-production-evidence.log + if-no-files-found: error + + pg-reporting-production-installed: + name: Installed production reporting (${{ matrix.cell }}, Python 3.10) + runs-on: ubuntu-latest + timeout-minutes: 40 + strategy: + fail-fast: false + matrix: + cell: [base-vcs, base-sdist, pg-vcs, pg-sdist] + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_production_installed_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v6 + - uses: actions/setup-python@v6 + id: production-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run installed source-equivalent production contract + shell: bash + timeout-minutes: 35 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_production_installed_test + ADCP_PYTHON310: ${{ steps.production-python310.outputs.python-path }} + ADCP_PRODUCTION_EVIDENCE: ${{ runner.temp }}/production-installed-evidence + PRODUCTION_CELL: ${{ matrix.cell }} + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_production_packaging.py \ + -k "$PRODUCTION_CELL" -v -s -ra | tee pg-reporting-production-installed.log + - name: Preserve installed origins, original inner logs and exact asset hashes + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-production-installed-${{ matrix.cell }}-${{ github.run_attempt }} + path: | + pg-reporting-production-installed.log + ${{ runner.temp }}/production-installed-evidence + if-no-files-found: error + + pg-reporting-production-compatibility: + name: B2.3 and hardening to installed B2.4 activation and restart + runs-on: ubuntu-latest + timeout-minutes: 50 + services: + postgres: + image: postgres:16 + env: + POSTGRES_HOST_AUTH_METHOD: trust + POSTGRES_DB: adcp_production_rolling_test + POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s + --health-timeout 5s --health-retries 10 + steps: + - uses: actions/checkout@v6 + - name: Fetch both independently approved artifacts + run: git fetch --no-tags origin 50e35f0ae3540f19b40e8fc460f5870dfe018bf9 a09878f67ab397a4b51b3314e3e8a5e87cf96da5 + - uses: actions/setup-python@v6 + id: production-python310 + with: + python-version: "3.10" + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: pyproject.toml + - name: Install test dependencies + run: pip install -e ".[dev,pg]" + - name: Run actual historical page one, activation, SIGKILL and cold continuations + shell: bash + timeout-minutes: 42 + env: + ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_production_rolling_test + ADCP_PYTHON310: ${{ steps.production-python310.outputs.python-path }} + ADCP_PRODUCTION_EVIDENCE: ${{ runner.temp }}/production-rolling-evidence + run: | + python scripts/reporting_test_harness.py pytest \ + tests/conformance/reporting/test_reporting_production_rolling.py \ + -v -s -ra | tee pg-reporting-production-rolling.log + - name: Preserve exact installed historical continuity and fence evidence + if: always() + uses: actions/upload-artifact@v7 + with: + name: pg-reporting-production-rolling-${{ github.run_attempt }} + path: | + pg-reporting-production-rolling.log + ${{ runner.temp }}/production-rolling-evidence + if-no-files-found: error + conventional-commits: name: Validate conventional commit format runs-on: ubuntu-latest diff --git a/.github/workflows/pr-title-check.yml b/.github/workflows/pr-title-check.yml index de0669486..3975fd4a7 100644 --- a/.github/workflows/pr-title-check.yml +++ b/.github/workflows/pr-title-check.yml @@ -3,7 +3,7 @@ name: PR Title Check on: pull_request: types: [opened, edited, synchronize, reopened] - branches: [main, conductor/1167b2-durable-managed-reporting, conductor/reporting-receipt-ingress-b22, conductor/reporting-frozen-account-feed-b23] + branches: [main, conductor/1167b2-durable-managed-reporting, conductor/reporting-receipt-ingress-b22, conductor/reporting-frozen-account-feed-b23, conductor/reporting-schema-proof-receipt-diagnostics-hardening] permissions: contents: read diff --git a/docs/reporting-production.md b/docs/reporting-production.md new file mode 100644 index 000000000..686394a23 --- /dev/null +++ b/docs/reporting-production.md @@ -0,0 +1,301 @@ +# Production reporting and versioned status + +`adcp.reporting.production` composes the existing durable producer, materializer, +receipt ingress and frozen feed with versioned private status. Use +[`examples/reporting_production.py`](../examples/reporting_production.py) for the +typed composition and authenticated MCP/A2A lifecycle. Existing Core polling +and eligible Core notification deployments keep their existing composition. + +## Provider and source contracts + +Each `ReportingProductionOffering` binds a complete public offering to one +actual `ReportingProducer`, source offering and installed verifier. A +`ReportingProductionDestination` supplies complete `ReportingProductionMethod` +values and resolves each authorized destination into a +`ReportingProductionDestinationBinding`. Pattern or transport labels alone +are insufficient: provider, destination modes, access mode, orchestration, +reader compatibility and the selected destination must match. Supply opaque +references, never credentials or bearer URLs. The same provider resolves +fresh, separately authorized write and readback sessions. + +The source implements `ReportingProductionSource.configuration_binding()`. +Construct its result with `ReportingProductionSourceBinding.for_configuration()` +using the authenticated account/catalog mapping and effective source capability +digest. Every media-buy/product pair is explicit. Report-definition IDs do not +identify products. Account, configuration ID and generation are all part of the +binding. Multiple offerings can share a definition; they still need distinct +applicable source contracts. Supported-offering discovery can precede the first +account binding. + +Admission durably freezes the source mapping, generation semantics and selected +provider method. Source acquisition and materialization compare the current +authoritative contracts to those frozen values. Withdrawal or incompatible +mutation fails closed; it cannot reinterpret an old generation. Resolve and +admit a new generation for a different contract. Historical pages and accepted +receipt evidence retain their original values. + +`ReportingProductionConfigurationTask` wraps the adopter's authenticated account +task. Its callback receives an `admit` function; call it with a +`ReportingConfigurationAdmission` containing the complete requested public +configuration and trusted resolved records before returning ready or inactive +state, including replay. The SDK checks actual schedule, scope, provider method, +finality, source generation and returned coverage. It completes the account's +versioned activation before returning ready, so newly admitted accounts need no +separate account-enumeration worker. Account provisioning and its durable +idempotency remain the account implementation's responsibility; do not claim +unsupported account features in its capability model. This producer admits +explicit full media-buy scopes and fixed-duration schedules that it can prove. +It does not turn an unsupported dynamic or partial scope into full coverage. + +Managed delivery-only polling needs the source, durable writer, verified +readback, complete status/exact reads and configuration route. It does not need +the receipt route or an HTTP notification worker. Reconciled additionally needs +official finality, canonical verification, and mounted revision/adjustment +receipt ingress. Neither the development writer nor the in-memory conformance +store advertises production durability. Capability checks bind the actual +mounted handler and running components; adopters do not maintain readiness +booleans. Protected capability fields cannot be supplied through `extra=`. + +## Migration, drain and activation + +1. Stop and drain autonomous legacy materializer writers. Resolve or explicitly + import uncertain legacy effects using the + [materializer recovery procedure](reporting-durable-materializer.md). Keep the + original external idempotency identity for pending SDK attempts. +2. Stop and drain old status projectors and clock sweepers. Ordinary compatible + historical readers/writers are a different compatibility claim from running + those incompatible autonomous workers. +3. Call `await store.create_schema()` on `PgReportingProductionStore` before + starting workers. Bootstrap remains transactional, serialized and repeatable. + Keep the private configuration lease-fairness objects: they deliberately + remain outside the old mandatory manifests. +4. Construct the verifier registry, trusted source/provider adapters, producer, + materializer, `PgReportingStatusProjection` and `ReportingProductionSupport`. + Mount **that support's handler** on the authenticated transports before + `await support.start()`. The typed example uses the combined server's startup + and shutdown hooks. +5. Activate existing accounts with `await support.activate(account_id=...)`, or + let validated configuration admission activate the affected account. Activation + fences old projection writers and incrementally consumes preserved captured + boundaries and baselines. An interruption resumes those original inputs; + it does not reconstruct historical status from today's records. + +The isolated projection and production manifests do not append objects to the +older ledger, materializer, receipt-ingestion or frozen-feed manifests. Old or +partial installations cannot prove production readiness. Catalog proofs are +positive caches scoped to the concrete support/pool and invalidation epoch; +live component and route checks still run. For later DDL, stop/drain support, +migrate, construct fresh support, validate, and restart. Arbitrary serving-time +DDL or search-path mutation is not automatically detected. Bound database +statement execution at the adopter/database layer. +The support also binds the pool's concrete identity and checks its open state +on each readiness request. Closing or replacing that pool withdraws the claim +even while the immutable catalog proof remains cached; rebuild the support for +a new pool. +The projection queue and all configured notification queues must retain the +same pool as the ledger. Warm discovery performs no pool checkout and stays +available when that valid pool is temporarily saturated. + +Activation never promotes epoch-zero work or readiness. Pending attempts keep +their original work identity, epoch and permanently quarantined events through +resume, replay and restart. Only genuinely new qualified work enters the +production epoch. The verified terminal result, immutable status capture, work +ACK and enabled logical `reporting.delivery_ready` enqueue share one transaction. +Enqueue failure rolls all of them back. An uncertain external effect resumes +the same destination identity; it must not allocate a fresh attempt. + +With notifications disabled, no logical enqueue occurs and complete polling +remains available. With notifications enabled, the logical queue is mandatory; +a broken enabled path fails closed. Optional `production_notification_workers` +compose the real Core, versioned status and production queues for crash-safe +recipient expansion and delivery. They never drain quarantined readiness. +Supply `ReportingProductionSigning(resolver, algorithms, brand_json_url=...)` with the actual +resolver's RFC 9421 algorithms. Its public signing declaration and every +resolved key must agree. Production notification registration and dispatch +reject legacy Bearer/HMAC fallback; rotation keeps the advertised algorithm +contract. Private key material stays in the resolver and sender. The operator +must publish its brand document and corresponding signing keys at the declared +identity; the SDK does not assert that a supplied URL proves external ownership. +The declaration is RFC 9421 `adcp/webhook-signing/v1`, the exact supplied +algorithm set, no legacy fallback, and an 86,400-second retry horizon. Each +queue reserves the first HTTP attempt and its key/body binding before peer I/O, +using database time in PostgreSQL. Window insertion, HTTP attempt and ordinal +reservation share one transaction; a failed reservation leaves none of them. +That committed reservation anchors an immutable +deadline: attempts are permitted strictly before `started_at + 86400 seconds`, +and refused at or after that instant. A reservation with unknown HTTP effect is +still the original anchor. Retries, key rotation, crashes and restart cannot extend it; expired +deliveries are suppressed with the existing closed `lease_expired` code. +Backoff is capped at the original deadline, including after configuration changes. +An HTTP attempt reserved before that deadline may finish afterward; the limit +prevents another attempt, and does not rewrite an already observed result. +A clock earlier than the retained first-attempt timestamp fails closed. +Delivery records and protected payload bindings are retained. Drain and rebuild +the support to change algorithms or operator identity. Receivers must retain +old public verification keys and authenticated deduplication state for the +advertised interval. The tests use public verification with pinned fixture +keys; external brand/JWKS discovery and live interoperability remain separate +cross-language gates. + +In the immutable `3.2.0-rc.3` capability schema, the descriptions at +`/properties/webhook_signing/properties/delivery_retry_horizon_seconds` and +`/properties/identity/description` require these declarations for applicable 3.2 +agents. The JSON fields deliberately remain optional for older documents. +Unmodified schema validation therefore **accepts their omission**. The mounted +semantic assertions in `test_reporting_production_readiness.py` establish the +missing-declaration defect; `test_reporting_production_signing_schema.py` +separately preserves the original schema acceptance and normative text. The +reservation, clock, restart, signature and expiry tests establish actual +behavior. This differs from the #1179 cached-schema rejection described below. + +The three workers also retain actual attempt activity through the inherited +SDK reservation/outcome transaction. `ReportingActivityProjector(worker.outbox)` +provides the corresponding authenticated account-activity read; applications +may compose that existing optional account surface. Expiry creates no fictitious +HTTP attempt or outcome. Reporting polling and immutable receipt evidence remain +the recovery path if the receiver did not acknowledge before the deadline. +Recipient fanout is separate from the finish transaction's immutable logical +enqueue. Stopping delivery may accumulate eligible pending events; it cannot +manufacture, promote or re-identify historical readiness. + +## Captured status, schedule and ownership + +Status keeps the canonical consumer private even when consumer feedback is off. +Revision selection examines complete history first. A unique official wins over +an unlinked retained snapshot even before the official has an artifact. Managed +needs that selected readable verified artifact. Reconciled also needs accepted +selected-official evidence and an accepted current receipt leaf for every +applicable official adjustment. Later valid artifacts or health degradation do +not erase accepted evidence or successful-materialization counts. Consumer +rejection is never a materializer retry signal. + +For #1179, a generation owes only complete periods whose start is at or after +activation and strictly before deactivation. A period already begun at +deactivation remains owed in full, including its SLA. The producer and feed use +this same rule. `next_expected_at` is the nearest strictly future committed +expectation in the selected captured scope, even when current health is complete. +The immutable Draft 7 cache for `3.2.0-rc.3` rejects that otherwise-valid response +at `/allOf/2/then/not`. The effective Python SDK validator and advertised MCP +schema remove only that exact known prohibition, only for this version. The +`if` and its requirements that `scope_closed` and `coverage_complete` are both +true remain enforced, as do types, formats and all other conditionals. A +different version or changed rule is left intact. Cached files and generated +status models are preserved; this is an explicit SDK correction, not a new +upstream schema version. The executable reproduction and constraint/mounted +regressions are in `test_reporting_schedule_schema.py`; the existing SDK issue +is [#1179](https://github.com/adcontextprotocol/adcp-client-python/issues/1179). + +Cross-language compatibility remains a separate blocking dependency. Every +selected compatible stable/skew client/server lane must successfully return +`complete` with the legitimate future expectation and correct semantics. +Unsupported-schema errors are acceptable only for explicitly unsupported +combinations. Python success does not establish TypeScript/protocol agreement +or release the later expert/four-quadrant gate. The coordinator owns upstream +schema/version coordination; neither suppress the expectation nor change an +otherwise-correct health value to satisfy the original prohibition. +It is absent when there is no applicable expectation. Captured timezones and +civil/DST boundaries survive later configuration changes. Producer turns keep +the 64-period maximum; durable cursors and bounded pending-acquisition queues +advance past completed windows without an unbounded history scan. Lease +acquisition takes the account lock before configuration rows and preserves +turn-primary fairness and account isolation. +Lease acquisition, release and recovery are bookkeeping, not new reporting +observations or materializer targets. The production PostgreSQL path retains +the inherited trigger and cancels only its lease-only candidate increment in +the same account-locked transaction. It never rewrites a pending attempt's +generation, epoch or external identity. Real configuration, revision and +readability changes retain their original fences. +PostgreSQL checks at most 32 admitted configuration candidates per lease turn. +A rejected source binding advances a separate durable probe rank so it cannot +permanently occupy that window. Rejection does not acquire a configuration lease +or change its frozen binding, pending acquisition, or external identity. +A held account lock instead advances a read-only sampling hint, because no +rank can be changed without that account lock. The hint belongs to one store +instance and one selected set of producer keys. Each pass examines at most 32 +candidates; an empty tail may wrap once to the beginning. Only a committed +turn updates the hint. A successful lease clears it and uses the durable +turn-primary order again, so an unlocked earlier account is revisited. A new +store begins at that same durable order; it may revisit one bounded window +before continuing. Concurrent workers can repeat a sample, but the account +lock, lease predicate and durable ranks still determine actual acquisition. +Hints never create work, acquire leases or alter frozen generation identities. + +Persisted PostgreSQL timestamps can omit trailing fractional zeros. Reporting +decoders accept the resulting aware precision and offset forms on Python 3.10 +without changing the captured bytes or represented microsecond instant. Naive, +invalid and excess-precision values remain refused. + +The shared public JSON Schema `date-time` checker has a different role: it +validates RFC 3339 wire strings without parsing them into Python timestamps. +It accepts arbitrary positive fractional widths as specified by +[RFC 3339 section 5.6](https://www.rfc-editor.org/rfc/rfc3339#section-5.6), including +the five-digit PostgreSQL form, on Python 3.10–3.13; validation preserves the exact input. +Its existing calendar, aware-offset, ASCII and seconds `00..59` requirements +remain enforced. In particular, the persisted decoder's six-digit bound and +historical seconds-bearing offsets are not imported into public validation. +This correction affects named schemas, task request/response validation and +the MCP/A2A validation paths that use them. It does not change a cached schema, +the separate #1179 conditional exception, or public-model work in #1190. +`test_schema_datetime_formats.py` checks the actual named/task schemas, +fractional precision, invalid inputs and `oneOf` selection; the configuration +mount tests check the unchanged raw timestamp through all three mounts. + +Opt-in revision ownership uses page-local +`ext.adcp.reporting_revision_ownership` version 1 bindings. Every returned revision +has exactly one owner and empty opted-in pages explicitly carry empty bindings. +The full bounded buyer walk checks ownership, dependencies and counts after +all pages arrive, with defaults of 2,048 pages and 200,000 records; +all-pages-absent remains conservative legacy mode. An exact +revision's binding alone cannot prove an otherwise unknown obligation. + +An in-flight B2.3 snapshot retains its original representation, ownership mode, +membership, order, counts and checkpoint after migration/activation/restart. +New snapshots may opt into the new representation. Authorization is checked on +every request: revocation can deny a continuation but cannot rebuild its history. + +For #1180 the four public task/notification fields are nullable with default +`None` in canonical and bundled generated models. Missing values stay absent in +standalone and nested serialization. Readiness requires Managed, receipts require +Reconciled, and ledger/status notifications are independently opt-in. The former +global reporting serializer mask is removed; generation owns the model contract. + +## Recovery and operational boundaries + +Close support with `await support.aclose()` before replacing components or +migrating. Preserve pending work, immutable journals, snapshots and exact receipt +batch responses. Restart with the same admitted contracts, complete migration and +let the owned bounded workers converge. Inspect typed closed failure codes; +provider bodies and credential contexts are not persistence or diagnostic data. +The [receipt ingress](reporting-receipt-ingress.md), +[frozen feed](reporting-frozen-feed.md) and original materializer recovery +contracts continue to apply. + +Full buyer adjustment/submission automation and the `client.reporting` facade +remain later buyer work. They do not substitute for seller financial validation. +This slice remains open and unmerged pending independent exact-head review and +the separately gated downstream interoperability program. + +## Seller acceptance ownership + +The PR evidence index binds executed commands, counts, artifacts and tested +head/tree to every row below. An upstream implementation identity identifies +an input; it does not replace execution against the final B2.4 child. +The mounted MCP and A2A 0.3/1.0 tests use in-process ASGI. Separate real-process +and SIGKILL tests establish restart behavior; these do not establish live TCP +or the later cross-language interoperability gate. + +| Requirement | Implementation owner | Current integration coverage | +| --- | --- | --- | +| Durable materializer | B2.1; B2.4 admission | Reserve/verify/finish/ACK/enabled enqueue, faults, leases, uncertain-effect identities and permanent epoch-zero quarantine. | +| Canonical authenticated consumer | B2.2; B2.4 reads | MCP/A2A trusted resolver, authorization on replay, account collisions and feedback-off/on private reads. | +| Immutable receipt transaction | B2.2 | Mixed receipt, feed, captured status and ordinal commit/rollback in both stores and notification modes. | +| Exact batch replay and mount | B2.2 | Shape preflight, semantic outcomes, original order/timestamps and concurrent/crashed final-response replay. | +| Middleware/schema boundary | B2.2; B2.4 | Actual pinned/unpinned mounts, diagnostics, #1179 narrow schema correction and #1180 nested/public models. | +| Receipt financial graph | B2.2 writes; B2.4 projection | Selected official, accepted artifact stability, rejected-leaf replacement, accepted terminality and official adjustments. | +| Frozen authorized combined feed | B2.3; B2.4 activation | Actual historical page one, installed child activation/SIGKILL, exact remaining bytes and captured schedules. | +| Feed checkpoint/closure | B2.3; B2.4 ownership | Scope-bound compact tokens, full dependency closure/counts, deterministic walk and unchanged final checkpoint. | +| Versioned status capture | B2.4 | Original captured boundaries/baselines, old-writer fence, ordered reconciliation-only and reversible health transitions. | +| Tier-correct status | B2.4; #1179 | Strict financial evidence, immutable counts/retention and committed future schedules, including complete health and DST. | +| Private scope and buyer ownership | B2.3 inputs; B2.4 wire/walk | Exact page-local ownership, malformed/mixed/conflicting walks, cross-page dependencies and conservative legacy compatibility. | +| Production tier capabilities | B2.4; #1180 | Full provider/source contracts, live component/mount checks, empty-seller discovery, polling and signing/retry truthfulness. | +| Rolling compatibility | Every slice | Eleven distinct historical inputs, isolated manifests, populated/repeated/interrupted migration, installed floor runtimes and actual restarts. | diff --git a/examples/reporting_production.py b/examples/reporting_production.py new file mode 100644 index 000000000..c5ea501a8 --- /dev/null +++ b/examples/reporting_production.py @@ -0,0 +1,121 @@ +"""One production seller composition, with actual provider and source contracts. + +The adopter owns trusted source/product mappings, provider grants, the account +task and token verification. The SDK owns admission, bounded discovery, the +materializer, captured status, receipts, exact reads and optional notification +delivery. Migrate and drain older workers before starting this composition. +""" + +from __future__ import annotations + +from adcp.decisioning.registry import BuyerAgentRegistry +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingMaterializerService, + ReportingRevisionVerifierRegistry, +) +from adcp.reporting.outbox import ( + ReportingEnvelopeCipher, + ReportingNotificationWorker, + ReportingSubscriptionResolver, +) +from adcp.reporting.production import ( + PgReportingProductionStore, + ReportingProductionConfigurationTask, + ReportingProductionDestination, + ReportingProductionOffering, + ReportingProductionSigning, + ReportingProductionSupport, + production_notification_workers, +) +from adcp.reporting.projection import PgReportingStatusProjection +from adcp.reporting.receipts import ReceiptAccountResolver +from adcp.server import serve +from adcp.server.auth import BearerTokenAuth, auth_context_factory + + +async def compose_reporting( + store: PgReportingProductionStore, + *, + destination: ReportingProductionDestination, + registry: ReportingRevisionVerifierRegistry, + offerings: tuple[ReportingProductionOffering, ...], + configuration_task: ReportingProductionConfigurationTask, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + consumer_status_enabled: bool = False, + subscriptions: ReportingSubscriptionResolver | None = None, + signing: ReportingProductionSigning | None = None, + cipher: ReportingEnvelopeCipher | None = None, +) -> ReportingProductionSupport: + """Prepare after draining old autonomous materializers/projectors/sweepers. + + Each offering names its actual producer, effective source offering and + installed verifier. Its source implements configuration_binding() with a + ReportingProductionSourceBinding.for_configuration(...) built from the + trusted account/catalog mapping. Product IDs are explicit; neither SDK nor + adopter may infer them from a report-definition ID. + + The destination implements configuration_binding() with the complete + ReportingProductionDestinationBinding resolved from its provider grant. + Credentials are acquired only inside separate write/readback sessions. + A changed method or source generation requires a new admitted identity. + + The account task calls its supplied admit(ReportingConfigurationAdmission) + for each ready/inactive reporting result, including replay. This completes + the account's activation before ready can be returned. Discovery needs no + first account, and workers do not require an adopter account inventory. + """ + await store.create_schema() + projection = PgReportingStatusProjection( + store, consumer_status_enabled=consumer_status_enabled, revision_ownership=True + ) + workers: tuple[ReportingNotificationWorker, ...] = () + if subscriptions is not None: + if cipher is None or signing is None: + raise ValueError( + "notification delivery requires an envelope cipher and signing contract" + ) + workers = production_notification_workers( + store, projection, subscriptions=subscriptions, signing=signing, cipher=cipher + ) + elif signing is not None or cipher is not None: + raise ValueError("notification delivery requires the subscription resolver") + return ReportingProductionSupport( + ReportingMaterializerService( + store, ReportingDestinationIO(registry, destination), destination + ), + projection, + offerings=offerings, + configuration_task=configuration_task, + resolve_account=resolve_account, + buyer_agents=buyer_agents, + notification_workers=workers, + ) + + +def serve_reporting( + support: ReportingProductionSupport, + *, + auth: BearerTokenAuth, + public_url: str, + allowed_hosts: tuple[str, ...], +) -> None: + """Mount both authenticated transports, then start and drain the SDK lifecycle. + + Configure store notifications explicitly. Polling needs no HTTP worker; + enabled notifications always retain atomic logical enqueue, even while + recipient delivery is stopped. Historical accounts may additionally be + activated by the operator with await support.activate(account_id=...). + """ + serve( + support.handler, + name="reporting-production", + transport="both", + auth=auth, + context_factory=auth_context_factory, + public_url=public_url, + allowed_hosts=allowed_hosts, + on_startup=[support.start], + on_shutdown=[support.aclose], + ) diff --git a/pyproject.toml b/pyproject.toml index b6ce77fc6..d98174941 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -195,6 +195,8 @@ adcp = [ "reporting/materializer/*.json", "reporting/receipts/*.json", "reporting/feed/*.json", + "reporting/projection/*.json", + "reporting/production/*.json", # PREVIEW: vendored sync_reporting_status schemas. They are the runtime # validator for the wire conditionals codegen cannot express, so the wheel # must carry them. Removed with the rest of _preview/ at rc.2. diff --git a/scripts/post_generate_fixes.py b/scripts/post_generate_fixes.py index 371c963b6..b89747112 100644 --- a/scripts/post_generate_fixes.py +++ b/scripts/post_generate_fixes.py @@ -2558,6 +2558,124 @@ def fix_unchanged_literal_defaults() -> None: print(" No unchanged field defaults needed fixing") +def fix_reporting_capability_defaults() -> None: + """Keep optional reporting promises absent in both generated model graphs. + + JSON Schema ``const`` restricts a supplied value; it does not advertise a + capability when the property is absent. Correct the annotations/defaults, + not just serialization (#1180). The scoped serializer also preserves + omission when an ordinary Pydantic parent does not set exclude_none. + """ + optional = { + "reliable_reporting_version", + "managed_delivery", + "reconciled_billing", + "configuration_task", + "status_task", + "consumer_status_task", + "revision_content_task", + "receipt_task", + "readiness_notification", + "status_notification", + "ledger_notification", + "supports_webhook_activity", + } + targets = ( + OUTPUT_DIR / "core/reporting_delivery_capabilities.py", + OUTPUT_DIR / "bundled/protocol/get_adcp_capabilities_response.py", + ) + for path in targets: + source = path.read_text() + lines = source.splitlines(keepends=True) + offsets = [0] + for line in lines: + offsets.append(offsets[-1] + len(line)) + changes: list[tuple[int, int, str]] = [] + classes = [ + node + for node in ast.parse(source).body + if isinstance(node, ast.ClassDef) + and re.fullmatch(r"ReportingDelivery(?:Capabilities)?\d*", node.name) + ] + if not classes: + raise ValueError(f"reporting capability model missing from {path.name}") + for node in classes: + for field in node.body: + if not ( + isinstance(field, ast.AnnAssign) + and isinstance(field.target, ast.Name) + and field.target.id in optional + and field.value is not None + ): + continue + annotation = field.annotation + if ( + isinstance(annotation, ast.Subscript) + and isinstance(annotation.value, ast.Name) + and annotation.value.id == "Annotated" + and isinstance(annotation.slice, ast.Tuple) + ): + annotation = annotation.slice.elts[0] + text = ast.get_source_segment(source, annotation) + assert text is not None + if not any( + isinstance(part, ast.Constant) and part.value is None + for part in ast.walk(annotation) + ): + changes.append( + ( + offsets[annotation.lineno - 1] + annotation.col_offset, + offsets[annotation.end_lineno - 1] + annotation.end_col_offset, + text + " | None", + ) + ) + default = field.value + changes.append( + ( + offsets[default.lineno - 1] + default.col_offset, + offsets[default.end_lineno - 1] + default.end_col_offset, + "None", + ) + ) + if not any( + isinstance(method, ast.FunctionDef) and method.name == "_validate_reporting_tiers" + for method in node.body + ): + methods = f""" + @model_validator(mode='after') + def _validate_reporting_tiers(self) -> {node.name}: + if self.reconciled_billing is True and self.managed_delivery is not True: + raise ValueError('reconciled_billing requires managed_delivery') + if self.readiness_notification is not None and self.managed_delivery is not True: + raise ValueError('readiness_notification requires managed_delivery') + if self.receipt_task is not None and self.reconciled_billing is not True: + raise ValueError('receipt_task requires reconciled_billing') + return self + + @model_serializer(mode='wrap') + def _omit_absent_reporting_promises( + self, handler: SerializerFunctionWrapHandler + ) -> dict[str, Any]: + return {{key: value for key, value in handler(self).items() if value is not None}} +""" + changes.append((offsets[node.end_lineno], offsets[node.end_lineno], methods)) + for start, end, text in sorted(changes, reverse=True): + source = source[:start] + text + source[end:] + imports = ( + "from typing import Any\n" + "from pydantic import SerializerFunctionWrapHandler, model_serializer, model_validator\n" + ) + if "from pydantic import SerializerFunctionWrapHandler," not in source: + source = source.replace( + "from __future__ import annotations\n", + "from __future__ import annotations\n\n" + imports, + 1, + ) + ast.parse(source) + path.write_text(source) + print(f" {path.relative_to(OUTPUT_DIR)}: optional reporting promises and tier validation") + + def fix_protocol_envelope_status_default() -> None: """Default response envelope status to completed for ergonomic construction. @@ -5899,6 +6017,7 @@ def main(argv: list[str] | None = None): widen_extension_point_lists_to_sequence, fix_canceled_literal_defaults, fix_unchanged_literal_defaults, + fix_reporting_capability_defaults, fix_protocol_envelope_status_default, fix_trusted_match_runtime_validators, fix_beta3_secure_url_constraints, diff --git a/src/adcp/reporting/_reconcile.py b/src/adcp/reporting/_reconcile.py index 3868b53bb..7cfc90c48 100644 --- a/src/adcp/reporting/_reconcile.py +++ b/src/adcp/reporting/_reconcile.py @@ -40,10 +40,13 @@ post_consumer_statuses, resolve_checkpointed_leaves, ) +from adcp.reporting.ownership import ReportingOwnershipError, page_revision_ownership from adcp.reporting.revision_selection import RevisionHistoryEntry, select_reporting_revision from adcp.types import ( GetReportingStatusRequest, GetReportingStatusResponse, + ReportingAdjustment, + ReportingAdjustmentReceipt, ReportingCanonicalContentDigest, ReportingControlTotal, ReportingDeliveryCapabilities, @@ -172,6 +175,11 @@ class ReportingLedger: #: from "new claim, must supersede", and re-filing the same claim under a #: new id churns the chain for no reason. consumer_statuses: list[Any] = field(default_factory=list) + # None is the all-pages-absent legacy mode. An empty mapping is an explicit + # new-mode empty snapshot; do not collapse those two meanings. + revision_ownership: dict[str, str] | None = None + adjustments: list[ReportingAdjustment] = field(default_factory=list) + adjustment_receipts: list[ReportingAdjustmentReceipt] = field(default_factory=list) @dataclass(frozen=True) @@ -296,9 +304,19 @@ async def load_reporting_ledger( request: GetReportingStatusRequest, *, max_snapshot_restarts: int = 2, + max_pages: int = 2048, + max_records: int = 200_000, ) -> ReportingLedger: """Exhaust a stable periods cursor and verify its declared record count.""" - + if ( + type(max_snapshot_restarts) is not int + or max_snapshot_restarts < 0 + or type(max_pages) is not int + or max_pages < 1 + or type(max_records) is not int + or max_records < 1 + ): + raise ValueError("reporting walk bounds must be positive (restarts may be zero)") base = request.model_dump(mode="json", exclude_none=True) base["view"] = "periods" base.pop("pagination", None) @@ -309,6 +327,11 @@ async def load_reporting_ledger( materializations: dict[str, ReportingMaterialization] = {} receipts: dict[str, ReportingReceipt] = {} consumer_statuses: dict[str, Any] = {} + adjustments: dict[str, ReportingAdjustment] = {} + adjustment_receipts: dict[str, ReportingAdjustmentReceipt] = {} + ownership: dict[str, str] = {} + ownership_mode: bool | None = None + frozen_metadata: str | None = None cursor: str | None = None seen_cursors: set[str] = set() snapshot_id: str | None = None @@ -317,7 +340,7 @@ async def load_reporting_ledger( scope: BaseModel | None = None total_count: int | None = None - while True: + for _page_number in range(max_pages): payload = dict(base) if cursor: payload["pagination"] = {"cursor": cursor} @@ -330,6 +353,38 @@ async def load_reporting_ledger( "STATUS_READ_FAILED", "get_reporting_status did not return a completed periods view", ) + raw_page = response.model_dump(mode="json", exclude_none=True) + if "ext" in response.model_fields_set and response.ext is None: + raw_page["ext"] = None + try: + local = page_revision_ownership(raw_page) + except ReportingOwnershipError: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "invalid page-local revision ownership" + ) from None + mode = local is not None + if ownership_mode is not None and mode != ownership_mode: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "mixed ownership modes within one snapshot" + ) + ownership_mode = mode + for revision_id, owner_id in (local or {}).items(): + if revision_id in ownership and ownership[revision_id] != owner_id: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "ownership changed within one snapshot" + ) + ownership[revision_id] = owner_id + metadata = _json( + { + k: raw_page.get(k) + for k in ("changes_checkpoint", "next_expected_at", "health", "issues") + } + ) + if mode and frozen_metadata is not None and metadata != frozen_metadata: + raise ReportingReconciliationError( + "SNAPSHOT_CHANGED", "frozen projection changed" + ) + frozen_metadata = metadata pagination = response.pagination if ( not response.ledger_snapshot_id @@ -359,6 +414,10 @@ async def load_reporting_ledger( account_id = response.account_id scope = response.scope total_count = pagination.total_count + if total_count is not None and total_count > max_records: + raise ReportingReconciliationError( + "LEDGER_LIMIT_EXCEEDED", "ledger record limit exceeded" + ) for obligation in response.periods or []: _add_immutable( obligations, @@ -377,6 +436,17 @@ async def load_reporting_ledger( ) for receipt in response.receipts or []: _add_immutable(receipts, receipt.reporting_receipt_id, receipt, "receipt") + for adjustment in response.adjustments or []: + _add_immutable( + adjustments, adjustment.reporting_adjustment_id, adjustment, "adjustment" + ) + for adjustment_receipt in response.adjustment_receipts or []: + _add_immutable( + adjustment_receipts, + adjustment_receipt.reporting_receipt_id, + adjustment_receipt, + "adjustment receipt", + ) for status in getattr(response, "consumer_statuses", None) or []: _add_immutable( consumer_statuses, @@ -385,14 +455,36 @@ async def load_reporting_ledger( "consumer status", ) + if ( + sum( + len(records) + for records in ( + obligations, + revisions, + materializations, + receipts, + consumer_statuses, + adjustments, + adjustment_receipts, + ) + ) + > max_records + ): + raise ReportingReconciliationError( + "LEDGER_LIMIT_EXCEEDED", "ledger record limit exceeded" + ) if not pagination.has_more: break cursor = pagination.cursor - if not cursor or cursor in seen_cursors: + if not cursor or len(cursor) > 2048 or cursor in seen_cursors: raise ReportingReconciliationError( "CURSOR_LOOP", "ledger pagination did not advance" ) seen_cursors.add(cursor) + else: + raise ReportingReconciliationError( + "LEDGER_LIMIT_EXCEEDED", "ledger page limit exceeded" + ) count = ( len(obligations) @@ -400,6 +492,8 @@ async def load_reporting_ledger( + len(materializations) + len(receipts) + len(consumer_statuses) + + len(adjustments) + + len(adjustment_receipts) ) if total_count is not None and total_count != count: raise ReportingReconciliationError( @@ -410,7 +504,7 @@ async def load_reporting_ledger( raise ReportingReconciliationError( "EMPTY_LEDGER_RESPONSE", "get_reporting_status returned no ledger page" ) - return ReportingLedger( + ledger = ReportingLedger( snapshot_id, ledger_as_of, account_id, @@ -420,13 +514,92 @@ async def load_reporting_ledger( list(materializations.values()), list(receipts.values()), list(consumer_statuses.values()), + ownership if ownership_mode else None, + list(adjustments.values()), + list(adjustment_receipts.values()), ) + if ownership_mode: + _validate_owned_ledger(ledger) + return ledger except ReportingReconciliationError as error: if error.code != "SNAPSHOT_CHANGED" or restart == max_snapshot_restarts: raise raise ReportingReconciliationError("SNAPSHOT_CHANGED", "ledger never stabilized") +def _validate_owned_ledger(ledger: ReportingLedger) -> None: + """Validate explicit ownership only after all bounded pages are present.""" + owners = {o.reporting_obligation_id: o for o in ledger.obligations} + revisions = {r.reporting_revision_id: r for r in ledger.revisions} + bindings = ledger.revision_ownership + + def invalid() -> None: + raise ReportingReconciliationError( + "INVALID_REVISION_OWNERSHIP", "incomplete or inconsistent ownership dependencies" + ) + + if bindings is None or set(bindings) != set(revisions): + invalid() + assert bindings is not None + if any(o.account_id != ledger.account_id for o in owners.values()): + invalid() + for revision_id, owner_id in bindings.items(): + if owner_id not in owners or not _revision_matches_obligation( + revisions[revision_id], owners[owner_id] + ): + invalid() + predecessor = revisions[revision_id].supersedes_reporting_revision_id + if predecessor is not None and bindings.get(predecessor) != owner_id: + invalid() + for owner in owners.values(): + if ( + sum(o == owner.reporting_obligation_id for o in bindings.values()) + != owner.revision_count + ): + invalid() + materials = {m.reporting_materialization_id: m for m in ledger.materializations} + owned_evidence: tuple[ReportingMaterialization | ReportingReceipt, ...] = ( + *ledger.materializations, + *ledger.receipts, + ) + for item in owned_evidence: + if bindings.get(item.reporting_revision_id) != item.reporting_obligation_id: + invalid() + for receipt in ledger.receipts: + material = materials.get(receipt.reporting_materialization_id) + if material is None or ( + material.reporting_revision_id != receipt.reporting_revision_id + or material.reporting_obligation_id != receipt.reporting_obligation_id + ): + invalid() + adjustments = {a.reporting_adjustment_id: a for a in ledger.adjustments} + for adjustment in adjustments.values(): + revision = revisions.get(adjustment.adjusts_reporting_revision_id) + if revision is None or _enum(revision.finality) != "official": + invalid() + for adjustment_receipt in ledger.adjustment_receipts: + target_adjustment = adjustments.get(adjustment_receipt.reporting_adjustment_id) + if ( + target_adjustment is None + or target_adjustment.adjusts_reporting_revision_id + != adjustment_receipt.adjusts_reporting_revision_id + ): + invalid() + + +def _owned_revisions( + obligation: ReportingObligation, ledger: ReportingLedger +) -> list[ReportingRevision]: + if ledger.revision_ownership is not None: + return [ + r + for r in ledger.revisions + if ledger.revision_ownership.get(r.reporting_revision_id) + == obligation.reporting_obligation_id + ] + return [r for r in ledger.revisions if _revision_matches_obligation(r, obligation)] + + def _select_current( obligation: ReportingObligation, ledger: ReportingLedger ) -> tuple[ReportingRevision | None, ReportingMaterialization | None, list[str]]: @@ -459,6 +632,18 @@ def _select_current( ) and item.reporting_revision_id not in owned_elsewhere ] + if ledger.revision_ownership is not None: + candidates = _owned_revisions(obligation, ledger) + elif any( + not any( + m.reporting_revision_id == item.reporting_revision_id for m in ledger.materializations + ) + and sum(_revision_matches_obligation(item, o) for o in ledger.obligations) > 1 + for item in candidates + ): + # Counts and equal semantic scopes are not an ownership declaration. + # An unmaterialized revision may still belong to either obligation. + reasons.append("AMBIGUOUS_REVISION_OWNERSHIP") receipts = [ item for item in ledger.receipts @@ -761,11 +946,20 @@ def evaluate_reporting_ledger( expected_periods: list[ExpectedReportingPeriod] | None = None, now: datetime | None = None, ) -> ReportingReconciliationResult: + if ledger.revision_ownership is not None: + _validate_owned_ledger(ledger) now = now or datetime.now(timezone.utc) outcomes: list[ObligationReconciliation] = [] unique_revisions: dict[str, ReportingRevision] = {} for obligation in ledger.obligations: revision, materialization, reasons = _select_current(obligation, ledger) + if obligation.adjustment_count or any( + a.adjusts_reporting_revision_id == getattr(revision, "reporting_revision_id", None) + for a in ledger.adjustments + ): + # Loading ownership/dependencies is additive. The separately owned + # buyer adjustment evidence/submission workflow remains required. + reasons.append("ADJUSTMENT_RECONCILIATION_REQUIRED") if _enum(obligation.health) != "complete": reasons.append(f"OBLIGATION_{_enum(obligation.health).upper()}") if ( @@ -909,11 +1103,7 @@ async def reconcile_reporting_core( definition=pinned_definition, revisions={item.reporting_revision_id: item for item in ledger.revisions}, obligation_revisions={ - obligation.reporting_obligation_id: [ - revision - for revision in ledger.revisions - if _revision_matches_obligation(revision, obligation) - ] + obligation.reporting_obligation_id: _owned_revisions(obligation, ledger) for obligation in ledger.obligations }, current_statuses=ledger.consumer_statuses, diff --git a/src/adcp/reporting/_timestamp.py b/src/adcp/reporting/_timestamp.py new file mode 100644 index 000000000..074577e8f --- /dev/null +++ b/src/adcp/reporting/_timestamp.py @@ -0,0 +1,43 @@ +"""Lossless aware timestamps for reporting boundaries on every supported Python.""" + +from __future__ import annotations + +import re +from datetime import datetime, timedelta, timezone + +from adcp.reporting.evidence import aware_utc + +_TIMESTAMP = re.compile( + r"(?P[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}" + r"(?:\.[0-9]{1,6})?)" + r"(?:Z|(?P[+-])(?P[01][0-9]|2[0-3]):(?P[0-5][0-9])" + r"(?::(?P[0-5][0-9])(?:\.(?P[0-9]{1,6}))?)?)" +) + + +def aware_timestamp(value: str) -> datetime: + """Decode the exact microsecond instant without changing captured wire bytes. + + PostgreSQL JSON omits trailing fractional zeros. Python 3.10's ISO parser + only accepts three or six fractional digits. Padding supplies equivalent + zeros; excess precision, naive values and invalid dates remain refused. + Historical PostgreSQL time zones can also carry an offset in seconds. + """ + matched = _TIMESTAMP.fullmatch(value) if type(value) is str else None + if matched is None: + raise ValueError("reporting timestamp requires an aware microsecond instant") + normalized = re.sub(r"\.([0-9]+)", lambda m: "." + m[1].ljust(6, "0"), matched["local"]) + try: + # datetime.fromisoformat also drops subsecond offsets when their + # whole-second part is zero. Construct the exact offset independently. + offset = timedelta( + hours=int(matched["hours"] or 0), + minutes=int(matched["minutes"] or 0), + seconds=int(matched["seconds"] or 0), + microseconds=int((matched["microseconds"] or "").ljust(6, "0")), + ) + if matched["sign"] == "-": + offset = -offset + return aware_utc(datetime.fromisoformat(normalized).replace(tzinfo=timezone(offset))) + except (ValueError, OverflowError): + raise ValueError("reporting timestamp requires an aware microsecond instant") from None diff --git a/src/adcp/reporting/feed/memory.py b/src/adcp/reporting/feed/memory.py index 7ddfd641b..f0bad7eef 100644 --- a/src/adcp/reporting/feed/memory.py +++ b/src/adcp/reporting/feed/memory.py @@ -33,6 +33,9 @@ class InMemoryReportingFeedStore(InMemoryReportingReceiptStore): _reporting_feed_snapshots: dict[str, tuple[bytes, str, bytes]] + def _feed_projection_options(self, caller: ReportingDeliveryPrincipal) -> dict[str, Any]: + return {} + def _feed_snapshot( self, snapshot_id: str, caller: ReportingDeliveryPrincipal ) -> StoredFeedSnapshot | None: @@ -90,6 +93,7 @@ def _capture_feed( receipt_boundaries=tuple( b for b in getattr(self, "_receipt_boundaries", ()) if b.caller == caller ), + **self._feed_projection_options(caller), ) @storage_errors diff --git a/src/adcp/reporting/feed/pg.py b/src/adcp/reporting/feed/pg.py index 778a0b9e4..867246426 100644 --- a/src/adcp/reporting/feed/pg.py +++ b/src/adcp/reporting/feed/pg.py @@ -9,7 +9,7 @@ from collections.abc import Awaitable, Callable from dataclasses import dataclass from importlib.resources import files -from typing import Any +from typing import Any, Literal from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.feed._errors import storage_errors @@ -41,6 +41,9 @@ class _CapturedFeed: changes: tuple[ReportingReconciliationChange, ...] materializer: tuple[dict[str, Any], ...] receipts: tuple[dict[str, Any], ...] + representation_version: Literal[1, 2] = 1 + revision_ownership: bool = False + activated_consumer_status_enabled: bool | None = None @dataclass(frozen=True, repr=False) @@ -71,6 +74,9 @@ def _prepare_feed( decode_materializer_boundary(r) for r in captured.materializer ), receipt_boundaries=tuple(decode_receipt_boundary(r) for r in captured.receipts), + representation_version=captured.representation_version, + revision_ownership=captured.revision_ownership, + activated_consumer_status_enabled=captured.activated_consumer_status_enabled, ) stored = StoredFeedSnapshot(snapshot, secrets.token_bytes(32)) document = canonical_json_utf8_v1(snapshot.to_storage()) diff --git a/src/adcp/reporting/feed/projection.py b/src/adcp/reporting/feed/projection.py index ed56e9cb3..bad09482b 100644 --- a/src/adcp/reporting/feed/projection.py +++ b/src/adcp/reporting/feed/projection.py @@ -3,7 +3,7 @@ from __future__ import annotations from dataclasses import replace -from typing import Any, NoReturn, cast +from typing import Any, Literal, NoReturn, cast from uuid import uuid4 from pydantic import TypeAdapter @@ -39,6 +39,7 @@ ReportingRevisionReceiptRecord, ) from adcp.reporting.ledger.notification_models import ReportingStatusScope +from adcp.reporting.ledger.schedule import next_reporting_expectation from adcp.reporting.ledger.status import ( _adjustment_to_wire, _consumer_status_to_wire, @@ -83,6 +84,9 @@ def capture_feed( consumer_status_enabled: bool, materializer_boundaries: tuple[ReportingMaterializerBoundary, ...] = (), receipt_boundaries: tuple[ReportingReceiptBoundary, ...] = (), + representation_version: Literal[1, 2] = 1, + revision_ownership: bool = False, + activated_consumer_status_enabled: bool | None = None, ) -> ReportingFeedSnapshot: """Project detached histories captured under one account-lock boundary. @@ -93,7 +97,15 @@ def capture_feed( order. The two sequence spaces are never compared or collapsed with max(). Closure records retain their original sort keys even below changes_after. """ - if core.account_id != caller.account_id: + if ( + core.account_id != caller.account_id + or representation_version not in {1, 2} + or (revision_ownership and representation_version != 2) + or ( + activated_consumer_status_enabled is not None + and activated_consumer_status_enabled != consumer_status_enabled + ) + ): _corrupt() # Filter foreign statements before deriving maxima, projection or membership. # IDs must resolve unambiguously; scope metadata never supplies ownership. @@ -231,13 +243,20 @@ def capture_feed( projection = StatusProjectionInput( core, ReportingStatusScope( - caller.account_id, consumer_id=caller.consumer_id if consumer_status_enabled else None + caller.account_id, + consumer_id=( + caller.consumer_id + if (representation_version == 2 or consumer_status_enabled) + else None + ), ), delivery_config_ids=tuple(scoped["delivery_config_ids"] or ()), media_buy_ids=tuple(scoped["media_buy_ids"] or ()), feed_purposes=tuple(scoped["feed_purposes"] or ()), period_start=_parse(scoped["period_start"]), period_end=_parse(scoped["period_end"]), + reconciliation=records if representation_version == 2 else None, + consumer_status_enabled=consumer_status_enabled, ) scope_result = project_status_scope(projection) selected_owners = {p.obligation.reporting_obligation_id for p in scope_result.obligations} @@ -292,12 +311,22 @@ def capture_feed( issues=result.issues, statuses=projected_obligation.statuses, ) + if projected_obligation.reconciliation is not None: + wires[identity].update(projected_obligation.reconciliation.wire) elif kind == "revision": owner_id = record.reporting_obligation_id if owner_id not in owners: _corrupt() revision_id = record_id wires[identity] = _revision_to_wire(record, owners[owner_id]) + if ( + representation_version == 2 + and owners[owner_id].generation_key in bindings + and record.canonical_content_digest is not None + ): + wires[identity][ + "canonical_content_digest" + ] = record.canonical_content_digest.to_wire() if record.supersedes_reporting_revision_id is not None: predecessor = revisions.get(record.supersedes_reporting_revision_id) if predecessor is None or predecessor.reporting_obligation_id != owner_id: @@ -310,6 +339,10 @@ def capture_feed( _corrupt() owner_id = target.reporting_obligation_id wires[identity] = _adjustment_to_wire(record) + if representation_version == 2 and owners[owner_id].generation_key in bindings: + from adcp.reporting.ledger.delivery import adjustment_to_wire + + wires[identity] = adjustment_to_wire(record) elif kind == "consumer_status": owner_id, revision_id = record.reporting_obligation_id, record.reporting_revision_id if owner_id is None and revision_id is not None: @@ -465,8 +498,8 @@ def capture_feed( _corrupt() inputs = { "version": 1, - "projection_version": 1, - "ownership_mode": "absent", + "projection_version": representation_version, + "ownership_mode": "bindings" if revision_ownership else "absent", "consumer_status_enabled": consumer_status_enabled, "core": _CORE.dump_python(frozen_core, mode="json"), "reconciliation": [payload(r) for r in records], @@ -501,6 +534,39 @@ def capture_feed( "health": scope_result.health, "issues": [issue.to_wire() for issue in scope_result.issues], } + if representation_version == 2: + configurations = tuple( + c + for c in scope_result.configurations + if (not finalities or c.required_finality in finalities) + and ( + not healths + or project_status_scope( + replace( + projection, + scope=ReportingStatusScope( + c.account_id, c.generation_key, consumer_id=caller.consumer_id + ), + ) + ).health + in healths + ) + ) + obligations = tuple( + p.obligation + for p in scope_result.obligations + if (not healths or p.projection.health in healths) + and (not finalities or p.obligation.required_finality in finalities) + ) + next_expected = next_reporting_expectation( + configurations, + obligations, + as_of=core.as_of, + period_start=projection.period_start, + period_end=projection.period_end, + ) + if next_expected is not None: + common["next_expected_at"] = next_expected.isoformat().replace("+00:00", "Z") return ReportingFeedSnapshot( caller, "rpfs_" + uuid4().hex, @@ -514,4 +580,6 @@ def capture_feed( for i in sorted(selected, key=keys.__getitem__) ), canonical_json_utf8_v1(inputs), + representation_version, + "bindings" if revision_ownership else "absent", ) diff --git a/src/adcp/reporting/feed/snapshot.py b/src/adcp/reporting/feed/snapshot.py index 90147eb58..069a1539d 100644 --- a/src/adcp/reporting/feed/snapshot.py +++ b/src/adcp/reporting/feed/snapshot.py @@ -74,7 +74,7 @@ class ReportingFeedSnapshot: records: tuple[ReportingFeedRecord, ...] = field(repr=False) inputs_json: bytes = field(repr=False) representation_version: int = 1 - ownership_mode: Literal["absent"] = "absent" + ownership_mode: Literal["absent", "bindings"] = "absent" @property def total_count(self) -> int: @@ -121,8 +121,8 @@ def decode_snapshot(value: Any) -> ReportingFeedSnapshot: or type(value["version"]) is not int or value["version"] != 1 or type(value["representation_version"]) is not int - or value["representation_version"] != 1 - or value["ownership_mode"] != "absent" + or (value["representation_version"], value["ownership_mode"]) + not in {(1, "absent"), (2, "absent"), (2, "bindings")} or type(value["filters"]) is not str or type(json.loads(value["filters"])) is not dict ): @@ -165,9 +165,39 @@ def decode_snapshot(value: Any) -> ReportingFeedSnapshot: canonical_json_utf8_v1(value["common"]), records, canonical_json_utf8_v1(value["inputs"]), + value["representation_version"], + value["ownership_mode"], ) if result.to_storage() != value: raise ValueError + if result.representation_version == 2: + inputs = value["inputs"] + if ( + inputs["projection_version"] != 2 + or inputs["ownership_mode"] != result.ownership_mode + ): + raise ValueError + core = inputs["core"] + owners = {o["reporting_obligation_id"] for o in core["obligations"]} + revisions = core["revisions"] + expected = {r["reporting_revision_id"]: r["reporting_obligation_id"] for r in revisions} + if len(expected) != len(revisions) or not set(expected.values()).issubset(owners): + raise ValueError + bindings = inputs["revision_ownership"] + if ( + type(bindings) is not list + or len(bindings) != len(expected) + or any( + type(b) is not dict + or set(b) != {"reporting_revision_id", "reporting_obligation_id"} + for b in bindings + ) + or {b["reporting_revision_id"]: b["reporting_obligation_id"] for b in bindings} + != expected + ): + raise ValueError + if any(r.record_id not in expected for r in records if r.kind == "revision"): + raise ValueError except (ValueError, TypeError, KeyError, IndexError, RecursionError): result = None if result is None: @@ -239,6 +269,19 @@ def page(self, offset: int, limit: int) -> dict[str, Any]: **({"cursor": self.token("cursor", end)} if more else {}), }, ) + if snapshot.ownership_mode == "bindings": + from adcp.reporting.ownership import ReportingOwnershipError, with_revision_ownership + + try: + ownership: dict[str, str] = {} + for item in snapshot.inputs["revision_ownership"]: + revision, owner = item["reporting_revision_id"], item["reporting_obligation_id"] + if revision in ownership: + raise ReportingOwnershipError() + ownership[revision] = owner + result = with_revision_ownership(result, ownership) + except (KeyError, TypeError, ReportingOwnershipError): + raise ReportingFeedError("REPORTING_FEED_HISTORY_CORRUPT") from None return result diff --git a/src/adcp/reporting/ledger/models.py b/src/adcp/reporting/ledger/models.py index f021c68b8..935d0379c 100644 --- a/src/adcp/reporting/ledger/models.py +++ b/src/adcp/reporting/ledger/models.py @@ -26,6 +26,7 @@ from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from typing import Any, Literal +from zoneinfo import ZoneInfo from adcp.reporting.currency import validate_currency, validate_currency_units from adcp.reporting.evidence import ( @@ -233,6 +234,33 @@ def __post_init__(self) -> None: raise ValueError("expected_at cannot precede the period end") +def _schedule_clock( + schedule: ReportingScheduleSpec, account_timezone: str +) -> tuple[ZoneInfo, timedelta, datetime]: + zone = ZoneInfo(schedule.timezone_name(account_timezone)) + duration = iso_duration_to_timedelta(schedule.period_duration) + if duration <= timedelta(0): + raise ValueError("period_duration must be positive") + anchor = ( + _utc(schedule.period_anchor) + if schedule.period_anchor is not None + else datetime(1970, 1, 1, tzinfo=timezone.utc) + ) + return zone, duration, anchor.astimezone(zone).replace(tzinfo=None) + + +def _period_instants( + schedule: ReportingScheduleSpec, account_timezone: str, ordinal: int +) -> tuple[datetime, datetime]: + zone, duration, anchor = _schedule_clock(schedule, account_timezone) + # Civil-time boundaries use the first occurrence of an ambiguous local + # time. A spring-forward gap can collapse a slot to zero elapsed time; + # the shared schedule iterator skips that slot, never inventing a report. + start = (anchor + duration * ordinal).replace(tzinfo=zone).astimezone(timezone.utc) + end = (anchor + duration * (ordinal + 1)).replace(tzinfo=zone).astimezone(timezone.utc) + return start, end + + def derive_period( schedule: ReportingScheduleSpec, *, @@ -253,27 +281,11 @@ def derive_period( begins at the next boundary -- a partial first period would be reported as complete and understate delivery. """ - from zoneinfo import ZoneInfo - zone_name = schedule.timezone_name(account_timezone) - zone = ZoneInfo(zone_name) - duration = iso_duration_to_timedelta(schedule.period_duration) - if duration <= timedelta(0): - raise ValueError("period_duration must be positive") sla = iso_duration_to_timedelta(schedule.delivery_sla) - - anchor = ( - _utc(schedule.period_anchor) - if schedule.period_anchor is not None - else datetime(1970, 1, 1, tzinfo=timezone.utc) - ) # Walk whole periods from the anchor in local wall-clock terms so a DST # transition shifts the instant without changing which period it is. - local_anchor = anchor.astimezone(zone).replace(tzinfo=None) - start_local = local_anchor + duration * ordinal - end_local = local_anchor + duration * (ordinal + 1) - start = start_local.replace(tzinfo=zone).astimezone(timezone.utc) - end = end_local.replace(tzinfo=zone).astimezone(timezone.utc) + start, end = _period_instants(schedule, account_timezone, ordinal) if activated_at is not None and _utc(activated_at) > start: raise ValueError( @@ -299,23 +311,18 @@ def first_ordinal_after( A configuration activated at 00:20 with hourly aligned periods owes ``[01:00, 02:00)`` first, not a 40-minute stub. """ - ordinal = 0 - # Seek coarsely then step back, so a long-lived configuration does not walk - # every period since the epoch one at a time. - step = 1 << 20 - while step: - candidate = derive_period( - schedule, account_timezone=account_timezone, ordinal=ordinal + step - ) - if _utc(candidate.start) <= _utc(activated_at): - ordinal += step - else: - step //= 2 - while True: - candidate = derive_period(schedule, account_timezone=account_timezone, ordinal=ordinal) - if _utc(candidate.start) >= _utc(activated_at): - return ordinal + zone, duration, anchor = _schedule_clock(schedule, account_timezone) + at = _utc(activated_at) + local = at.astimezone(zone).replace(tzinfo=None) + ordinal = (local - anchor) // duration + # Seek directly in civil time, then resolve timezone folds/gaps against + # instants. This also supports an explicit anchor after activation without + # scanning from the Unix epoch or constructing overflowing probe dates. + while _period_instants(schedule, account_timezone, ordinal)[0] < at: ordinal += 1 + while _period_instants(schedule, account_timezone, ordinal - 1)[0] >= at: + ordinal -= 1 + return ordinal @dataclass(frozen=True) diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index 273912457..b35b4e45d 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -35,7 +35,7 @@ from collections.abc import Awaitable, Callable, Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import datetime, timedelta, timezone -from typing import Any, TypeAlias +from typing import TYPE_CHECKING, Any, TypeAlias from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.currency import ( @@ -50,7 +50,7 @@ ReportingObligationRecord, ReportingPeriodBoundary, ReportingRevisionRecord, - derive_period, + iso_duration_to_timedelta, ) from adcp.reporting.ledger.store import ( LeasedConfiguration, @@ -75,6 +75,10 @@ parse_verified_source_batch_manifest_v1, ) +if TYPE_CHECKING: + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + from adcp.reporting.materializer.verification import ReportingRevisionVerifier + __all__ = [ "CurrencyResolver", "FixedCurrencyResolver", @@ -223,6 +227,7 @@ def __init__( max_periods_per_turn: int = 64, clock: Callable[[], datetime] | None = None, currency_resolver: CurrencyResolver | None = None, + revision_verifier: ReportingRevisionVerifier | None = None, ) -> None: self._source = source self._offerings = offerings @@ -238,6 +243,7 @@ def __init__( if currency_resolver is not None else FixedCurrencyResolver(offerings.currency) ) + self._revision_verifier = revision_verifier @property def store(self) -> ReportingLedgerStore: @@ -378,8 +384,12 @@ async def _close_elapsed_periods( *, now: datetime, ) -> list[ReportingObligationRecord]: + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + + progress = self._store if isinstance(self._store, ReportingProducerProgress) else None + after = None if progress is None else await progress.producer_closed_through(configuration) committed: list[ReportingObligationRecord] = [] - for boundary in self._elapsed_periods(configuration, now=now): + for boundary in self._elapsed_periods(configuration, now=now, after=after): existing = await self._store.find_obligation( account_id=configuration.account_id, delivery_config_id=configuration.delivery_config_id, @@ -388,6 +398,11 @@ async def _close_elapsed_periods( period_end=boundary.end, ) if existing is not None: + if progress is not None: + await progress.commit_producer_period( + configuration, existing, previous_end=after + ) + after = boundary.end continue obligation = ReportingObligationRecord( reporting_obligation_id=self._obligation_id(configuration, boundary), @@ -414,48 +429,51 @@ async def _close_elapsed_periods( resolved = self._currency_resolver(configuration, obligation) currency = await resolved if inspect.isawaitable(resolved) else resolved obligation = replace(obligation, currency=validate_currency(currency)) - stored = await self._store.commit_obligation(obligation) + stored = ( + await self._store.commit_obligation(obligation) + if progress is None + else await progress.commit_producer_period( + configuration, obligation, previous_end=after + ) + ) + after = boundary.end committed.append(stored) turn.obligations_committed.append(stored.reporting_obligation_id) return committed def _elapsed_periods( - self, configuration: ReportingConfiguration, *, now: datetime + self, + configuration: ReportingConfiguration, + *, + now: datetime, + after: datetime | None = None, ) -> list[ReportingPeriodBoundary]: """Every eligible period that has closed but is not yet obligated. - A period is eligible once its *end* is at or before now. A snapshot - taken exactly at the boundary does not expose it -- the obligation - appears in the first snapshot strictly after it, which is the rule both - sides derive independently. + A period is eligible once its *end* is at or before now. Activation + owes the first full period; deactivation after a period has started + retains that whole period and its original SLA. Polling forecasts use + the same committed-generation iterator. """ - from adcp.reporting.ledger.models import first_ordinal_after + from itertools import islice + + from adcp.reporting.ledger.schedule import committed_periods - activated_at = configuration.activated_at - if activated_at is None: - return [] - ordinal = first_ordinal_after( - configuration.schedule, - account_timezone=configuration.account_timezone, - activated_at=activated_at, - ) boundaries: list[ReportingPeriodBoundary] = [] - for _ in range(self._max_periods_per_turn): - boundary = derive_period( - configuration.schedule, - account_timezone=configuration.account_timezone, - ordinal=ordinal, - ) + near = ( + None + if after is None + else after + iso_duration_to_timedelta(configuration.schedule.delivery_sla) + ) + periods = ( + period + for period in committed_periods(configuration, near=near) + if after is None or period.end > after + ) + for boundary in islice(periods, self._max_periods_per_turn): if _utc(boundary.end) > _utc(now): break - if configuration.deactivated_at is not None and _utc(boundary.start) >= _utc( - configuration.deactivated_at - ): - # Deactivation still owes a period that already started, but - # not one that had not begun when the configuration stopped. - break boundaries.append(boundary) - ordinal += 1 return boundaries @staticmethod @@ -481,6 +499,11 @@ def _obligation_id( async def _acquire_pending( self, configuration: ReportingConfiguration, turn: WorkerTurn, *, now: datetime ) -> None: + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + + if isinstance(self._store, ReportingProducerProgress): + await self._acquire_progress(self._store, configuration, turn, now=now) + return for boundary in self._elapsed_periods(configuration, now=now): obligation = await self._store.find_obligation( account_id=configuration.account_id, @@ -507,6 +530,37 @@ async def _acquire_pending( turn.slices_failed.append(obligation.reporting_obligation_id) self._note_escalation(obligation, turn, now=now) + async def _acquire_progress( + self, + progress: ReportingProducerProgress, + configuration: ReportingConfiguration, + turn: WorkerTurn, + *, + now: datetime, + ) -> None: + identifiers = await progress.next_producer_obligations( + configuration, now=now, limit=self._max_periods_per_turn + ) + for identifier in identifiers: + obligation = await self._store.get_obligation( + account_id=configuration.account_id, reporting_obligation_id=identifier + ) + if obligation is None or obligation.generation_key != configuration.generation_key: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer history is unavailable") + try: + await self.acquire_obligation(configuration, obligation, turn=turn, now=now) + except (ReportingCurrencyError, LedgerConflictError) as error: + if isinstance(error, LedgerConflictError) and error.code not in { + "HISTORY_UNAVAILABLE", + "EMPTY_DENOMINATOR", + }: + raise + turn.slices_failed.append(identifier) + self._note_escalation(obligation, turn, now=now) + await progress.finish_producer_acquisition( + configuration, reporting_obligation_id=identifier + ) + async def acquire_obligation( self, configuration: ReportingConfiguration, @@ -568,8 +622,20 @@ async def acquire_obligation( f"this producer declares no source offering for {finality} reporting", ) + from adcp.reporting.ledger.producer_progress import ReportingProducerProgress + + constituents = ( + await self._store.producer_constituents(configuration, obligation) + if isinstance(self._store, ReportingProducerProgress) + else None + ) request = self._build_slice( - configuration, obligation, offering_id, now=now, observation=len(revisions) + configuration, + obligation, + offering_id, + now=now, + observation=len(revisions), + constituents=constituents, ) cancel = asyncio.Event() try: @@ -732,6 +798,10 @@ async def commit_revision_from_manifest( source_publication_id=manifest.publication_id, source_manifest_sha256=manifest.content_fingerprint.split(":", 1)[-1], ) + if self._revision_verifier is not None: + from adcp.reporting.materializer.publication import verified_publication + + revision = verified_publication(self._revision_verifier, obligation, revision, rows) committed = await self._store.commit_revision(revision, rows) turn.revisions_committed.append(committed.reporting_revision_id) return committed @@ -811,6 +881,7 @@ def _build_slice( *, now: datetime, observation: int = 0, + constituents: tuple[ReportingConstituent, ...] | None = None, ) -> ReportingSourceSliceRequestV1: """Freeze one slice request from the obligation. @@ -827,15 +898,19 @@ def _build_slice( not from the clock, so neither behavior depends on wall time. """ offering = self._source.capabilities.offering(offering_id) - constituents: list[ReportingConstituent] = [ - MediaBuyConstituentV1( - constituent_id=media_buy_id, - product_id=obligation.report_definition_id, - media_buy_id=media_buy_id, - ) - for media_buy_id in obligation.media_buy_ids - ] - if not constituents: + resolved_constituents: list[ReportingConstituent] = ( + list(constituents) + if constituents is not None + else [ + MediaBuyConstituentV1( + constituent_id=media_buy_id, + product_id=obligation.report_definition_id, + media_buy_id=media_buy_id, + ) + for media_buy_id in obligation.media_buy_ids + ] + ) + if not resolved_constituents: raise LedgerConflictError( "EMPTY_DENOMINATOR", "an obligation with no media buys has no source work; it is a platform-owned " @@ -892,8 +967,8 @@ def _build_slice( trigger="scheduled_poll", coverage=ReportingSourceCoverageRequestV1( expected="full", - constituents=constituents, - denominator_fingerprint=coverage_denominator_fingerprint_v1(constituents), + constituents=resolved_constituents, + denominator_fingerprint=coverage_denominator_fingerprint_v1(resolved_constituents), ), requested_metrics=list(self._offerings.requested_metrics), requested_dimensions=list(self._offerings.requested_dimensions), diff --git a/src/adcp/reporting/ledger/producer_progress.py b/src/adcp/reporting/ledger/producer_progress.py new file mode 100644 index 000000000..dfdd4715d --- /dev/null +++ b/src/adcp/reporting/ledger/producer_progress.py @@ -0,0 +1,93 @@ +"""Optional bounded producer progress; the original ledger protocol stays intact.""" + +from __future__ import annotations + +from collections.abc import Sequence +from datetime import datetime +from typing import Literal, Protocol, runtime_checkable + +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingObligationRecord, + ReportingRevisionRecord, + iso_duration_to_timedelta, +) +from adcp.reporting.ledger.schedule import committed_periods +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.revision_selection import select_reporting_revision +from adcp.reporting.source import ReportingConstituent + + +@runtime_checkable +class ReportingProducerProgress(Protocol): + """Durable, generation-scoped closing position and indexed unfinished work. + + Only the production participant installs this optional path. An acquired + configuration still uses the original lease/fairness and source execution + identities. Closing position, obligation and its work item co-commit. + Acquisition completion reselects immutable history under the account lock. + """ + + async def producer_closed_through( + self, configuration: ReportingConfiguration + ) -> datetime | None: ... + + async def producer_constituents( + self, configuration: ReportingConfiguration, obligation: ReportingObligationRecord + ) -> tuple[ReportingConstituent, ...]: ... + + async def commit_producer_period( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + previous_end: datetime | None, + ) -> ReportingObligationRecord: ... + + async def next_producer_obligations( + self, configuration: ReportingConfiguration, *, now: datetime, limit: int + ) -> tuple[str, ...]: ... + + async def finish_producer_acquisition( + self, configuration: ReportingConfiguration, *, reporting_obligation_id: str + ) -> None: ... + + +def check_next_period( + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + previous_end: datetime | None, +) -> None: + near = ( + None + if previous_end is None + else previous_end + iso_duration_to_timedelta(configuration.schedule.delivery_sla) + ) + expected = next( + ( + p + for p in committed_periods(configuration, near=near) + if previous_end is None or p.end > previous_end + ), + None, + ) + if obligation.generation_key != configuration.generation_key or expected != obligation.period: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer closing position is invalid") + + +def acquisition_state( + obligation: ReportingObligationRecord, revisions: Sequence[ReportingRevisionRecord] +) -> Literal["pending", "settled", "parked"]: + """The existing acquire-obligation predicates, without a new re-read policy.""" + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + if selection.kind == "corrupt": + return "parked" + current = selection.revision if selection.kind == "selected" else None + if current is not None and (current.finality == "official" or current.readable): + return "settled" + return "pending" diff --git a/src/adcp/reporting/ledger/reconciliation_projection.py b/src/adcp/reporting/ledger/reconciliation_projection.py new file mode 100644 index 000000000..c671a4913 --- /dev/null +++ b/src/adcp/reporting/ledger/reconciliation_projection.py @@ -0,0 +1,349 @@ +"""Pure tier evidence over one captured private record history. + +Selection precedes artifact/receipt inspection. Acceptance is validated against +its immutable admission prefix; later artifacts and checks cannot revoke it. +Current readability is a separate condition for delivery health. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from datetime import datetime +from typing import Any, Literal + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger._delivery_state import ( + _verify_materialization, + _verify_receipt, + adjustment_sha256, + current_receipt, + fingerprint, + iso, + principal, + record_identity, +) +from adcp.reporting.ledger.delivery import ReportingMaterializationView +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.health import issue_id_for +from adcp.reporting.ledger.models import ( + ReportingAdjustmentRecord, + ReportingIssue, + ReportingObligationRecord, + ReportingRevisionRecord, +) +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.revision_selection import select_reporting_revision + + +@dataclass(frozen=True) +class ReconciliationProjection: + wire_json: bytes + evidence_json: bytes + issues: tuple[ReportingIssue, ...] = () + satisfied: bool = True + deadlines: tuple[datetime, ...] = () + + @property + def wire(self) -> dict[str, Any]: + return dict(json.loads(self.wire_json)) + + +def project_reconciliation( + obligation: ReportingObligationRecord, + revisions: tuple[ReportingRevisionRecord, ...], + adjustments: tuple[ReportingAdjustmentRecord, ...], + records: tuple[ReportingDeliveryRecord, ...], + *, + consumer_id: str | None, + as_of: datetime, +) -> ReconciliationProjection: + """All inputs belong to the captured boundary; no live store is consulted.""" + selection = select_reporting_revision( + revisions, + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + required_finality=obligation.required_finality, + ) + revision = selection.revision if selection.kind == "selected" else None + owned_ids = {r.reporting_revision_id for r in revisions} + owned_adjustments = tuple( + a for a in adjustments if a.adjusts_reporting_revision_id in owned_ids + ) + caller = ReportingDeliveryPrincipal(obligation.account_id, consumer_id) if consumer_id else None + history = tuple(r for r in records if caller is not None and principal(r) == caller) + scoped = tuple( + r + for r in history + if ( + r.generation_key == obligation.generation_key + if isinstance(r, ReportingDestinationBinding) + else r.scope.reporting_obligation_id == obligation.reporting_obligation_id + ) + ) + if len({record_identity(r) for r in scoped}) != len(scoped): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "reconciliation history is inconsistent") + evidence: dict[str, Any] = { + "reporting_obligation_id": obligation.reporting_obligation_id, + "consumer_id": consumer_id, + "selected": revision.reporting_revision_id if revision else None, + "records": [fingerprint(r) for r in scoped], + "adjustments": [ + [ + a.reporting_adjustment_id, + hashlib.sha256( + json.dumps(asdict(a), default=iso, sort_keys=True).encode() + ).hexdigest(), + ] + for a in owned_adjustments + ], + } + wire: dict[str, Any] = {"adjustment_count": len(owned_adjustments)} + bindings = [r for r in scoped if isinstance(r, ReportingDestinationBinding)] + if len(bindings) > 1: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "reconciliation binding is inconsistent") + if not bindings: + if scoped: + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "reconciliation binding is unavailable" + ) + return ReconciliationProjection( + canonical_json_utf8_v1(wire), canonical_json_utf8_v1(evidence) + ) + binding = bindings[0] + deliveries = [r for r in scoped if isinstance(r, ReportingObligationDeliveryRecord)] + if len(deliveries) > 1: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "reconciliation delivery is inconsistent") + delivery = deliveries[0] if deliveries else None + attempts = { + r.reporting_materialization_id: r + for r in scoped + if isinstance(r, ReportingMaterializationAttempt) + } + outcomes = tuple(r for r in scoped if isinstance(r, ReportingMaterializationRecord)) + successes = tuple(r for r in outcomes if r.status in {"available", "delivered"}) + receipts = tuple(r for r in scoped if isinstance(r, ReportingRevisionReceiptRecord)) + adjustment_receipts = tuple( + r for r in scoped if isinstance(r, ReportingAdjustmentReceiptRecord) + ) + checks = tuple(r for r in scoped if isinstance(r, ReportingMaterializationCheck)) + by_revision = {r.reporting_revision_id: r for r in revisions} + by_adjustment = {a.reporting_adjustment_id: a for a in owned_adjustments} + readable: dict[str, bool] = {} + deadlines: set[datetime] = set() + for outcome in outcomes: + attempt, target = attempts.get(outcome.reporting_materialization_id), by_revision.get( + outcome.reporting_revision_id + ) + if ( + attempt is None + or target is None + or delivery is None + or attempt.scope != outcome.scope + or attempt.reporting_revision_id != outcome.reporting_revision_id + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "materialization dependencies are unavailable" + ) + if outcome.status != "failed": + _verify_materialization(outcome, binding, delivery, target, obligation) + view = ReportingMaterializationView( + attempt, + binding, + outcome, + tuple( + c + for c in checks + if c.reporting_materialization_id == outcome.reporting_materialization_id + ), + ) + readable[outcome.reporting_materialization_id] = view.readable_at(as_of) and target.readable + if outcome.resource is not None: + deadlines.add(outcome.resource.expires_at) + leaves = {} + for index, record in enumerate(history): + if record not in (*receipts, *adjustment_receipts): + continue + if isinstance(record, ReportingRevisionReceiptRecord): + target = by_revision.get(record.reporting_revision_id) + if target is None: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "receipt revision is unavailable") + _verify_receipt(record, history[: index + 1], target) + leaves[("revision", record.reporting_revision_id)] = current_receipt(history, record) + elif isinstance(record, ReportingAdjustmentReceiptRecord): + adjustment = by_adjustment.get(record.reporting_adjustment_id) + if ( + adjustment is None + or adjustment.adjusts_reporting_revision_id != record.adjusts_reporting_revision_id + or ( + record.status == "accepted" + and record.observed_adjustment_sha256 != adjustment_sha256(adjustment) + ) + ): + raise LedgerConflictError( + "HISTORY_UNAVAILABLE", "adjustment receipt is inconsistent" + ) + leaves[("adjustment", record.reporting_adjustment_id)] = current_receipt( + history, record + ) + selected_successes = tuple( + r + for r in successes + if revision is not None and r.reporting_revision_id == revision.reporting_revision_id + ) + current = max( + selected_successes, + key=lambda r: attempts[r.reporting_materialization_id].attempt, + default=None, + ) + artifact_readable = bool(current and readable[current.reporting_materialization_id]) + evidence["readable"] = readable + # Counts describe immutable successful outcomes, not today's health checks. + wire.update( + destination_ref=binding.destination_ref, + reconciliation_mode=binding.reconciliation_mode, + materialization_count=len(outcomes), + successful_materialization_count=len(successes), + ) + retained = [o.resource.expires_at for o in successes if o.resource is not None] + if retained: + wire["resource_retained_until"] = iso(min(retained)) + requires_receipt = binding.reconciliation_mode == "consumer_receipt" + selected_receipt = ( + leaves.get(("revision", revision.reporting_revision_id)) if revision else None + ) + accepted = bool(selected_receipt and selected_receipt.status == "accepted") + applicable = tuple( + a + for a in owned_adjustments + if revision is not None + and a.adjusts_reporting_revision_id == revision.reporting_revision_id + ) + pending_adjustments = tuple( + a + for a in applicable + if ( + (leaf := leaves.get(("adjustment", a.reporting_adjustment_id))) is None + or leaf.status != "accepted" + ) + ) + if requires_receipt: + wire.update( + receipt_count=len(receipts), + accepted_receipt_count=sum(r.status == "accepted" for r in receipts), + adjustment_receipt_count=len(adjustment_receipts), + accepted_adjustment_receipt_count=sum( + r.status == "accepted" for r in adjustment_receipts + ), + pending_adjustment_count=len(pending_adjustments), + ) + rejected = bool(selected_receipt and selected_receipt.status == "rejected") or any( + (leaf := leaves.get(("adjustment", a.reporting_adjustment_id))) is not None + and leaf.status == "rejected" + for a in applicable + ) + wire["reconciliation_status"] = ( + "rejected" + if rejected + else "accepted" if accepted and not pending_adjustments else "pending" + ) + else: + wire["reconciliation_status"] = "not_required" + issues: list[ReportingIssue] = [] + + def issue(code: str, *, buyer: bool = False, immediate: bool = False) -> None: + if not immediate and as_of < obligation.period.expected_at: + return + severity: Literal["delayed", "action_required"] = ( + "action_required" + if immediate or as_of >= obligation.automated_recovery_deadline_at + else "delayed" + ) + # An immutable evidence change differentiates a recurring artifact/receipt + # failure while the ordered checkpoint retains its monotonic generation. + occurrence = hashlib.sha256(canonical_json_utf8_v1(evidence)).hexdigest() + issues.append( + ReportingIssue( + issue_id_for( + "tier-status-v2", + obligation.account_id, + consumer_id, + obligation.reporting_obligation_id, + code, + occurrence, + ), + code, + severity, + "buyer" if buyer else "seller", + "contact_buyer" if buyer else "contact_seller", + reporting_obligation_id=obligation.reporting_obligation_id, + delivery_config_id=obligation.delivery_config_id, + delivery_config_version=obligation.delivery_config_version, + feed_purpose=obligation.feed_purpose, + media_buy_ids=obligation.media_buy_ids, + expected_at=obligation.period.expected_at, + ) + ) + + if revision is not None: + if not artifact_readable: + expired = ( + current is not None + and current.resource is not None + and current.resource.expires_at <= as_of + ) + issue( + "RESOURCE_EXPIRED" if expired else "DELIVERY_FAILED", immediate=current is not None + ) + if requires_receipt: + if revision.finality != "official" or revision.canonical_content_digest is None: + issue("HISTORY_UNAVAILABLE", immediate=True) + if not accepted: + issue( + "RECEIPT_REJECTED" if selected_receipt else "RECEIPT_REQUIRED", + buyer=True, + immediate=True, + ) + for adjustment in pending_adjustments: + leaf = leaves.get(("adjustment", adjustment.reporting_adjustment_id)) + issue( + "ADJUSTMENT_RECEIPT_REJECTED" if leaf else "ADJUSTMENT_RECEIPT_REQUIRED", + buyer=True, + immediate=True, + ) + satisfied = bool( + revision is not None + and artifact_readable + and ( + not requires_receipt + or ( + revision.finality == "official" + and revision.canonical_content_digest is not None + and accepted + and not pending_adjustments + ) + ) + ) + # This bound is independent of transient corruption/readability; never + # claim the provider's retention beyond the frozen generation commitment. + if "resource_retained_until" in wire: + assert delivery is not None + wire["resource_retained_until"] = iso(min(delivery.resource_retained_until, *retained)) + return ReconciliationProjection( + canonical_json_utf8_v1(wire), + canonical_json_utf8_v1(evidence), + tuple(issues), + satisfied, + tuple(sorted(deadlines)), + ) diff --git a/src/adcp/reporting/ledger/reporting_production.sql b/src/adcp/reporting/ledger/reporting_production.sql new file mode 100644 index 000000000..209363432 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_production.sql @@ -0,0 +1,669 @@ +-- B2.4 production admission. Earlier work/queues and their mandatory manifests +-- remain byte-for-byte unchanged. The SDK uses the same materializer state +-- machine with a closed connection adapter for these new participants. +DO $production$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + CREATE TABLE IF NOT EXISTS reporting_production_delivery_windows ( + account_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + queue TEXT NOT NULL CHECK (queue IN ('core','status','ready')), + body_sha256 TEXT NOT NULL CHECK (body_sha256 ~ '^[0-9a-f]{64}$'), + started_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (account_id,idempotency_key), + CHECK (expires_at=started_at+interval '86400 seconds') + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_delivery_windows'::regclass + AND tgname='reporting_production_delivery_window_immutable') THEN + CREATE TRIGGER reporting_production_delivery_window_immutable BEFORE UPDATE OR DELETE + ON reporting_production_delivery_windows FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_accounts ( + account_id TEXT COLLATE "C" PRIMARY KEY REFERENCES reporting_projection_accounts, + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch=2), + activated_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + policy JSONB NOT NULL CHECK (jsonb_typeof(policy)='object'), + CHECK (jsonb_typeof(policy->'verification_keys') IS NOT DISTINCT FROM 'array'), + CHECK (jsonb_array_length(policy->'verification_keys') > 0), + CHECK (jsonb_typeof(policy->'notifications_enabled') IS NOT DISTINCT FROM 'boolean') + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_accounts'::regclass + AND tgname='reporting_production_activation_immutable') THEN + CREATE TRIGGER reporting_production_activation_immutable BEFORE UPDATE OR DELETE + ON reporting_production_accounts FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_work ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_production_accounts, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version BIGINT NOT NULL, + reporting_obligation_id TEXT COLLATE "C" NOT NULL, + reporting_revision_id TEXT COLLATE "C" NOT NULL, + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + attempt_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization_attempt' + CHECK (attempt_namespace = 'materialization_attempt'), + generation BIGINT NOT NULL CHECK (generation > 0), + binding_sha256 TEXT COLLATE "C" NOT NULL CHECK (binding_sha256 ~ '^[0-9a-f]{64}$'), + verification_key_sha256 TEXT COLLATE "C" NOT NULL + CHECK (verification_key_sha256 ~ '^[0-9a-f]{64}$'), + external_id TEXT COLLATE "C" NOT NULL CHECK (external_id ~ '^rwm_[0-9a-f]{64}$'), + state TEXT COLLATE "C" NOT NULL DEFAULT 'pending' CHECK (state IN ('pending','acked')), + retry_allowed BOOLEAN NOT NULL DEFAULT FALSE, + reason TEXT COLLATE "C" NOT NULL DEFAULT 'ready' CHECK (reason IN ( + 'ready','verified','retry','inactive','revision_not_ready','revision_unreadable', + 'target_changed','history_corrupt','legacy_pending','legacy_terminal', + 'component_unavailable','binding_changed','effect_unknown','operator_required')), + created_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + acknowledged_at TIMESTAMPTZ, + completion_token UUID, + lease_token UUID, + lease_until TIMESTAMPTZ, + due_at TIMESTAMPTZ NOT NULL DEFAULT clock_timestamp(), + imported BOOLEAN NOT NULL DEFAULT FALSE, + notifications_enabled BOOLEAN NOT NULL, + -- Only new reservations enter this epoch; no historical work is copied. + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch = 2), + PRIMARY KEY (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, consumer_id, external_id), + FOREIGN KEY (account_id, consumer_id, delivery_config_id, delivery_config_version, + reporting_obligation_id) REFERENCES reporting_materializer_candidates, + FOREIGN KEY (account_id, consumer_id, attempt_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + FOREIGN KEY (account_id, reporting_obligation_id, reporting_revision_id) + REFERENCES reporting_revisions(account_id, reporting_obligation_id, reporting_revision_id), + CHECK ((lease_token IS NULL) = (lease_until IS NULL)), + CHECK ((state = 'acked') = (acknowledged_at IS NOT NULL)), + CHECK ((state = 'acked') = (completion_token IS NOT NULL)), + CHECK (state <> 'acked' OR lease_token IS NULL), + CHECK (NOT retry_allowed OR state = 'acked') + ); + CREATE UNIQUE INDEX IF NOT EXISTS reporting_production_one_pending + ON reporting_production_work (account_id, consumer_id, delivery_config_id, + delivery_config_version, reporting_obligation_id) WHERE state = 'pending'; + CREATE INDEX IF NOT EXISTS reporting_production_work_due + ON reporting_production_work (account_id, due_at, reporting_materialization_id) + WHERE state = 'pending'; + + CREATE TABLE IF NOT EXISTS reporting_production_status_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + max_sequence BIGINT NOT NULL CHECK (max_sequence > 0), + PRIMARY KEY (account_id, consumer_id) + ); + CREATE TABLE IF NOT EXISTS reporting_production_status_boundaries ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + sequence BIGINT NOT NULL CHECK (sequence > 0), + account_sequence BIGINT NOT NULL CHECK (account_sequence > 0), + reporting_materialization_id TEXT COLLATE "C" NOT NULL, + outcome_namespace TEXT COLLATE "C" NOT NULL DEFAULT 'materialization' + CHECK (outcome_namespace = 'materialization'), + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL CHECK (content_sha256 ~ '^[0-9a-f]{64}$'), + PRIMARY KEY (account_id, consumer_id, sequence), + UNIQUE (account_id, consumer_id, reporting_materialization_id), + UNIQUE (account_id, account_sequence), + FOREIGN KEY (account_id, consumer_id, reporting_materialization_id) + REFERENCES reporting_production_work, + FOREIGN KEY (account_id, consumer_id, outcome_namespace, reporting_materialization_id) + REFERENCES reporting_reconciliation_records(account_id, consumer_id, namespace, record_id), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'reporting_materialization_id') IS NOT DISTINCT FROM reporting_materialization_id), + CHECK ((input->>'sequence')::bigint IS NOT DISTINCT FROM sequence), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK (content_sha256 = reporting_payload_sha256(input)), + CHECK (input - ARRAY['version','account_id','consumer_id','reporting_materialization_id', + 'sequence','account_sequence','as_of','core','reconciliation'] = '{}'::jsonb) + ); + CREATE TABLE IF NOT EXISTS reporting_production_notification_events ( + account_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type='reporting.delivery_ready'), + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind='materialization_ready'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation=1), + consumer_namespace TEXT COLLATE "C" NOT NULL CHECK (length(consumer_namespace)>0), + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch = 2), + fired_at TIMESTAMPTZ NOT NULL, + snapshot JSONB NOT NULL, + reporting_materialization_id TEXT COLLATE "C" GENERATED ALWAYS AS + (snapshot #>> '{cause,reporting_materialization_id}') STORED, + PRIMARY KEY (account_id, consumer_namespace, notification_id), + UNIQUE (account_id, consumer_namespace, notification_type, cause_kind, cause_id, cause_generation), + UNIQUE (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, reporting_materialization_id) + REFERENCES reporting_production_status_boundaries + (account_id, consumer_id, reporting_materialization_id), + CHECK ((snapshot->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((snapshot->>'notification_id') IS NOT DISTINCT FROM notification_id), + CHECK (cause_id::jsonb IS NOT DISTINCT FROM + jsonb_build_array(consumer_namespace, reporting_materialization_id)), + CHECK ((snapshot #>> '{cause,consumer_id}') IS NOT DISTINCT FROM consumer_namespace) + ); + CREATE TABLE IF NOT EXISTS reporting_production_notification_expansions ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation>0), + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending','leased','complete','suppressed','quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network','retryable_http','permanent_http','signing_unavailable', + 'permanent_scope','subscription_unavailable','subscription_changed', + 'invalid_configuration','invalid_payload','integrity_failure','lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, notification_id, emission_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id) + REFERENCES reporting_production_notification_events + ); + CREATE INDEX IF NOT EXISTS reporting_production_notification_due + ON reporting_production_notification_expansions (account_id, due_at) + WHERE state IN ('pending','leased'); + + CREATE TABLE IF NOT EXISTS reporting_production_notification_deliveries ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type = 'reporting.delivery_ready'), + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + idempotency_key TEXT COLLATE "C" NOT NULL, + destination_sha256 TEXT NOT NULL, + subscription_fingerprint TEXT NOT NULL, + signing_scope_id TEXT COLLATE "C", + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind = 'materialization_ready'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + auth_mode TEXT NOT NULL, + body_sha256 TEXT NOT NULL, + envelope_version INTEGER NOT NULL, + key_version TEXT COLLATE "C" NOT NULL, + envelope BYTEA NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, delivery_id), + UNIQUE (account_id, consumer_namespace, notification_id, emission_generation, subscriber_id), + UNIQUE (account_id, consumer_namespace, idempotency_key), + CHECK (length(principal_id) > 0 AND (consumer_namespace = '' OR consumer_namespace = principal_id)), + FOREIGN KEY (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation) + REFERENCES reporting_production_notification_events + (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id, emission_generation) + REFERENCES reporting_production_notification_expansions + (account_id, consumer_namespace, notification_id, emission_generation) + ); + CREATE INDEX IF NOT EXISTS reporting_production_notification_deliveries_due + ON reporting_production_notification_deliveries (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + + CREATE TABLE IF NOT EXISTS reporting_production_webhook_attempt_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL CHECK (length(principal_id) > 0), + subscriber_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + last_attempt BIGINT NOT NULL CHECK (last_attempt > 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + ); + CREATE TABLE IF NOT EXISTS reporting_production_webhook_attempts ( + account_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + attempt BIGINT NOT NULL CHECK (attempt > 0), + delivery_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + lease_token TEXT NOT NULL, + reservation_token TEXT NOT NULL, + binding JSONB NOT NULL, + fired_at TIMESTAMPTZ NOT NULL, + url TEXT NOT NULL CONSTRAINT reporting_production_webhook_url_safe + CHECK (length(url) <= 8192 AND url !~ '[?#@]' AND url ~ '^https?://'), + payload_size_bytes BIGINT NOT NULL CHECK (payload_size_bytes >= 0), + status TEXT NOT NULL DEFAULT 'pending' CHECK + (status IN ('pending', 'success', 'failed', 'timeout', 'connection_error')), + completed_at TIMESTAMPTZ, + http_status_code INTEGER, + response_time_ms BIGINT CHECK (response_time_ms >= 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key, attempt), + UNIQUE (account_id, consumer_namespace, principal_id, delivery_id, lease_token), + FOREIGN KEY (account_id, consumer_namespace, delivery_id) + REFERENCES reporting_production_notification_deliveries, + FOREIGN KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + REFERENCES reporting_production_webhook_attempt_heads + (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key), + CONSTRAINT reporting_production_webhook_identity CHECK ( + (binding->>'account_id') IS NOT DISTINCT FROM account_id AND + (binding->>'principal_id') IS NOT DISTINCT FROM principal_id AND + (binding->>'subscriber_id') IS NOT DISTINCT FROM subscriber_id AND + (binding->>'notification_id') IS NOT DISTINCT FROM notification_id AND + (binding->>'idempotency_key') IS NOT DISTINCT FROM idempotency_key AND + (binding->>'delivery_id') IS NOT DISTINCT FROM delivery_id AND + (binding->>'consumer_namespace') IS NOT DISTINCT FROM consumer_namespace), + CONSTRAINT reporting_production_webhook_completion CHECK ((status = 'pending') = (completed_at IS NULL)), + CONSTRAINT reporting_production_webhook_timestamps CHECK (completed_at >= fired_at), + CONSTRAINT reporting_production_webhook_outcome CHECK ( + (status IN ('pending', 'timeout', 'connection_error') + AND http_status_code IS NULL AND response_time_ms IS NULL) + OR (status IN ('success', 'failed') AND http_status_code BETWEEN 100 AND 599 + AND http_status_code IS NOT NULL AND response_time_ms IS NOT NULL + AND ((status = 'success') = (http_status_code BETWEEN 200 AND 299)))) + ); + CREATE INDEX IF NOT EXISTS reporting_production_webhook_activity_newest ON reporting_production_webhook_attempts + (account_id, principal_id, fired_at DESC, notification_id DESC, + idempotency_key DESC, subscriber_id DESC, attempt DESC, delivery_id DESC); + CREATE INDEX IF NOT EXISTS reporting_production_webhook_activity_retention ON reporting_production_webhook_attempts + (account_id, principal_id, completed_at) WHERE completed_at IS NOT NULL; + + -- Retention never resets a logical delivery's sequence, even after all + -- terminal attempts are purged. Writers always lock parent, then head. + CREATE OR REPLACE FUNCTION reporting_production_webhook_head_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $head_guard$ + BEGIN + IF TG_OP = 'DELETE' OR + (TG_OP = 'INSERT' AND NEW.last_attempt <> 1) OR + (TG_OP = 'UPDATE' AND (NEW.last_attempt <> OLD.last_attempt + 1 OR + (to_jsonb(NEW) - 'last_attempt') <> (to_jsonb(OLD) - 'last_attempt'))) THEN + RAISE EXCEPTION 'reporting activity counter must advance and be retained' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $head_guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_production_webhook_attempt_heads'::regclass + AND tgname = 'reporting_production_webhook_head_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_production_webhook_head_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_webhook_attempt_heads FOR EACH ROW EXECUTE FUNCTION reporting_production_webhook_head_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_production_webhook_attempt_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $guard$ + DECLARE + delivery reporting_production_notification_deliveries; + BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.completed_at IS NULL THEN + RAISE EXCEPTION 'pending reporting activity must be retained' USING ERRCODE = '23514'; + END IF; + RETURN OLD; + END IF; + IF TG_OP = 'UPDATE' AND ( + OLD.status <> 'pending' OR NEW.status = 'pending' OR + (to_jsonb(NEW) - ARRAY['status','completed_at','http_status_code','response_time_ms']) <> + (to_jsonb(OLD) - ARRAY['status','completed_at','http_status_code','response_time_ms'])) THEN + RAISE EXCEPTION 'immutable reporting activity reservation' USING ERRCODE = '23514'; + END IF; + -- The opaque reservation token fences terminalization in the UPDATE + -- predicate. A known late response may finish its own reservation after + -- lease expiry/reclaim; it must not require or modify the parent lease. + IF TG_OP = 'UPDATE' THEN + RETURN NEW; + END IF; + IF TG_OP = 'INSERT' AND NEW.status <> 'pending' THEN + RAISE EXCEPTION 'reporting activity requires reservation' USING ERRCODE = '23514'; + END IF; + SELECT * INTO delivery FROM reporting_production_notification_deliveries + WHERE account_id = NEW.account_id AND principal_id = NEW.principal_id + AND consumer_namespace = NEW.consumer_namespace AND delivery_id = NEW.delivery_id + AND subscriber_id = NEW.subscriber_id AND notification_id = NEW.notification_id + AND idempotency_key = NEW.idempotency_key + AND state = 'leased' AND lease_token = NEW.lease_token + AND lease_expires_at > coalesce(NEW.completed_at, NEW.fired_at) FOR UPDATE; + IF NOT FOUND OR + NEW.binding->>'account_id' IS DISTINCT FROM NEW.account_id OR + NEW.binding->>'principal_id' IS DISTINCT FROM NEW.principal_id OR + NEW.binding->>'subscriber_id' IS DISTINCT FROM NEW.subscriber_id OR + NEW.binding->>'notification_id' IS DISTINCT FROM NEW.notification_id OR + NEW.binding->>'idempotency_key' IS DISTINCT FROM NEW.idempotency_key OR + NEW.binding->>'delivery_id' IS DISTINCT FROM NEW.delivery_id OR + NEW.binding->>'consumer_namespace' IS DISTINCT FROM NEW.consumer_namespace OR + NEW.binding->>'notification_type' IS DISTINCT FROM delivery.notification_type OR + NEW.binding->>'body_sha256' IS DISTINCT FROM delivery.body_sha256 THEN + RAISE EXCEPTION 'reporting activity lease or identity mismatch' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_production_webhook_attempts'::regclass + AND tgname = 'reporting_production_webhook_attempt_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_production_webhook_attempt_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_webhook_attempts FOR EACH ROW EXECUTE FUNCTION reporting_production_webhook_attempt_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_production_retained_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + BEGIN + RAISE EXCEPTION 'reporting_materializer_evidence_immutable' USING ERRCODE='23514'; + END + $body$ $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_status_boundaries'::regclass + AND tgname='reporting_production_boundary_immutable') THEN + CREATE TRIGGER reporting_production_boundary_immutable BEFORE UPDATE OR DELETE + ON reporting_production_status_boundaries FOR EACH ROW + EXECUTE FUNCTION reporting_production_retained_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_notification_events'::regclass + AND tgname='reporting_production_event_immutable') THEN + CREATE TRIGGER reporting_production_event_immutable BEFORE UPDATE OR DELETE + ON reporting_production_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_production_retained_guard(); + END IF; + + EXECUTE $function$ + CREATE OR REPLACE FUNCTION reporting_production_work_guard() RETURNS trigger + LANGUAGE plpgsql AS $body$ + DECLARE a JSONB; result RECORD; activation RECORD; + BEGIN + IF TG_OP = 'DELETE' THEN + RAISE EXCEPTION 'reporting_materializer_history_immutable' USING ERRCODE='23514'; + END IF; + IF TG_OP = 'UPDATE' AND ( + (to_jsonb(NEW) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) IS DISTINCT FROM + (to_jsonb(OLD) - ARRAY['lease_token','lease_until','due_at','state','retry_allowed', + 'reason','acknowledged_at','completion_token']) OR OLD.state = 'acked' + ) THEN + RAISE EXCEPTION 'reporting_materializer_identity_immutable' USING ERRCODE='23514'; + END IF; + SELECT payload INTO a FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization_attempt' AND record_id = NEW.reporting_materialization_id; + IF a IS NULL OR a->>'reporting_revision_id' IS DISTINCT FROM NEW.reporting_revision_id + OR a #>> '{scope,reporting_obligation_id}' IS DISTINCT FROM NEW.reporting_obligation_id + OR a #>> '{scope,generation_key,delivery_config_id}' IS DISTINCT FROM NEW.delivery_config_id + OR (a #>> '{scope,generation_key,delivery_config_version}')::bigint + IS DISTINCT FROM NEW.delivery_config_version THEN + RAISE EXCEPTION 'reporting_materializer_attempt_mismatch' USING ERRCODE='23514'; + END IF; + SELECT activated_at, policy INTO activation FROM reporting_production_accounts + WHERE account_id=NEW.account_id; + IF NOT FOUND OR NEW.admission_epoch <> 2 OR NEW.imported + OR (a->>'created_at')::timestamptz < activation.activated_at + OR NOT (activation.policy->'verification_keys' ? NEW.verification_key_sha256) + OR (activation.policy->>'notifications_enabled')::boolean + IS DISTINCT FROM NEW.notifications_enabled THEN + RAISE EXCEPTION 'reporting_production_admission_required' USING ERRCODE='23514'; + END IF; + IF TG_OP='INSERT' AND EXISTS (SELECT 1 FROM reporting_materializer_work previous_work + WHERE previous_work.account_id=NEW.account_id AND previous_work.consumer_id=NEW.consumer_id + AND (previous_work.reporting_materialization_id=NEW.reporting_materialization_id OR + (previous_work.state='pending' AND previous_work.delivery_config_id=NEW.delivery_config_id + AND previous_work.delivery_config_version=NEW.delivery_config_version + AND previous_work.reporting_obligation_id=NEW.reporting_obligation_id))) THEN + RAISE EXCEPTION 'reporting_production_historical_work_conflict' USING ERRCODE='23514'; + END IF; + IF NEW.state = 'acked' THEN + IF TG_OP <> 'UPDATE' OR OLD.state <> 'pending' OR OLD.lease_token IS NULL + OR OLD.lease_until <= clock_timestamp() + OR NEW.completion_token IS DISTINCT FROM OLD.lease_token THEN + RAISE EXCEPTION 'reporting_materializer_fence_required' USING ERRCODE='23514'; + END IF; + SELECT payload INTO result FROM reporting_reconciliation_records + WHERE account_id = NEW.account_id AND consumer_id = NEW.consumer_id + AND namespace = 'materialization' AND record_id = NEW.reporting_materialization_id; + IF NOT FOUND OR (NEW.retry_allowed AND result.payload->>'status' <> 'failed') THEN + RAISE EXCEPTION 'reporting_materializer_outcome_required' USING ERRCODE='23514'; + END IF; + IF NOT EXISTS (SELECT 1 FROM reporting_production_status_boundaries b + WHERE b.account_id=NEW.account_id AND b.consumer_id=NEW.consumer_id + AND b.reporting_materialization_id=NEW.reporting_materialization_id + AND b.as_of=NEW.acknowledged_at + AND b.as_of=(result.payload->>'completed_at')::timestamptz) THEN + RAISE EXCEPTION 'reporting_materializer_capture_required' USING ERRCODE='23514'; + END IF; + IF NEW.notifications_enabled AND result.payload->>'status' <> 'failed' + AND NOT EXISTS (SELECT 1 FROM reporting_production_notification_events e + WHERE e.account_id=NEW.account_id AND e.consumer_namespace=NEW.consumer_id + AND e.reporting_materialization_id=NEW.reporting_materialization_id + AND e.admission_epoch=NEW.admission_epoch) THEN + RAISE EXCEPTION 'reporting_materializer_event_required' USING ERRCODE='23514'; + END IF; + END IF; + RETURN NEW; + END + $body$ $function$; + + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_work'::regclass + AND tgname='reporting_production_guard') THEN + CREATE TRIGGER reporting_production_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_work FOR EACH ROW + EXECUTE FUNCTION reporting_production_work_guard(); + END IF; + CREATE OR REPLACE FUNCTION reporting_production_old_reservation_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + DECLARE activation TIMESTAMPTZ; attempt JSONB; + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + SELECT activated_at INTO activation FROM reporting_production_accounts + WHERE account_id=NEW.account_id; + IF FOUND THEN + SELECT payload INTO attempt FROM reporting_reconciliation_records + WHERE account_id=NEW.account_id AND consumer_id=NEW.consumer_id + AND namespace='materialization_attempt' + AND record_id=NEW.reporting_materialization_id; + IF NOT NEW.imported OR attempt IS NULL + OR (attempt->>'created_at')::timestamptz >= activation THEN + RAISE EXCEPTION 'reporting_production_old_worker_fenced' USING ERRCODE='23514'; + END IF; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_materializer_work'::regclass + AND tgname='reporting_production_old_reservation_guard') THEN + CREATE TRIGGER reporting_production_old_reservation_guard BEFORE INSERT + ON reporting_materializer_work FOR EACH ROW + EXECUTE FUNCTION reporting_production_old_reservation_guard(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_generations ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + producer_key TEXT COLLATE "C" NOT NULL CHECK (producer_key ~ '^[0-9a-f]{64}$'), + source_binding JSONB NOT NULL CHECK (jsonb_typeof(source_binding)='object'), + PRIMARY KEY (account_id,delivery_config_id,delivery_config_version), + FOREIGN KEY (account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_configurations + ); + CREATE INDEX IF NOT EXISTS reporting_production_source_generations + ON reporting_production_generations(producer_key,account_id,delivery_config_id,delivery_config_version); + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_production_generations'::regclass + AND tgname='reporting_production_generation_immutable') THEN + CREATE TRIGGER reporting_production_generation_immutable BEFORE UPDATE OR DELETE + ON reporting_production_generations FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_production_destination_bindings ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + method JSONB NOT NULL CHECK (jsonb_typeof(method)='object'), + PRIMARY KEY (account_id,consumer_id,delivery_config_id,delivery_config_version), + FOREIGN KEY (account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_destination_bindings'::regclass + AND tgname='reporting_production_destination_immutable') THEN + CREATE TRIGGER reporting_production_destination_immutable BEFORE UPDATE OR DELETE + ON reporting_production_destination_bindings FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; +END +$production$; + +-- Global due sampling for the owned optional workers; account claims retain +-- the original SDK outbox transactions and recipient ordering. +DO $production_delivery_discovery$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + CREATE INDEX IF NOT EXISTS reporting_production_core_expansions_due + ON reporting_notification_expansions (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_core_deliveries_due + ON reporting_notification_deliveries (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_status_expansions_due + ON reporting_projection_notification_expansions (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_status_deliveries_due + ON reporting_projection_notification_deliveries (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_ready_expansions_due + ON reporting_production_notification_expansions (due_at,account_id) WHERE state IN ('pending','leased'); + CREATE INDEX IF NOT EXISTS reporting_production_ready_deliveries_due + ON reporting_production_notification_deliveries (due_at,account_id) WHERE state IN ('pending','leased'); +END +$production_delivery_discovery$; + +-- Generation-scoped producer closing and unfinished acquisition work. The +-- cursor never replaces original obligations, revision history or source keys. +DO $production_source_progress$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + -- Rejected admitted generations advance through the bounded discovery + -- window without acquiring an ordinary lease or changing frozen bindings. + CREATE TABLE IF NOT EXISTS reporting_production_source_probe_turns ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + probe_turn BIGINT NOT NULL CHECK (probe_turn>0), + PRIMARY KEY(account_id,delivery_config_id,delivery_config_version), + FOREIGN KEY(account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + CREATE TABLE IF NOT EXISTS reporting_production_source_progress ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + closed_through TIMESTAMPTZ, + acquisition_turn BIGINT NOT NULL DEFAULT 0 CHECK (acquisition_turn>=0), + PRIMARY KEY(account_id,delivery_config_id,delivery_config_version), + FOREIGN KEY(account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + CREATE TABLE IF NOT EXISTS reporting_production_source_work ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + delivery_config_version INTEGER NOT NULL, + reporting_obligation_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_obligations, + period_end TIMESTAMPTZ NOT NULL, + acquisition_turn BIGINT NOT NULL DEFAULT 0 CHECK (acquisition_turn>=0), + state TEXT COLLATE "C" NOT NULL DEFAULT 'pending' + CHECK (state IN ('pending','settled','parked')), + PRIMARY KEY(account_id,reporting_obligation_id), + FOREIGN KEY(account_id,delivery_config_id,delivery_config_version) + REFERENCES reporting_production_generations + ); + CREATE INDEX IF NOT EXISTS reporting_production_source_pending + ON reporting_production_source_work + (account_id,delivery_config_id,delivery_config_version,acquisition_turn,reporting_obligation_id) + INCLUDE(period_end) WHERE state='pending'; + + CREATE OR REPLACE FUNCTION reporting_production_source_progress_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + IF TG_OP='DELETE' OR (TG_OP='UPDATE' AND ( + (to_jsonb(NEW)-ARRAY['closed_through','acquisition_turn']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['closed_through','acquisition_turn']) + OR NEW.acquisition_turn < OLD.acquisition_turn + OR (OLD.closed_through IS NOT NULL AND + (NEW.closed_through IS NULL OR NEW.closed_through < OLD.closed_through)))) THEN + RAISE EXCEPTION 'reporting_production_source_progress_immutable' USING ERRCODE='23514'; + END IF; + IF NEW.closed_through IS NOT NULL AND NOT EXISTS ( + SELECT 1 FROM reporting_obligations o WHERE o.account_id=NEW.account_id + AND o.delivery_config_id=NEW.delivery_config_id + AND o.delivery_config_version=NEW.delivery_config_version + AND o.period_end=NEW.closed_through + ) THEN + RAISE EXCEPTION 'reporting_production_source_obligation_required' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_production_source_work_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + IF TG_OP='DELETE' OR (TG_OP='UPDATE' AND ( + (to_jsonb(NEW)-ARRAY['state','acquisition_turn']) IS DISTINCT FROM + (to_jsonb(OLD)-ARRAY['state','acquisition_turn']) + OR NEW.acquisition_turn < OLD.acquisition_turn)) THEN + RAISE EXCEPTION 'reporting_production_source_identity_immutable' USING ERRCODE='23514'; + END IF; + IF NOT EXISTS (SELECT 1 FROM reporting_obligations o + WHERE o.reporting_obligation_id=NEW.reporting_obligation_id + AND o.account_id=NEW.account_id AND o.delivery_config_id=NEW.delivery_config_id + AND o.delivery_config_version=NEW.delivery_config_version AND o.period_end=NEW.period_end) THEN + RAISE EXCEPTION 'reporting_production_source_obligation_required' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_production_source_dirty() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + -- SDK domain writers already hold the account lock before their row + -- mutation. New triggers retain that order and do no external I/O. + INSERT INTO reporting_production_source_work + (account_id,delivery_config_id,delivery_config_version,reporting_obligation_id,period_end) + SELECT o.account_id,o.delivery_config_id,o.delivery_config_version, + o.reporting_obligation_id,o.period_end FROM reporting_obligations o + JOIN reporting_production_generations g + USING(account_id,delivery_config_id,delivery_config_version) + WHERE o.reporting_obligation_id=NEW.reporting_obligation_id + AND o.account_id=NEW.account_id + ON CONFLICT(account_id,reporting_obligation_id) DO UPDATE SET state='pending'; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_source_progress'::regclass + AND tgname='reporting_production_source_progress_guard') THEN + CREATE TRIGGER reporting_production_source_progress_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_source_progress FOR EACH ROW + EXECUTE FUNCTION reporting_production_source_progress_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_production_source_work'::regclass + AND tgname='reporting_production_source_work_guard') THEN + CREATE TRIGGER reporting_production_source_work_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_production_source_work FOR EACH ROW + EXECUTE FUNCTION reporting_production_source_work_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_obligations'::regclass + AND tgname='reporting_production_source_obligation') THEN + CREATE TRIGGER reporting_production_source_obligation AFTER INSERT ON reporting_obligations + FOR EACH ROW EXECUTE FUNCTION reporting_production_source_dirty(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_revisions'::regclass + AND tgname='reporting_production_source_revision') THEN + CREATE TRIGGER reporting_production_source_revision AFTER INSERT OR UPDATE OF readable + ON reporting_revisions FOR EACH ROW EXECUTE FUNCTION reporting_production_source_dirty(); + END IF; +END +$production_source_progress$; diff --git a/src/adcp/reporting/ledger/reporting_projection.sql b/src/adcp/reporting/ledger/reporting_projection.sql new file mode 100644 index 000000000..04b4d0f86 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_projection.sql @@ -0,0 +1,242 @@ +-- B2.4 v2 capture and activation. All earlier manifests/SQL remain immutable. +-- create_schema() installs this inside the same serialized schema transaction. +DO $migration$ +DECLARE source_name TEXT; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + PERFORM account_id FROM reporting_status_scope_checkpoints LIMIT 0; + + CREATE TABLE IF NOT EXISTS reporting_projection_accounts ( + account_id TEXT COLLATE "C" PRIMARY KEY, + version INTEGER NOT NULL DEFAULT 2 CHECK (version = 2), + activated_at TIMESTAMPTZ NOT NULL, + notifications_enabled BOOLEAN NOT NULL, + consumer_status_enabled BOOLEAN NOT NULL, + ownership_enabled BOOLEAN NOT NULL, + policy JSONB NOT NULL, + legacy_through BIGINT NOT NULL CHECK (legacy_through >= 0), + legacy_capture_through BIGINT NOT NULL CHECK (legacy_capture_through >= 0), + legacy_capture_cursor BIGINT NOT NULL DEFAULT 0 + CHECK (legacy_capture_cursor BETWEEN 0 AND legacy_capture_through), + legacy_generation_floor BIGINT NOT NULL CHECK (legacy_generation_floor >= 0), + max_sequence BIGINT NOT NULL DEFAULT 0 CHECK (max_sequence >= 0), + cursor BIGINT NOT NULL DEFAULT 0 CHECK (cursor BETWEEN 0 AND max_sequence), + checkpoint_floor BIGINT NOT NULL DEFAULT 0 CHECK (checkpoint_floor >= 0), + current_input JSONB, + current_as_of TIMESTAMPTZ, + CHECK ((current_input IS NULL) = (current_as_of IS NULL)) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_pending + ON reporting_projection_accounts (account_id) WHERE cursor < max_sequence; + CREATE INDEX IF NOT EXISTS reporting_projection_activation_pending + ON reporting_projection_accounts (account_id) WHERE current_input IS NULL; + CREATE TABLE IF NOT EXISTS reporting_projection_writes ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + transaction_id TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, transaction_id) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_inputs ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + sequence BIGINT NOT NULL CHECK (sequence > 0), + transaction_id TEXT COLLATE "C" NOT NULL, + as_of TIMESTAMPTZ NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY (account_id, sequence), + UNIQUE (account_id, transaction_id), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(input)), + CHECK ((input->>'version')::integer IS NOT DISTINCT FROM 2), + CHECK ((input->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((input->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of) + ); + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_projection_inputs'::regclass + AND tgname='reporting_projection_input_immutable') THEN + CREATE TRIGGER reporting_projection_input_immutable BEFORE UPDATE OR DELETE + ON reporting_projection_inputs FOR EACH ROW + EXECUTE FUNCTION reporting_receipt_ingestion_immutable(); + END IF; + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_baselines ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + scope_key TEXT COLLATE "C" NOT NULL, + checkpoint JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY(account_id,scope_key), + CHECK (content_sha256=reporting_receipt_ingestion_sha256(checkpoint)) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_inputs ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + account_sequence BIGINT NOT NULL CHECK (account_sequence>0), + consumer_id TEXT COLLATE "C" NOT NULL, + kind TEXT COLLATE "C" NOT NULL CHECK (kind IN ('materializer','receipt')), + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY(account_id,account_sequence), + CHECK (content_sha256=reporting_receipt_ingestion_sha256(input)), + CHECK (input->>'account_id' IS NOT DISTINCT FROM account_id), + CHECK (input->>'consumer_id' IS NOT DISTINCT FROM consumer_id), + CHECK ((input->>'account_sequence')::bigint IS NOT DISTINCT FROM account_sequence) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_steps ( + account_id TEXT COLLATE "C" NOT NULL, + account_sequence BIGINT NOT NULL, + scope_key TEXT COLLATE "C" NOT NULL, + input JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + PRIMARY KEY(account_id,account_sequence,scope_key), + FOREIGN KEY(account_id,account_sequence) + REFERENCES reporting_projection_legacy_inputs, + CHECK (content_sha256=reporting_receipt_ingestion_sha256(input)), + CHECK ((input->>'admission_epoch')::integer IS NOT DISTINCT FROM 0) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_legacy_checkpoints ( + account_id TEXT COLLATE "C" NOT NULL REFERENCES reporting_projection_accounts, + consumer_id TEXT COLLATE "C" NOT NULL, + scope_key TEXT COLLATE "C" NOT NULL, + checkpoint JSONB NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + CHECK (content_sha256=reporting_receipt_ingestion_sha256(checkpoint)), + PRIMARY KEY(account_id,scope_key) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_legacy_consumer + ON reporting_projection_legacy_checkpoints(account_id,consumer_id,scope_key); + FOREACH source_name IN ARRAY ARRAY['reporting_projection_legacy_baselines', + 'reporting_projection_legacy_inputs','reporting_projection_legacy_steps'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=source_name::regclass + AND tgname='reporting_projection_legacy_immutable') THEN + EXECUTE format('CREATE TRIGGER reporting_projection_legacy_immutable' + ' BEFORE UPDATE OR DELETE ON %I FOR EACH ROW' + ' EXECUTE FUNCTION reporting_receipt_ingestion_immutable()',source_name); + END IF; + END LOOP; +END +$migration$; + +DO $migration$ +DECLARE source_name TEXT; +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + ALTER TABLE reporting_status_scope_checkpoints + ADD COLUMN IF NOT EXISTS projection_writer_floor INTEGER NOT NULL DEFAULT 1; + CREATE INDEX IF NOT EXISTS reporting_projection_due_clock + ON reporting_status_scope_checkpoints (next_due_at,account_id) + WHERE projection_writer_floor=2 AND next_due_at IS NOT NULL; + IF NOT EXISTS (SELECT 1 FROM pg_constraint + WHERE conrelid='reporting_status_scope_checkpoints'::regclass + AND conname='reporting_projection_writer_floor') THEN + ALTER TABLE reporting_status_scope_checkpoints + ADD CONSTRAINT reporting_projection_writer_floor CHECK (projection_writer_floor IN (1,2)); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_checkpoint_guard() RETURNS trigger + LANGUAGE plpgsql AS $function$ + DECLARE floor INTEGER; + BEGIN + floor := CASE WHEN TG_OP='INSERT' THEN NEW.projection_writer_floor + ELSE OLD.projection_writer_floor END; + IF floor=2 OR EXISTS (SELECT 1 FROM reporting_projection_accounts + WHERE account_id=NEW.account_id) THEN + IF current_setting('adcp.reporting.projection_version',true) IS DISTINCT FROM '2' THEN + RAISE EXCEPTION 'status_projection_writer_fenced' USING ERRCODE='23514'; + END IF; + NEW.projection_writer_floor := 2; + END IF; + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_status_scope_checkpoints'::regclass + AND tgname='reporting_projection_checkpoint_guard') THEN + CREATE TRIGGER reporting_projection_checkpoint_guard BEFORE INSERT OR UPDATE + ON reporting_status_scope_checkpoints FOR EACH ROW + EXECUTE FUNCTION reporting_projection_checkpoint_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_document(owner TEXT, at_time TIMESTAMPTZ) + RETURNS JSONB LANGUAGE plpgsql STABLE AS $function$ + DECLARE core JSONB; records JSONB; counts JSONB := '{}'; name TEXT; + BEGIN + core := reporting_status_projection_input(owner, at_time); + SELECT coalesce(jsonb_agg(jsonb_build_object( + 'consumer_id', c.consumer_id, 'sequence', c.seq, 'record', r.payload) + ORDER BY c.consumer_id COLLATE "C", c.seq), '[]') INTO records + FROM reporting_reconciliation_changes c JOIN reporting_reconciliation_records r + USING(account_id,consumer_id,namespace,record_id) + WHERE c.account_id=owner; + FOREACH name IN ARRAY ARRAY['configurations','obligations','revisions','statuses', + 'lifecycles','issue_scopes','adjustments','changes'] LOOP + counts := counts || jsonb_build_object(name, jsonb_array_length(core->name)); + END LOOP; + RETURN jsonb_build_object('version',2,'account_id',owner,'as_of',at_time, + 'core_format','sql-v1','core',core,'reconciliation',records, + 'counts',counts || jsonb_build_object('reconciliation',jsonb_array_length(records))); + END + $function$; + + CREATE OR REPLACE FUNCTION reporting_projection_capture(owner TEXT) RETURNS BIGINT + LANGUAGE plpgsql AS $function$ + DECLARE sequence BIGINT; document JSONB; at_time TIMESTAMPTZ; existing BIGINT; + BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || owner)); + SELECT i.sequence INTO existing FROM reporting_projection_inputs i + WHERE account_id=owner AND transaction_id=pg_current_xact_id()::text; + IF FOUND THEN RETURN existing; END IF; + at_time := nullif(current_setting('adcp.reporting.projection_clock',true),'')::timestamptz; + at_time := coalesce(at_time,clock_timestamp()); + UPDATE reporting_projection_accounts SET max_sequence=max_sequence+1 + WHERE account_id=owner RETURNING max_sequence INTO sequence; + IF NOT FOUND THEN RAISE EXCEPTION 'status_projection_activation_required' USING ERRCODE='23514'; END IF; + document := reporting_projection_document(owner,at_time); + INSERT INTO reporting_projection_inputs + (account_id,sequence,transaction_id,as_of,input,content_sha256) + VALUES(owner,sequence,pg_current_xact_id()::text,at_time,document, + reporting_receipt_ingestion_sha256(document)); + RETURN sequence; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_projection_mark() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + IF current_setting('adcp.reporting.projection_internal',true)='on' OR NOT EXISTS + (SELECT 1 FROM reporting_projection_accounts WHERE account_id=NEW.account_id) THEN + RETURN NEW; + END IF; + IF TG_OP='UPDATE' AND to_jsonb(NEW)=to_jsonb(OLD) THEN RETURN NEW; END IF; + IF TG_TABLE_NAME='reporting_configurations' AND TG_OP='UPDATE' AND + (to_jsonb(NEW)-ARRAY['lease_worker_id','lease_expires_at']) = + (to_jsonb(OLD)-ARRAY['lease_worker_id','lease_expires_at']) THEN + RETURN NEW; + END IF; + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting:' || NEW.account_id)); + IF EXISTS (SELECT 1 FROM reporting_projection_inputs + WHERE account_id=NEW.account_id AND transaction_id=pg_current_xact_id()::text) THEN + RAISE EXCEPTION 'status_projection_boundary_already_captured' USING ERRCODE='23514'; + END IF; + INSERT INTO reporting_projection_writes VALUES(NEW.account_id,pg_current_xact_id()::text) + ON CONFLICT DO NOTHING; + RETURN NEW; + END + $function$; + CREATE OR REPLACE FUNCTION reporting_projection_commit() RETURNS trigger + LANGUAGE plpgsql AS $function$ + BEGIN + PERFORM reporting_projection_capture(NEW.account_id); + RETURN NEW; + END + $function$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid='reporting_projection_writes'::regclass + AND tgname='reporting_projection_commit') THEN + CREATE CONSTRAINT TRIGGER reporting_projection_commit AFTER INSERT + ON reporting_projection_writes DEFERRABLE INITIALLY DEFERRED + FOR EACH ROW EXECUTE FUNCTION reporting_projection_commit(); + END IF; + FOREACH source_name IN ARRAY ARRAY['reporting_configurations','reporting_obligations', + 'reporting_revisions','reporting_adjustments','reporting_consumer_statuses', + 'reporting_issue_lifecycle','reporting_issue_status_scopes','reporting_reconciliation_changes'] LOOP + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid=source_name::regclass + AND tgname='reporting_projection_mark') THEN + EXECUTE format('CREATE TRIGGER reporting_projection_mark AFTER INSERT OR UPDATE ON %I' + ' FOR EACH ROW EXECUTE FUNCTION reporting_projection_mark()',source_name); + END IF; + END LOOP; +END +$migration$; diff --git a/src/adcp/reporting/ledger/reporting_projection_feed.sql b/src/adcp/reporting/ledger/reporting_projection_feed.sql new file mode 100644 index 000000000..9f5311909 --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_projection_feed.sql @@ -0,0 +1,43 @@ +-- B2.4 snapshots use a new table; legacy B2.3 rows keep their original representation. +DO $migration$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.projection.feed.schema')); + CREATE TABLE IF NOT EXISTS reporting_projection_feed_snapshots ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_id TEXT COLLATE "C" NOT NULL, + snapshot_id TEXT COLLATE "C" NOT NULL CHECK (snapshot_id ~ '^rpfs_[0-9a-f]{32}$'), + as_of TIMESTAMPTZ NOT NULL, + representation_version INTEGER NOT NULL CHECK (representation_version = 2), + ownership_mode TEXT COLLATE "C" NOT NULL CHECK (ownership_mode IN ('absent','bindings')), + document TEXT NOT NULL, + content_sha256 TEXT COLLATE "C" NOT NULL, + signing_key BYTEA NOT NULL CHECK (octet_length(signing_key) = 32), + PRIMARY KEY (account_id, consumer_id, snapshot_id), + UNIQUE (snapshot_id), + CHECK (content_sha256 = encode(sha256(convert_to(document,'UTF8')), 'hex')), + CHECK (content_sha256 = reporting_receipt_ingestion_sha256(document::jsonb)), + CHECK ((document::jsonb->>'version')::integer IS NOT DISTINCT FROM 1), + CHECK ((document::jsonb->>'account_id') IS NOT DISTINCT FROM account_id), + CHECK ((document::jsonb->>'consumer_id') IS NOT DISTINCT FROM consumer_id), + CHECK ((document::jsonb->>'snapshot_id') IS NOT DISTINCT FROM snapshot_id), + CHECK ((document::jsonb->>'as_of')::timestamptz IS NOT DISTINCT FROM as_of), + CHECK ((document::jsonb->>'representation_version')::integer IS NOT DISTINCT FROM representation_version), + CHECK ((document::jsonb->>'ownership_mode') IS NOT DISTINCT FROM ownership_mode), + CHECK (jsonb_typeof(document::jsonb->'records') IS NOT DISTINCT FROM 'array'), + CHECK (jsonb_array_length(document::jsonb->'records') IS NOT DISTINCT FROM (document::jsonb->>'total_count')::integer), + CHECK (jsonb_typeof(document::jsonb->'inputs') IS NOT DISTINCT FROM 'object'), + CHECK (document::jsonb - ARRAY['version','representation_version','ownership_mode', + 'account_id','consumer_id','snapshot_id','as_of','after','through','filters', + 'common','total_count','records','inputs'] = '{}'::jsonb) + ); + CREATE OR REPLACE FUNCTION reporting_projection_feed_immutable() + RETURNS TRIGGER LANGUAGE plpgsql AS $function$ + BEGIN + RAISE EXCEPTION 'reporting feed snapshot is immutable' USING ERRCODE = '23514'; + END + $function$; + DROP TRIGGER IF EXISTS reporting_projection_feed_immutable ON reporting_projection_feed_snapshots; + CREATE TRIGGER reporting_projection_feed_immutable BEFORE UPDATE OR DELETE ON reporting_projection_feed_snapshots + FOR EACH ROW EXECUTE FUNCTION reporting_projection_feed_immutable(); +END +$migration$; diff --git a/src/adcp/reporting/ledger/reporting_projection_notifications.sql b/src/adcp/reporting/ledger/reporting_projection_notifications.sql new file mode 100644 index 000000000..feb9ad8dc --- /dev/null +++ b/src/adcp/reporting/ledger/reporting_projection_notifications.sql @@ -0,0 +1,296 @@ +-- B2.4 status events have isolated queues. A/B/C workers cannot claim them. +-- The same reviewed checkpoint, fanout and delivery state machines apply. +DO $projection_notifications$ +BEGIN + PERFORM pg_advisory_xact_lock(hashtext('adcp.reporting.schema'), hashtext(current_schema())); + CREATE TABLE IF NOT EXISTS reporting_projection_notification_events ( + account_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK + (notification_type = 'reporting.status_changed'), + cause_kind TEXT COLLATE "C" NOT NULL CHECK + (cause_kind = 'status_changed'), + cause_id TEXT COLLATE "C" NOT NULL CHECK (length(cause_id) > 0), + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + delivery_config_id TEXT COLLATE "C" NOT NULL, + version BIGINT NOT NULL CHECK (version > 0), + scope_kind TEXT COLLATE "C" NOT NULL CHECK (scope_kind IN ('configuration','obligation')), + obligation_namespace TEXT COLLATE "C" NOT NULL, + fingerprint TEXT NOT NULL CHECK (fingerprint ~ '^[0-9a-f]{64}$'), + admission_epoch BIGINT NOT NULL DEFAULT 2 CHECK (admission_epoch=2), + fired_at TIMESTAMPTZ NOT NULL, + snapshot JSONB NOT NULL, + PRIMARY KEY (account_id, consumer_namespace, notification_id), + UNIQUE (account_id, consumer_namespace, delivery_config_id, version, + scope_kind, obligation_namespace, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, delivery_config_id, version, + scope_kind, obligation_namespace) + REFERENCES reporting_status_scope_checkpoints, + CHECK ((scope_kind = 'configuration') = (obligation_namespace = '')), + UNIQUE (account_id, consumer_namespace, notification_type, + cause_kind, cause_id, cause_generation), + UNIQUE (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + CHECK ((snapshot #>> '{cause,kind}') IS NOT DISTINCT FROM cause_kind), + CHECK ((snapshot #>> '{cause,fingerprint}') IS NOT DISTINCT FROM fingerprint), + CHECK ((snapshot #>> '{cause,checkpoint_generation}')::bigint IS NOT DISTINCT FROM cause_generation), + CHECK ((snapshot #>> '{cause,scope,account_id}') IS NOT DISTINCT FROM account_id), + CHECK ((snapshot #>> '{cause,scope,generation_key,account_id}') IS NOT DISTINCT FROM account_id), + CHECK (coalesce(snapshot #>> '{cause,scope,consumer_id}', '') = consumer_namespace), + CHECK ((snapshot #>> '{cause,scope,generation_key,delivery_config_id}') IS NOT DISTINCT FROM delivery_config_id), + CHECK ((snapshot #>> '{cause,scope,generation_key,delivery_config_version}')::bigint IS NOT DISTINCT FROM version), + CHECK (coalesce(snapshot #>> '{cause,scope,reporting_obligation_id}', '') = obligation_namespace), + CHECK ((snapshot->>'account_id') IS NOT DISTINCT FROM account_id AND + (snapshot->>'notification_id') IS NOT DISTINCT FROM notification_id), + CHECK ((snapshot->>'cause_generation')::bigint IS NOT DISTINCT FROM cause_generation), + CHECK ((snapshot->>'fired_at')::timestamptz IS NOT DISTINCT FROM fired_at) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_notification_expansions ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL DEFAULT '', + notification_id TEXT COLLATE "C" NOT NULL, + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, notification_id, emission_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id) + REFERENCES reporting_projection_notification_events (account_id, consumer_namespace, notification_id) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_notification_expansions_due + ON reporting_projection_notification_expansions (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + CREATE TABLE IF NOT EXISTS reporting_projection_notification_deliveries ( + account_id TEXT COLLATE "C" NOT NULL, + delivery_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + notification_type TEXT COLLATE "C" NOT NULL CHECK (notification_type = 'reporting.status_changed'), + emission_generation BIGINT NOT NULL CHECK (emission_generation > 0), + idempotency_key TEXT COLLATE "C" NOT NULL, + destination_sha256 TEXT NOT NULL, + subscription_fingerprint TEXT NOT NULL, + signing_scope_id TEXT COLLATE "C", + cause_kind TEXT COLLATE "C" NOT NULL CHECK (cause_kind = 'status_changed'), + cause_id TEXT COLLATE "C" NOT NULL, + cause_generation BIGINT NOT NULL CHECK (cause_generation > 0), + consumer_namespace TEXT COLLATE "C" NOT NULL, + auth_mode TEXT NOT NULL, + body_sha256 TEXT NOT NULL, + envelope_version INTEGER NOT NULL, + key_version TEXT COLLATE "C" NOT NULL, + envelope BYTEA NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' CHECK + (state IN ('pending', 'leased', 'complete', 'suppressed', 'quarantined')), + due_at TIMESTAMPTZ NOT NULL, + lease_token TEXT, + lease_expires_at TIMESTAMPTZ, + claim_count BIGINT NOT NULL DEFAULT 0, + error_code TEXT CHECK (error_code IN ( + 'network', 'retryable_http', 'permanent_http', 'signing_unavailable', + 'permanent_scope', 'subscription_unavailable', 'subscription_changed', + 'invalid_configuration', 'invalid_payload', 'integrity_failure', 'lease_expired')), + PRIMARY KEY (account_id, consumer_namespace, delivery_id), + UNIQUE (account_id, consumer_namespace, notification_id, emission_generation, subscriber_id), + UNIQUE (account_id, consumer_namespace, idempotency_key), + CHECK (length(principal_id) > 0 AND (consumer_namespace = '' OR consumer_namespace = principal_id)), + FOREIGN KEY (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation) + REFERENCES reporting_projection_notification_events + (account_id, consumer_namespace, notification_id, cause_kind, cause_id, cause_generation), + FOREIGN KEY (account_id, consumer_namespace, notification_id, emission_generation) + REFERENCES reporting_projection_notification_expansions + (account_id, consumer_namespace, notification_id, emission_generation) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_notification_deliveries_due + ON reporting_projection_notification_deliveries (account_id, due_at) + WHERE state IN ('pending', 'leased'); + + + CREATE TABLE IF NOT EXISTS reporting_projection_webhook_attempt_heads ( + account_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL CHECK (length(principal_id) > 0), + subscriber_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + last_attempt BIGINT NOT NULL CHECK (last_attempt > 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + ); + CREATE TABLE IF NOT EXISTS reporting_projection_webhook_attempts ( + account_id TEXT COLLATE "C" NOT NULL, + principal_id TEXT COLLATE "C" NOT NULL, + subscriber_id TEXT COLLATE "C" NOT NULL, + notification_id TEXT COLLATE "C" NOT NULL, + idempotency_key TEXT COLLATE "C" NOT NULL, + attempt BIGINT NOT NULL CHECK (attempt > 0), + delivery_id TEXT COLLATE "C" NOT NULL, + consumer_namespace TEXT COLLATE "C" NOT NULL, + lease_token TEXT NOT NULL, + reservation_token TEXT NOT NULL, + binding JSONB NOT NULL, + fired_at TIMESTAMPTZ NOT NULL, + url TEXT NOT NULL CONSTRAINT reporting_projection_webhook_url_safe + CHECK (length(url) <= 8192 AND url !~ '[?#@]' AND url ~ '^https?://'), + payload_size_bytes BIGINT NOT NULL CHECK (payload_size_bytes >= 0), + status TEXT NOT NULL DEFAULT 'pending' CHECK + (status IN ('pending', 'success', 'failed', 'timeout', 'connection_error')), + completed_at TIMESTAMPTZ, + http_status_code INTEGER, + response_time_ms BIGINT CHECK (response_time_ms >= 0), + PRIMARY KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key, attempt), + UNIQUE (account_id, consumer_namespace, principal_id, delivery_id, lease_token), + FOREIGN KEY (account_id, consumer_namespace, delivery_id) + REFERENCES reporting_projection_notification_deliveries, + FOREIGN KEY (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key) + REFERENCES reporting_projection_webhook_attempt_heads + (account_id, consumer_namespace, principal_id, subscriber_id, idempotency_key), + CONSTRAINT reporting_projection_webhook_identity CHECK ( + (binding->>'account_id') IS NOT DISTINCT FROM account_id AND + (binding->>'principal_id') IS NOT DISTINCT FROM principal_id AND + (binding->>'subscriber_id') IS NOT DISTINCT FROM subscriber_id AND + (binding->>'notification_id') IS NOT DISTINCT FROM notification_id AND + (binding->>'idempotency_key') IS NOT DISTINCT FROM idempotency_key AND + (binding->>'delivery_id') IS NOT DISTINCT FROM delivery_id AND + (binding->>'consumer_namespace') IS NOT DISTINCT FROM consumer_namespace), + CONSTRAINT reporting_projection_webhook_completion CHECK ((status = 'pending') = (completed_at IS NULL)), + CONSTRAINT reporting_projection_webhook_timestamps CHECK (completed_at >= fired_at), + CONSTRAINT reporting_projection_webhook_outcome CHECK ( + (status IN ('pending', 'timeout', 'connection_error') + AND http_status_code IS NULL AND response_time_ms IS NULL) + OR (status IN ('success', 'failed') AND http_status_code BETWEEN 100 AND 599 + AND http_status_code IS NOT NULL AND response_time_ms IS NOT NULL + AND ((status = 'success') = (http_status_code BETWEEN 200 AND 299)))) + ); + CREATE INDEX IF NOT EXISTS reporting_projection_webhook_activity_newest ON reporting_projection_webhook_attempts + (account_id, principal_id, fired_at DESC, notification_id DESC, + idempotency_key DESC, subscriber_id DESC, attempt DESC, delivery_id DESC); + CREATE INDEX IF NOT EXISTS reporting_projection_webhook_activity_retention ON reporting_projection_webhook_attempts + (account_id, principal_id, completed_at) WHERE completed_at IS NOT NULL; + + -- Retention never resets a logical delivery's sequence, even after all + -- terminal attempts are purged. Writers always lock parent, then head. + CREATE OR REPLACE FUNCTION reporting_projection_webhook_head_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $head_guard$ + BEGIN + IF TG_OP = 'DELETE' OR + (TG_OP = 'INSERT' AND NEW.last_attempt <> 1) OR + (TG_OP = 'UPDATE' AND (NEW.last_attempt <> OLD.last_attempt + 1 OR + (to_jsonb(NEW) - 'last_attempt') <> (to_jsonb(OLD) - 'last_attempt'))) THEN + RAISE EXCEPTION 'reporting activity counter must advance and be retained' + USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $head_guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_projection_webhook_attempt_heads'::regclass + AND tgname = 'reporting_projection_webhook_head_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_projection_webhook_head_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_projection_webhook_attempt_heads FOR EACH ROW EXECUTE FUNCTION reporting_projection_webhook_head_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_webhook_attempt_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $guard$ + DECLARE + delivery reporting_projection_notification_deliveries; + BEGIN + IF TG_OP = 'DELETE' THEN + IF OLD.completed_at IS NULL THEN + RAISE EXCEPTION 'pending reporting activity must be retained' USING ERRCODE = '23514'; + END IF; + RETURN OLD; + END IF; + IF TG_OP = 'UPDATE' AND ( + OLD.status <> 'pending' OR NEW.status = 'pending' OR + (to_jsonb(NEW) - ARRAY['status','completed_at','http_status_code','response_time_ms']) <> + (to_jsonb(OLD) - ARRAY['status','completed_at','http_status_code','response_time_ms'])) THEN + RAISE EXCEPTION 'immutable reporting activity reservation' USING ERRCODE = '23514'; + END IF; + -- The opaque reservation token fences terminalization in the UPDATE + -- predicate. A known late response may finish its own reservation after + -- lease expiry/reclaim; it must not require or modify the parent lease. + IF TG_OP = 'UPDATE' THEN + RETURN NEW; + END IF; + IF TG_OP = 'INSERT' AND NEW.status <> 'pending' THEN + RAISE EXCEPTION 'reporting activity requires reservation' USING ERRCODE = '23514'; + END IF; + SELECT * INTO delivery FROM reporting_projection_notification_deliveries + WHERE account_id = NEW.account_id AND principal_id = NEW.principal_id + AND consumer_namespace = NEW.consumer_namespace AND delivery_id = NEW.delivery_id + AND subscriber_id = NEW.subscriber_id AND notification_id = NEW.notification_id + AND idempotency_key = NEW.idempotency_key + AND state = 'leased' AND lease_token = NEW.lease_token + AND lease_expires_at > coalesce(NEW.completed_at, NEW.fired_at) FOR UPDATE; + IF NOT FOUND OR + NEW.binding->>'account_id' IS DISTINCT FROM NEW.account_id OR + NEW.binding->>'principal_id' IS DISTINCT FROM NEW.principal_id OR + NEW.binding->>'subscriber_id' IS DISTINCT FROM NEW.subscriber_id OR + NEW.binding->>'notification_id' IS DISTINCT FROM NEW.notification_id OR + NEW.binding->>'idempotency_key' IS DISTINCT FROM NEW.idempotency_key OR + NEW.binding->>'delivery_id' IS DISTINCT FROM NEW.delivery_id OR + NEW.binding->>'consumer_namespace' IS DISTINCT FROM NEW.consumer_namespace OR + NEW.binding->>'notification_type' IS DISTINCT FROM delivery.notification_type OR + NEW.binding->>'body_sha256' IS DISTINCT FROM delivery.body_sha256 THEN + RAISE EXCEPTION 'reporting activity lease or identity mismatch' USING ERRCODE = '23514'; + END IF; + RETURN NEW; + END; + $guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger WHERE tgrelid = 'reporting_projection_webhook_attempts'::regclass + AND tgname = 'reporting_projection_webhook_attempt_guard' AND NOT tgisinternal) THEN + CREATE TRIGGER reporting_projection_webhook_attempt_guard BEFORE INSERT OR UPDATE OR DELETE + ON reporting_projection_webhook_attempts FOR EACH ROW EXECUTE FUNCTION reporting_projection_webhook_attempt_guard(); + END IF; + + CREATE OR REPLACE FUNCTION reporting_projection_event_guard() + RETURNS TRIGGER LANGUAGE plpgsql AS $event_guard$ + DECLARE checkpoint reporting_status_scope_checkpoints; ids jsonb; + BEGIN + IF NOT EXISTS (SELECT 1 FROM reporting_projection_accounts + WHERE account_id=NEW.account_id AND current_input IS NOT NULL + AND notifications_enabled) THEN + RAISE EXCEPTION 'status_projection_activation_required' USING ERRCODE='23514'; + END IF; + SELECT * INTO checkpoint FROM reporting_status_scope_checkpoints + WHERE account_id=NEW.account_id AND consumer_namespace=NEW.consumer_namespace + AND delivery_config_id=NEW.delivery_config_id AND version=NEW.version + AND scope_kind=NEW.scope_kind AND obligation_namespace=NEW.obligation_namespace; + IF NOT FOUND OR NOT checkpoint.initialized OR NOT checkpoint.publishable + OR checkpoint.generation<>NEW.cause_generation OR checkpoint.fingerprint<>NEW.fingerprint + OR checkpoint.snapshot->>'health' IS DISTINCT FROM NEW.snapshot #>> '{cause,health}' THEN + RAISE EXCEPTION 'invalid status event checkpoint' USING ERRCODE='23514'; + END IF; + SELECT coalesce(jsonb_agg(issue_id ORDER BY issue_id COLLATE "C"), '[]') INTO ids + FROM (SELECT DISTINCT i->>'issue_id' COLLATE "C" AS issue_id + FROM jsonb_array_elements(checkpoint.snapshot->'issues') i + ORDER BY issue_id LIMIT 16) selected; + IF ids IS DISTINCT FROM NEW.snapshot #> '{cause,issue_ids}' THEN + RAISE EXCEPTION 'invalid status event issues' USING ERRCODE='23514'; + END IF; + RETURN NEW; + END + $event_guard$; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_projection_notification_events'::regclass + AND tgname='reporting_projection_event_guard') THEN + CREATE TRIGGER reporting_projection_event_guard BEFORE INSERT + ON reporting_projection_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_projection_event_guard(); + END IF; + IF NOT EXISTS (SELECT 1 FROM pg_trigger + WHERE tgrelid='reporting_projection_notification_events'::regclass + AND tgname='reporting_projection_event_immutable') THEN + CREATE TRIGGER reporting_projection_event_immutable BEFORE UPDATE OR DELETE + ON reporting_projection_notification_events FOR EACH ROW + EXECUTE FUNCTION reporting_notification_immutable(); + END IF; +END +$projection_notifications$; diff --git a/src/adcp/reporting/ledger/schedule.py b/src/adcp/reporting/ledger/schedule.py new file mode 100644 index 000000000..13e6c8aac --- /dev/null +++ b/src/adcp/reporting/ledger/schedule.py @@ -0,0 +1,95 @@ +"""One captured-generation clock for producer obligations and status forecasts. + +A generation owes full periods starting at/after activation and strictly before +deactivation. Deactivation after a period starts keeps that entire period and +its original SLA. Forecasting never creates an obligation or changes health. +""" + +from __future__ import annotations + +from collections.abc import Iterator, Sequence +from datetime import datetime + +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingObligationRecord, + ReportingPeriodBoundary, + _period_instants, + _schedule_clock, + derive_period, + first_ordinal_after, + iso_duration_to_timedelta, +) + + +def committed_periods( + configuration: ReportingConfiguration, *, near: datetime | None = None +) -> Iterator[ReportingPeriodBoundary]: + """Yield full committed periods, optionally seeking near an expected-at time. + + ``near`` is an optimization, not a filter: the caller still compares exact + instants. Two predecessor civil slots retain the period containing the + requested instant, including its DST fold. No wall clock is consulted. + """ + activated = configuration.activated_at + if activated is None: + return + schedule, timezone = configuration.schedule, configuration.account_timezone + zone, duration, anchor = _schedule_clock(schedule, timezone) + ordinal = first_ordinal_after(schedule, account_timezone=timezone, activated_at=activated) + if near is not None: + candidate = first_ordinal_after( + schedule, + account_timezone=timezone, + activated_at=near - iso_duration_to_timedelta(schedule.delivery_sla), + ) + ordinal = max(ordinal, candidate - 2) + while True: + start, end = _period_instants(schedule, timezone, ordinal) + if configuration.deactivated_at is not None and start >= configuration.deactivated_at: + return + local_start = anchor + duration * ordinal + if end > start and start.astimezone(zone).replace(tzinfo=None) == local_start: + yield derive_period(schedule, account_timezone=timezone, ordinal=ordinal) + ordinal += 1 + + +def next_reporting_expectation( + configurations: Sequence[ReportingConfiguration], + obligations: Sequence[ReportingObligationRecord], + *, + as_of: datetime, + period_start: datetime | None = None, + period_end: datetime | None = None, +) -> datetime | None: + """Nearest future due instant in the selected frozen schedule/period scope. + + Existing obligations remain commitments even if their registry generation + has aged out. They supplement, rather than replace, captured schedules. + This value is independent of current health and of record pagination. + """ + future = [ + item.period.expected_at + for item in obligations + if item.period.expected_at > as_of + and (period_start is None or item.period.end > period_start) + and (period_end is None or item.period.start < period_end) + ] + for configuration in configurations: + # A requested historical horizon does not create an unbounded walk. + # Seek close to the later of the SLA cutoff and its lower period edge. + near = as_of + if period_start is not None: + near = max( + near, period_start + iso_duration_to_timedelta(configuration.schedule.delivery_sla) + ) + for period in committed_periods(configuration, near=near): + if period_end is not None and period.start >= period_end: + break + if period.expected_at <= as_of or ( + period_start is not None and period.end <= period_start + ): + continue + future.append(period.expected_at) + break + return min(future) if future else None diff --git a/src/adcp/reporting/ledger/status.py b/src/adcp/reporting/ledger/status.py index f2941bb85..5c9702000 100644 --- a/src/adcp/reporting/ledger/status.py +++ b/src/adcp/reporting/ledger/status.py @@ -36,6 +36,7 @@ from typing import Any, Literal from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.models import ( ConsumerStatusRecord, ReportingAdjustmentRecord, @@ -47,6 +48,7 @@ ReportingRevisionRecord, ) from adcp.reporting.ledger.notification_models import ReportingStatusScope +from adcp.reporting.ledger.schedule import next_reporting_expectation from adcp.reporting.ledger.status_projection import ( ReportingStatusSnapshot, StatusProjectionInput, @@ -216,6 +218,8 @@ def render_snapshot( *, caller: ReportingStatusCaller, snapshot: ReportingStatusSnapshot, + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None, + revision_ownership: bool = False, ) -> dict[str, Any]: """Render captured database evidence without any store calls or clock reads.""" view = request.get("view", "summary") @@ -226,19 +230,42 @@ def render_snapshot( filters = _filters(request) scope = ReportingStatusScope( caller.account_id, - consumer_id=caller.consumer_id if self._consumer_status_enabled else None, + consumer_id=( + caller.consumer_id + if self._consumer_status_enabled or reconciliation is not None + else None + ), ) snapshot_id = ( "rpls_" + _fingerprint( [ caller.account_id, - caller.consumer_id if self._consumer_status_enabled else None, + ( + caller.consumer_id + if self._consumer_status_enabled or reconciliation is not None + else None + ), filters, snapshot.max_sequence, ] )[:32] ) + if reconciliation is not None: + from adcp.reporting.ledger._delivery_state import fingerprint, principal + + reconciliation = tuple( + r + for r in reconciliation + if principal(r).account_id == caller.account_id + and principal(r).consumer_id == caller.consumer_id + ) + snapshot_id = ( + "rpls_" + + _fingerprint( + [snapshot_id, [fingerprint(r) for r in reconciliation], _iso(snapshot.as_of)] + )[:32] + ) offset = 0 cursor = (request.get("pagination") or {}).get("cursor") if cursor: @@ -261,6 +288,8 @@ def render_snapshot( tuple(filters["feed_purposes"] or ()), _parse(filters["period_start"]), _parse(filters["period_end"]), + reconciliation=reconciliation, + consumer_status_enabled=self._consumer_status_enabled, ) result = project_status_scope(value) if result.intents: @@ -295,7 +324,7 @@ def render_snapshot( ), None, ) - return { + response = { **common, "revision": _revision_to_wire(revision, owner), "adjustments": [ @@ -307,6 +336,19 @@ def render_snapshot( "receipts": [], "pagination": {"total_count": 1, "has_more": False}, } + if reconciliation is not None: + from adcp.reporting.projection.wire import exact_revision_evidence + + response.update( + exact_revision_evidence(snapshot, revision, owner, reconciliation, caller) + ) + if revision_ownership: + from adcp.reporting.ownership import with_revision_ownership + + response = with_revision_ownership( + response, {revision.reporting_revision_id: revision.reporting_obligation_id} + ) + return response common["scope"] = _scope_to_wire( result.configurations, ledger_as_of=snapshot.as_of, @@ -326,17 +368,43 @@ def render_snapshot( if self._consumer_status_enabled: counts["consumer_status_pending"] = result.pending_count watermark = _scope_data_through(p.projection for p in result.obligations) - next_expected = _next_expected(obligations, ledger_as_of=snapshot.as_of) + next_expected = next_reporting_expectation( + tuple( + c + for c in result.configurations + if (not request.get("finality") or c.required_finality in request["finality"]) + and ( + not request.get("health") + or project_status_scope( + replace( + value, + scope=ReportingStatusScope( + c.account_id, c.generation_key, consumer_id=scope.consumer_id + ), + ) + ).health + in request["health"] + ) + ), + tuple( + p.obligation + for p in result.obligations + if ( + not request.get("finality") + or p.obligation.required_finality in request["finality"] + ) + and (not request.get("health") or p.projection.health in request["health"]) + ), + as_of=snapshot.as_of, + period_start=value.period_start, + period_end=value.period_end, + ) return { **common, "health": result.health, "coverage": _coverage_roll_up(obligations, as_of=snapshot.as_of), "data_through": _iso(watermark) if watermark else None, - **( - {"next_expected_at": next_expected} - if next_expected is not None and result.health != "complete" - else {} - ), + **({"next_expected_at": _iso(next_expected)} if next_expected is not None else {}), "obligation_counts": counts, "issues": [i.to_wire() for i in result.issues], } @@ -513,17 +581,6 @@ def _scope_data_through(projections: Any) -> datetime | None: return min(watermarks) if watermarks else None -def _next_expected( - obligations: Sequence[ReportingObligationRecord], *, ledger_as_of: datetime -) -> str | None: - upcoming = [ - item.period.expected_at - for item in obligations - if _utc(item.period.expected_at) > _utc(ledger_as_of) - ] - return _iso(min(upcoming)) if upcoming else None - - def _scope_to_wire( configurations: Sequence[ReportingConfiguration], *, diff --git a/src/adcp/reporting/ledger/status_projection.py b/src/adcp/reporting/ledger/status_projection.py index 481969b9b..e33680674 100644 --- a/src/adcp/reporting/ledger/status_projection.py +++ b/src/adcp/reporting/ledger/status_projection.py @@ -11,7 +11,7 @@ from collections.abc import Sequence from dataclasses import asdict, dataclass, replace from datetime import datetime, timedelta -from typing import Any, Literal, cast +from typing import TYPE_CHECKING, Any, Literal, cast from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.ledger.consumer_status import ( @@ -21,6 +21,7 @@ project_consumer_mismatch, stale_received_grace_deadline, ) +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.health import ( ObligationProjection, aggregate_reporting_health, @@ -47,6 +48,9 @@ ) from adcp.reporting.revision_selection import select_reporting_revision +if TYPE_CHECKING: + from adcp.reporting.ledger.reconciliation_projection import ReconciliationProjection + @dataclass(frozen=True) class ReportingStatusSnapshot: @@ -85,6 +89,10 @@ class StatusProjectionInput: feed_purposes: tuple[str, ...] = () period_start: datetime | None = None period_end: datetime | None = None + # None selects the immutable legacy C representation. Versioned callers + # pass the complete captured account history, even when it is empty. + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None + consumer_status_enabled: bool = True @dataclass(frozen=True) @@ -93,6 +101,7 @@ class StatusObligationProjection: projection: ObligationProjection revisions: tuple[ReportingRevisionRecord, ...] statuses: tuple[ConsumerStatusRecord, ...] + reconciliation: ReconciliationProjection | None = None @dataclass(frozen=True) @@ -388,7 +397,7 @@ def status_retained_from( return max( ( ( - max(as_of - timedelta(days=c.status_retention_days), c.activated_at) + max(as_of - timedelta(days=c.status_retention_days), min(c.activated_at, as_of)) if c.activated_at is not None else as_of - timedelta(days=c.status_retention_days) ) @@ -463,7 +472,9 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ intents = tuple( i for i in lifecycle_intents(snapshot) - if _selected(i.scope, scope) and i.scope.generation_key in generations + if _selected(i.scope, scope) + and i.scope.generation_key in generations + and (value.consumer_status_enabled or i.scope.consumer_id is None) ) live = {i.issue_key: i for i in snapshot.lifecycles if i.live} issue_scopes = dict(snapshot.issue_scopes) @@ -537,7 +548,9 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ statuses = tuple( s for s in snapshot.statuses - if s.consumer_id == scope.consumer_id and status_matches_obligation(s, obligation) + if value.consumer_status_enabled + and s.consumer_id == scope.consumer_id + and status_matches_obligation(s, obligation) ) projection = project_obligation_health( obligation, @@ -568,7 +581,8 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ projection = replace(projection, health="action_required") current = current_consumer_statement(statuses) if ( - scope.consumer_id is not None + value.consumer_status_enabled + and scope.consumer_id is not None and current is None and (snapshot.as_of >= obligation.automated_recovery_deadline_at) ): @@ -617,8 +631,55 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ candidates.add( lifecycle.opened_at + value.escalation.consumer_mismatch_escalation ) + reconciliation = None + if value.reconciliation is not None: + from adcp.reporting.ledger.reconciliation_projection import project_reconciliation + from adcp.reporting.ledger.store import LedgerConflictError + + try: + reconciliation = project_reconciliation( + obligation, + revisions, + snapshot.adjustments, + value.reconciliation, + consumer_id=scope.consumer_id, + as_of=snapshot.as_of, + ) + except LedgerConflictError: + local.append( + ReportingIssue( + issue_id_for( + "reconciliation-history-v2", + snapshot.account_id, + scope.consumer_id, + obligation.reporting_obligation_id, + ), + "HISTORY_UNAVAILABLE", + "action_required", + "seller", + "contact_seller", + reporting_obligation_id=obligation.reporting_obligation_id, + delivery_config_id=obligation.delivery_config_id, + delivery_config_version=obligation.delivery_config_version, + feed_purpose=obligation.feed_purpose, + ) + ) + projection = replace(projection, health="action_required", satisfied=False) + else: + local.extend(reconciliation.issues) + candidates.update(reconciliation.deadlines) + if not reconciliation.satisfied: + projection = replace(projection, satisfied=False) + if projection.health in {"healthy", "complete"}: + projection = replace(projection, health="waiting") + if any(i.severity == "action_required" for i in local): + projection = replace(projection, health="action_required") + elif any(i.severity == "delayed" for i in local): + projection = replace(projection, health="delayed") projection = replace(projection, issues=_sorted_issues(local)) - projected.append(StatusObligationProjection(obligation, projection, revisions, statuses)) + projected.append( + StatusObligationProjection(obligation, projection, revisions, statuses, reconciliation) + ) issues.extend(local) candidates.update( ( @@ -631,7 +692,11 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ # A pre-obligation missing statement remains a configuration-scoped issue. mismatch_keys = {mismatch_key(s) for s in snapshot.statuses} for status in snapshot.statuses: - if status.superseded or status.consumer_id != scope.consumer_id: + if ( + not value.consumer_status_enabled + or status.superseded + or status.consumer_id != scope.consumer_id + ): continue if status.generation_key not in generations or scope.reporting_obligation_id is not None: continue @@ -717,6 +782,19 @@ def _project(value: StatusProjectionInput) -> tuple[StatusProjectionResult, set[ "period_end": value.period_end.isoformat() if value.period_end else None, }, } + if value.reconciliation is not None: + canonical["version"] = 2 + canonical["reconciliation"] = [ + ( + p.reconciliation.evidence_json.decode("utf-8") + if p.reconciliation is not None + else { + "reporting_obligation_id": p.obligation.reporting_obligation_id, + "unavailable": True, + } + ) + for p in projected + ] fingerprint = hashlib.sha256(canonical_json_utf8_v1(canonical)).hexdigest() return ( StatusProjectionResult( diff --git a/src/adcp/reporting/ledger/status_snapshot.py b/src/adcp/reporting/ledger/status_snapshot.py index 3ee1a9e1e..9bee1670b 100644 --- a/src/adcp/reporting/ledger/status_snapshot.py +++ b/src/adcp/reporting/ledger/status_snapshot.py @@ -6,6 +6,7 @@ from datetime import datetime from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable +from adcp.reporting._timestamp import aware_timestamp from adcp.reporting.ledger.models import ConsumerStatusRecord from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp.reporting.ledger.status_projection import ( @@ -253,11 +254,7 @@ def decode(key: str, columns: str, builder: Callable[[Any], Any]) -> tuple[Any, result = [] for row in raw.get(key, []): values = [ - ( - datetime.fromisoformat(row[n]) - if n in dates and row.get(n) is not None - else row.get(n) - ) + (aware_timestamp(row[n]) if n in dates and row.get(n) is not None else row.get(n)) for n in names ] result.append(builder(values)) @@ -280,7 +277,7 @@ def decode(key: str, columns: str, builder: Callable[[Any], Any]) -> tuple[Any, consumers.update(i.consumer_id for i in lifecycles if i.consumer_id is not None) return ReportingStatusSnapshot( account_id=raw["account_id"], - as_of=datetime.fromisoformat(raw["as_of"]), + as_of=aware_timestamp(raw["as_of"]), configurations=configurations, obligations=decode("obligations", _OBLIGATION_COLUMNS, _obligation_from_row), revisions=decode("revisions", _REVISION_COLUMNS, _revision_from_row), diff --git a/src/adcp/reporting/ledger/store.py b/src/adcp/reporting/ledger/store.py index 6f25b531f..c182c22f9 100644 --- a/src/adcp/reporting/ledger/store.py +++ b/src/adcp/reporting/ledger/store.py @@ -1561,6 +1561,9 @@ def _obligation_for( # -- leasing --------------------------------------------------------- + def _configuration_lease_eligible(self, configuration: ReportingConfiguration) -> bool: + return True + async def lease_period_close( self, *, worker_id: str, now: datetime, lease_seconds: float ) -> LeasedConfiguration | None: @@ -1588,6 +1591,8 @@ async def lease_period_close( ] ] = [] for key in self._configurations: + if not self._configuration_lease_eligible(self._configurations[key]): + continue turn = self._lease_turns.get(key, 0) held = self._leases.get(key) tail = (key.account_id, key.delivery_config_id, key.delivery_config_version) diff --git a/src/adcp/reporting/materializer/memory.py b/src/adcp/reporting/materializer/memory.py index dd922be0a..7dd91f36e 100644 --- a/src/adcp/reporting/materializer/memory.py +++ b/src/adcp/reporting/materializer/memory.py @@ -24,6 +24,7 @@ ) from adcp.reporting.ledger.models import ReportingConfiguration from adcp.reporting.ledger.notification_events import delivery_dirty, materialization_event +from adcp.reporting.ledger.notification_models import ReportingDomainEvent from adcp.reporting.ledger.store import LedgerConflictError from adcp.reporting.materializer._errors import ( ReportingMaterializerUsageError, @@ -89,6 +90,8 @@ class _Work: class InMemoryReportingMaterializerStore(InMemoryReportingReconciliationStore): + _materializer_writer_epoch = 0 + def __init__(self, **kwargs: Any) -> None: super().__init__(**kwargs) self._materializer_candidates: dict[ReportingDeliveryScope, _Candidate] = {} @@ -243,6 +246,9 @@ def _park( candidate.reason, candidate.due_at = reason, due return ReportingMaterializerTurn("parked", reason) + def _materializer_candidate_enabled(self, account_id: str) -> bool: + return True + @materializer_errors async def claim_materialization( self, @@ -265,7 +271,10 @@ async def claim_materialization( candidates = [ c for c in self._materializer_candidates.values() - if c.due_at is not None and c.due_at <= now and c.scope not in pending + if c.due_at is not None + and c.due_at <= now + and c.scope not in pending + and self._materializer_candidate_enabled(c.scope.principal.account_id) ] accounts = {w.scope.principal.account_id for w in works} | { c.scope.principal.account_id for c in candidates @@ -313,6 +322,10 @@ async def claim_materialization( return self._park( candidate, "legacy_terminal" if owned is None else "operator_required" ) + try: + admission_epoch = self._new_admission_epoch(context, key) + except ReportingWriterError: + return self._park(candidate, "component_unavailable") if context.delivery is None: if context.obligation.currency is None: return self._park(candidate, "operator_required") @@ -340,6 +353,7 @@ async def claim_materialization( request, now, self._notification_state is not None, + admission_epoch=admission_epoch, ) self._materializer_work[self._work_key(attempt)] = work self._park(candidate, "ready") @@ -351,6 +365,8 @@ def _lease( keys: tuple[ReportingVerificationKey, ...], seconds: int, ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + if work.admission_epoch > self._materializer_writer_epoch: + raise failure("UNSUPPORTED_VERIFICATION") if work.notifications_enabled != (self._notification_state is not None): return self._park_work(work, "component_unavailable") try: @@ -382,6 +398,7 @@ def _lease( work.request, context, work.notifications_enabled, + work.admission_epoch, ) def _park_work(self, work: _Work, reason: MaterializerReason) -> ReportingMaterializerTurn: @@ -404,6 +421,8 @@ def _held(self, lease: ReportingMaterializerLease) -> _Work | None: or work.generation != lease.generation or work.notifications_enabled != lease.notifications_enabled or work.notifications_enabled != (self._notification_state is not None) + or work.admission_epoch != lease.admission_epoch + or work.admission_epoch > self._materializer_writer_epoch ): raise failure("BINDING_MISMATCH") return work @@ -546,15 +565,7 @@ async def finish_materialization( ) if event is None: raise failure("BINDING_MISMATCH") - assert self._materializer_outbox is not None - self._materializer_outbox.enqueue(event) - if ( - self._materializer_outbox.events.get( - (event.account_id, event.consumer_namespace, event.notification_id) - ) - != event - ): - raise failure("BINDING_MISMATCH") + self._enqueue_materializer(event, lease) if self._held(lease) is None: raise failure("LEASE_LOST") work.completion_token = work.token @@ -591,6 +602,26 @@ async def finish_materialization( stored.reporting_materialization_id, ) + def _new_admission_epoch( + self, context: MaterializerContext, key: ReportingVerificationKey + ) -> int: + return 0 + + def _enqueue_materializer( + self, event: ReportingDomainEvent, lease: ReportingMaterializerLease + ) -> None: + if lease.admission_epoch != 0: + raise failure("UNSUPPORTED_VERIFICATION") + assert self._materializer_outbox is not None + self._materializer_outbox.enqueue(event) + if ( + self._materializer_outbox.events.get( + (event.account_id, event.consumer_namespace, event.notification_id) + ) + != event + ): + raise failure("BINDING_MISMATCH") + def _materializer_dirty( self, outcome: ReportingMaterializationRecord, context: MaterializerContext ) -> None: @@ -601,11 +632,12 @@ def _materializer_dirty( core = settle_memory_snapshot(self, scope.account_id) core = replace(core, as_of=outcome.completed_at) - sequence = self._materializer_status_heads.get(outcome.scope.principal, 0) + 1 - self._materializer_status_heads[outcome.scope.principal] = sequence + heads, boundaries = self._materializer_capture_collections(outcome) + sequence = heads.get(outcome.scope.principal, 0) + 1 + heads[outcome.scope.principal] = sequence account_sequence = self._materializer_account_heads.get(scope.account_id, 0) + 1 self._materializer_account_heads[scope.account_id] = account_sequence - self._materializer_boundaries.append( + boundaries.append( ReportingMaterializerBoundary( outcome.scope.principal, sequence, @@ -617,6 +649,11 @@ def _materializer_dirty( ) ) + def _materializer_capture_collections( + self, outcome: ReportingMaterializationRecord + ) -> tuple[dict[ReportingDeliveryPrincipal, int], list[ReportingMaterializerBoundary]]: + return self._materializer_status_heads, self._materializer_boundaries + @materializer_errors async def read_materializer_boundaries( self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 diff --git a/src/adcp/reporting/materializer/pg.py b/src/adcp/reporting/materializer/pg.py index 9ee925d6f..642d5a0a2 100644 --- a/src/adcp/reporting/materializer/pg.py +++ b/src/adcp/reporting/materializer/pg.py @@ -523,6 +523,7 @@ async def _lease_on( request, context, work["notifications_enabled"], + work["admission_epoch"], ) async def _park_work_on( @@ -567,6 +568,7 @@ async def _held_on( or work["binding_sha256"] != lease.request.binding_fingerprint or work["notifications_enabled"] != lease.notifications_enabled or work["notifications_enabled"] != self._notifications_enabled + or work["admission_epoch"] != lease.admission_epoch ): raise failure("BINDING_MISMATCH") return work diff --git a/src/adcp/reporting/materializer/publication.py b/src/adcp/reporting/materializer/publication.py new file mode 100644 index 000000000..70e3b0070 --- /dev/null +++ b/src/adcp/reporting/materializer/publication.py @@ -0,0 +1,63 @@ +"""SDK-owned canonical evidence before an immutable source revision is committed.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Sequence +from dataclasses import replace +from decimal import Decimal +from typing import Any + +from adcp.reporting.evidence import ReportingCanonicalDigest +from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.materializer.contracts import failure +from adcp.reporting.materializer.verification import ReportingRevisionVerifier, _same_definition + + +def verified_publication( + verifier: ReportingRevisionVerifier, + obligation: ReportingObligationRecord, + revision: ReportingRevisionRecord, + rows: Sequence[dict[str, Any]], +) -> ReportingRevisionRecord: + """Validate the source rows and totals; never reinterpret an existing revision. + + Core publications retain their original representation when no verifier is + installed. Production publications use the exact same installed contract as + destination verification, before the first immutable commit. + """ + from adcp.reporting.ledger.producer import revision_content_sha256 + + key = verifier.key + if ( + not _same_definition(key, obligation.definition) + or (key.report_definition_id, key.reporting_profile) + != (obligation.report_definition_id, obligation.reporting_profile) + or revision.row_count != len(rows) + ): + raise failure("SOURCE_INVALID") + encoded, totals = verifier.canonicalize(rows) + expected = {t.name: Decimal(t.value) for t in totals} + actual = {name: Decimal(value) for name, value in revision.control_totals} + if len(actual) != len(revision.control_totals) or expected != actual: + raise failure("SOURCE_INVALID") + contract = key.canonicalization + pairs = tuple((t.name, t.value) for t in totals) + return replace( + revision, + control_totals=pairs, + managed_control_totals=totals, + canonical_content_digest=ReportingCanonicalDigest( + hashlib.sha256(b"[" + b",".join(encoded) + b"]").hexdigest(), + contract.canonicalization_id, + contract.canonicalization_uri, + contract.canonicalization_sha256, + ), + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision.reporting_revision_id, + row_count=revision.row_count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + ) diff --git a/src/adcp/reporting/materializer/work.py b/src/adcp/reporting/materializer/work.py index e2f306f15..9bed997e6 100644 --- a/src/adcp/reporting/materializer/work.py +++ b/src/adcp/reporting/materializer/work.py @@ -201,6 +201,7 @@ class ReportingMaterializerLease: request: ReportingDestinationRequest context: MaterializerContext notifications_enabled: bool + admission_epoch: int = 0 def __post_init__(self) -> None: invalid = False @@ -209,6 +210,8 @@ def __post_init__(self) -> None: type(self.generation) is not int or self.generation < 1 or type(self.notifications_enabled) is not bool + or type(self.admission_epoch) is not int + or self.admission_epoch not in {0, 2} or UUID(self.token).version != 4 or self.attempt.scope != self.scope or self.context.scope != self.scope diff --git a/src/adcp/reporting/outbox/_activity_pg.py b/src/adcp/reporting/outbox/_activity_pg.py index 80103ebb6..8549b2858 100644 --- a/src/adcp/reporting/outbox/_activity_pg.py +++ b/src/adcp/reporting/outbox/_activity_pg.py @@ -104,6 +104,12 @@ async def _activity_fence(self, conn: Any, lease: DeliveryLease) -> datetime | N async def reserve_attempt( self, lease: DeliveryLease, *, request: ActivityRequest, now: datetime + ) -> WebhookAttempt | None: + async with self._activity_transaction() as conn: + return await self._reserve_attempt_on(conn, lease, request=request) + + async def _reserve_attempt_on( + self, conn: Any, lease: DeliveryLease, *, request: ActivityRequest ) -> WebhookAttempt | None: from adcp.reporting.outbox.pg import database_now @@ -111,47 +117,46 @@ async def reserve_attempt( consumer = canonical_consumer(b.principal_id) request = ActivityRequest(request.url, request.payload_size_bytes) key = (b.account_id, consumer, b.subscriber_id, b.idempotency_key) - async with self._activity_transaction() as conn: - if await self._activity_fence(conn, lease) is None: - return None - duplicate = await ( - await conn.execute( - "SELECT 1 FROM reporting_webhook_attempts WHERE account_id = %s" - " AND principal_id = %s AND consumer_namespace = %s" - " AND delivery_id = %s AND lease_token = %s", - (b.account_id, consumer, b.consumer_namespace, b.delivery_id, lease.token), - ) - ).fetchone() - if duplicate is not None: - return None - number = await self._next_attempt_on(conn, b) - at = await database_now(conn, self._clock) - # The row stays locked from the fence through this commit. A later - # reclaim can never mutate this reservation, including after purge. - if at >= lease.expires_at: - # Roll back the increment as well; no reservation means no HTTP. - raise ReportingNotificationError("activity_lease_expired") - reservation = token_hex(32) + if await self._activity_fence(conn, lease) is None: + return None + duplicate = await ( await conn.execute( - "INSERT INTO reporting_webhook_attempts (account_id, principal_id, subscriber_id," - " idempotency_key, notification_id, attempt, delivery_id, consumer_namespace," - " lease_token, reservation_token, binding, fired_at, url, payload_size_bytes)" - " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s,%s,%s)", - ( - *key, - b.notification_id, - number, - b.delivery_id, - b.consumer_namespace, - lease.token, - reservation, - json.dumps(asdict(b)), - at, - request.url, - request.payload_size_bytes, - ), + "SELECT 1 FROM reporting_webhook_attempts WHERE account_id = %s" + " AND principal_id = %s AND consumer_namespace = %s" + " AND delivery_id = %s AND lease_token = %s", + (b.account_id, consumer, b.consumer_namespace, b.delivery_id, lease.token), ) - return WebhookAttempt(b, number, lease.token, reservation, at, request) + ).fetchone() + if duplicate is not None: + return None + number = await self._next_attempt_on(conn, b) + at = await database_now(conn, self._clock) + # The row stays locked from the fence through this commit. A later + # reclaim can never mutate this reservation, including after purge. + if at >= lease.expires_at: + # Roll back the increment as well; no reservation means no HTTP. + raise ReportingNotificationError("activity_lease_expired") + reservation = token_hex(32) + await conn.execute( + "INSERT INTO reporting_webhook_attempts (account_id, principal_id, subscriber_id," + " idempotency_key, notification_id, attempt, delivery_id, consumer_namespace," + " lease_token, reservation_token, binding, fired_at, url, payload_size_bytes)" + " VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s,%s,%s)", + ( + *key, + b.notification_id, + number, + b.delivery_id, + b.consumer_namespace, + lease.token, + reservation, + json.dumps(asdict(b)), + at, + request.url, + request.payload_size_bytes, + ), + ) + return WebhookAttempt(b, number, lease.token, reservation, at, request) async def _next_attempt_on(self, conn: Any, binding: DeliveryBinding) -> int: b = binding diff --git a/src/adcp/reporting/outbox/memory.py b/src/adcp/reporting/outbox/memory.py index 4976696b0..ea5794453 100644 --- a/src/adcp/reporting/outbox/memory.py +++ b/src/adcp/reporting/outbox/memory.py @@ -335,6 +335,12 @@ async def finish_delivery( async def reserve_attempt( self, lease: DeliveryLease, *, request: ActivityRequest, now: datetime + ) -> WebhookAttempt | None: + async with self._store._lock: + return self._reserve_attempt_locked(lease, request=request, now=now) + + def _reserve_attempt_locked( + self, lease: DeliveryLease, *, request: ActivityRequest, now: datetime ) -> WebhookAttempt | None: binding = lease.delivery.binding consumer = canonical_consumer(binding.principal_id) @@ -346,29 +352,28 @@ async def reserve_attempt( binding.subscriber_id, binding.idempotency_key, ) - async with self._store._lock: - item = self._state.deliveries.get( - (binding.account_id, binding.consumer_namespace, binding.delivery_id) - ) - if ( - item is None - or item[0] != lease.delivery - or item[1].expires_at != lease.expires_at - or not item[1].held(lease.token, now) - ): - return None - if any( - row.lease_token == lease.token and row.binding == binding - for row in self._state.activity.values() - ): - return None - number = self._state.activity_heads.get(key, 0) + 1 - attempt = WebhookAttempt( - binding, number, lease.token, token_hex(32), aware_utc(now), request - ) - self._state.activity_heads[key] = number - self._state.activity[(*key, number)] = attempt - return attempt + item = self._state.deliveries.get( + (binding.account_id, binding.consumer_namespace, binding.delivery_id) + ) + if ( + item is None + or item[0] != lease.delivery + or item[1].expires_at != lease.expires_at + or not item[1].held(lease.token, now) + ): + return None + if any( + row.lease_token == lease.token and row.binding == binding + for row in self._state.activity.values() + ): + return None + number = self._state.activity_heads.get(key, 0) + 1 + attempt = WebhookAttempt( + binding, number, lease.token, token_hex(32), aware_utc(now), request + ) + self._state.activity_heads[key] = number + self._state.activity[(*key, number)] = attempt + return attempt async def complete_attempt( self, attempt: WebhookAttempt, *, outcome: ActivityOutcome, now: datetime diff --git a/src/adcp/reporting/outbox/status_memory.py b/src/adcp/reporting/outbox/status_memory.py index e3a9b89d0..2732a4eef 100644 --- a/src/adcp/reporting/outbox/status_memory.py +++ b/src/adcp/reporting/outbox/status_memory.py @@ -7,8 +7,12 @@ from secrets import token_hex from typing import Any, Literal +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.models import ReportingDeliveryEscalation -from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) from adcp.reporting.ledger.status_projection import ( ReportingStatusSnapshot, StatusProjectionInput, @@ -42,6 +46,11 @@ class _StatusMemoryState: class InMemoryReportingStatusOutbox(InMemoryReportingOutbox): + def __init__(self, store: InMemoryReportingLedgerStore) -> None: + if store._status_notification_state is None: + raise ValueError("construct a status projection participant first") + self._store = store + @property def _state(self) -> NotificationState: state: _StatusMemoryState = self._store._status_notification_state @@ -92,6 +101,7 @@ async def create_schema(self) -> None: pass def _cursor(self, account_id: str) -> int: + self._projection_fence(account_id) account = self._state.accounts.get(account_id) if account is None: raise ReportingNotificationError("status_baseline_required") @@ -99,8 +109,16 @@ def _cursor(self, account_id: str) -> int: raise ReportingNotificationError("status_policy_conflict") return account[0] + def _projection_fence(self, account_id: str) -> None: + if ( + account_id in getattr(self.ledger, "_projection_accounts", {}) + and getattr(self, "_projection_version", 1) != 2 + ): + raise ReportingNotificationError("status_projection_writer_fenced") + async def baseline(self, *, account_id: str) -> bool: async with self.ledger._mutation(): + self._projection_fence(account_id) if account_id in self._state.accounts: if not self._needs_rebuild(account_id): self._cursor(account_id) @@ -144,6 +162,7 @@ def _needs_rebuild(self, account_id: str) -> bool: ) def _rebuild(self, account_id: str) -> StatusTurn: + self._projection_fence(account_id) if not self._needs_rebuild(account_id): return StatusTurn(False) if self._state.selector_accounts.get(account_id) != "transitioning": @@ -200,8 +219,16 @@ async def rebuild_one(self) -> StatusTurn: return self._rebuild(account_id) if account_id is not None else StatusTurn(False) def _apply( - self, snapshot: ReportingStatusSnapshot, *, through: int, baseline: bool = False + self, + snapshot: ReportingStatusSnapshot, + *, + through: int, + baseline: bool = False, + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None, + consumer_status_enabled: bool = True, + enqueue: bool = True, ) -> int: + self._projection_fence(snapshot.account_id) snapshot = settled_replay(snapshot) events = 0 scopes = {s.checkpoint_key: s for s in projection_scopes(snapshot)} @@ -211,7 +238,15 @@ def _apply( if c.scope.account_id == snapshot.account_id ) for _, scope in sorted(scopes.items()): - result = project_status_scope(StatusProjectionInput(snapshot, scope, self.escalation)) + result = project_status_scope( + StatusProjectionInput( + snapshot, + scope, + self.escalation, + reconciliation=reconciliation, + consumer_status_enabled=consumer_status_enabled, + ) + ) checkpoint, event = advance_checkpoint( self._state.checkpoints.get(scope.checkpoint_key), result, @@ -220,11 +255,14 @@ def _apply( baseline=baseline, ) self._state.checkpoints[scope.checkpoint_key] = checkpoint - if event is not None: - self._state.outbox.enqueue(event) + if event is not None and enqueue: + self._enqueue_status(event) events += 1 return events + def _enqueue_status(self, event: ReportingDomainEvent) -> None: + self._state.outbox.enqueue(event) + def _project(self, account_id: str) -> StatusTurn: cursor = self._cursor(account_id) assert self.ledger._notification_state is not None diff --git a/src/adcp/reporting/outbox/status_pg.py b/src/adcp/reporting/outbox/status_pg.py index 4ac861d90..eb59a5cf8 100644 --- a/src/adcp/reporting/outbox/status_pg.py +++ b/src/adcp/reporting/outbox/status_pg.py @@ -17,6 +17,7 @@ from pydantic import TypeAdapter +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord from adcp.reporting.ledger.models import ReportingDeliveryEscalation, ReportingIssueLifecycle from adcp.reporting.ledger.notification_models import ( ReportingDomainEvent, @@ -291,6 +292,9 @@ async def _apply_on( *, through: int, baseline: bool = False, + reconciliation: tuple[ReportingDeliveryRecord, ...] | None = None, + consumer_status_enabled: bool = True, + enqueue: bool = True, ) -> int: await self._lock_scopes_on(connection, snapshot) snapshot = settled_replay(snapshot) @@ -311,7 +315,15 @@ async def _apply_on( scope.checkpoint_key, ) ).fetchone() - result = project_status_scope(StatusProjectionInput(snapshot, scope, self.escalation)) + result = project_status_scope( + StatusProjectionInput( + snapshot, + scope, + self.escalation, + reconciliation=reconciliation, + consumer_status_enabled=consumer_status_enabled, + ) + ) checkpoint, event = advance_checkpoint( _checkpoint(row), result, @@ -320,11 +332,14 @@ async def _apply_on( baseline=baseline, ) await self._write_on(connection, checkpoint) - if event is not None: - await _enqueue_on(connection, event) + if event is not None and enqueue: + await self._enqueue_status_on(connection, event) count += 1 return count + async def _enqueue_status_on(self, connection: Any, event: ReportingDomainEvent) -> None: + await _enqueue_on(connection, event) + async def baseline(self, *, account_id: str) -> bool: from adcp.reporting.outbox.status_schema import validate_status_schema diff --git a/src/adcp/reporting/outbox/worker.py b/src/adcp/reporting/outbox/worker.py index 0370a5cf3..27d95ae16 100644 --- a/src/adcp/reporting/outbox/worker.py +++ b/src/adcp/reporting/outbox/worker.py @@ -7,7 +7,7 @@ from collections.abc import Callable from dataclasses import dataclass from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Protocol import httpx @@ -60,6 +60,25 @@ class _Outcome: class _HttpObservation: reservation: WebhookAttempt | None = None started_ns: int = 0 + retry_window_expired: bool = False + retry_deadline: datetime | None = None + + +class ReportingDeliveryWindow(Protocol): + """Optional additive SDK admission for a durable per-key retry horizon.""" + + async def inspect( + self, lease: DeliveryLease, *, now: datetime + ) -> tuple[datetime | None, bool]: ... + + async def reserve_attempt( + self, + activity: ReportingActivityStore, + lease: DeliveryLease, + *, + request: ActivityRequest, + now: datetime, + ) -> tuple[WebhookAttempt | None, datetime | None, bool]: ... class ReportingNotificationWorker: @@ -85,6 +104,7 @@ def __init__( lease_seconds: float = 60, retry_seconds: float = 5, activity: ReportingActivityStore | None = None, + delivery_window: ReportingDeliveryWindow | None = None, ) -> None: if lease_seconds < 1 or retry_seconds <= 0: raise ValueError("positive retry and at least one second of lease are required") @@ -99,6 +119,7 @@ def __init__( if activity is not None and id(activity) != id(outbox): raise ReportingNotificationError("activity_requires_reporting_outbox") self.activity = activity + self.delivery_window = delivery_window async def advertised_notifications( self, @@ -203,16 +224,23 @@ async def deliver_one(self, *, account_id: str) -> bool: ) if lease is None: return False + observation = _HttpObservation() + if self.delivery_window is not None: + observation.retry_deadline, observation.retry_window_expired = ( + await self.delivery_window.inspect(lease, now=self._clock()) + ) try: opened = self.cipher.open(lease.delivery) except (ReportingNotificationError, ValueError, TypeError): outcome = _Outcome("quarantined", "integrity_failure") else: - observation = _HttpObservation() try: - outcome = await asyncio.wait_for( - self._attempt(lease, opened, observation), timeout=self.lease_seconds * 0.8 - ) + if observation.retry_window_expired: + outcome = _Outcome("suppressed", "lease_expired") + else: + outcome = await asyncio.wait_for( + self._attempt(lease, opened, observation), timeout=self.lease_seconds * 0.8 + ) except (TimeoutError, asyncio.TimeoutError): # Worker cancellation is not an observed HTTP timeout. A # reservation remains pending until a known result is ACKed. @@ -220,6 +248,9 @@ async def deliver_one(self, *, account_id: str) -> bool: return True outcome = _Outcome("pending", "network") now = self._clock() + retry_at = now + timedelta(seconds=self.retry_seconds) + if observation.retry_deadline is not None: + retry_at = min(retry_at, observation.retry_deadline) # A DB failure after HTTP acceptance is intentionally not converted to # success. Expiry/restart retries these exact protected body bytes/key. await self.outbox.finish_delivery( @@ -227,9 +258,7 @@ async def deliver_one(self, *, account_id: str) -> bool: now=now, state=outcome.state, error_code=outcome.error, - retry_at=( - now + timedelta(seconds=self.retry_seconds) if outcome.state == "pending" else None - ), + retry_at=retry_at if outcome.state == "pending" else None, ) return True @@ -291,14 +320,25 @@ async def current_fence() -> bool: callback_used = True if not await self.outbox.delivery_lease_current(lease, now=self._clock()): return False + if self.delivery_window is not None and self.activity is None: + raise ReportingNotificationError("activity_requires_reporting_outbox") if self.activity is not None: # Signing, URL/DNS preparation, and the final fence # precede this transaction. No awaitable preparation # remains between reservation and starting peer I/O. try: - observation.reservation = await self.activity.reserve_attempt( - lease, request=request, now=self._clock() - ) + if self.delivery_window is not None: + ( + observation.reservation, + observation.retry_deadline, + observation.retry_window_expired, + ) = await self.delivery_window.reserve_attempt( + self.activity, lease, request=request, now=self._clock() + ) + else: + observation.reservation = await self.activity.reserve_attempt( + lease, request=request, now=self._clock() + ) except ReportingNotificationError as error: if error.code == "activity_lease_expired": raise PreparedWebhookAttemptExpiredError( @@ -315,7 +355,9 @@ async def current_fence() -> bool: sender, opened.prepared, before_attempt=current_fence ) except PreparedWebhookAttemptExpiredError: - return _Outcome("pending", "lease_expired") + return _Outcome( + "suppressed" if observation.retry_window_expired else "pending", "lease_expired" + ) except ReportingNotificationError: # Failed/unknown reservation commit means no HTTP and no ACK. raise diff --git a/src/adcp/reporting/ownership.py b/src/adcp/reporting/ownership.py new file mode 100644 index 000000000..9b2b61d62 --- /dev/null +++ b/src/adcp/reporting/ownership.py @@ -0,0 +1,129 @@ +"""Additive, page-local revision ownership without changing protocol schemas. + +The extension is evidence, never authorization. A revision response can check +its own binding; only a complete periods walk can establish the named owner. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from copy import deepcopy +from typing import Any + +from adcp.reporting.evidence import reporting_identifier + +_NAME = "reporting_revision_ownership" + + +class ReportingOwnershipError(ValueError): + def __init__(self) -> None: + super().__init__("invalid reporting revision ownership") + + +def page_revision_ownership(page: Mapping[str, Any]) -> dict[str, str] | None: + """Read exactly one binding per returned revision, or the absent legacy mode. + + Duplicate bindings on one page are invalid, including identical duplicates. + The full-walk accumulator may deduplicate identical records on later pages. + Empty opted-in pages must contain an explicit empty bindings array. + """ + if "ext" not in page: + return None + ext = page["ext"] + if type(ext) is not dict: + raise ReportingOwnershipError() + if "adcp" not in ext: + return None + adcp = ext["adcp"] + if type(adcp) is not dict: + raise ReportingOwnershipError() + if _NAME not in adcp: + return None + value = adcp[_NAME] + if ( + type(value) is not dict + or set(value) != {"version", "bindings"} + # A2A's protobuf Struct represents every JSON number as a double. + # JSON Schema's integer 1 includes 1.0, but never booleans or strings. + or type(value["version"]) not in {int, float} + or value["version"] != 1 + or type(value["bindings"]) is not list + ): + raise ReportingOwnershipError() + bindings: dict[str, str] = {} + for binding in value["bindings"]: + if type(binding) is not dict or set(binding) != { + "reporting_revision_id", + "reporting_obligation_id", + }: + raise ReportingOwnershipError() + revision, owner = binding["reporting_revision_id"], binding["reporting_obligation_id"] + try: + reporting_identifier(revision, maximum=255) + reporting_identifier(owner, maximum=255) + except (ValueError, TypeError): + raise ReportingOwnershipError() from None + if revision in bindings: + raise ReportingOwnershipError() + bindings[revision] = owner + revisions = _page_revisions(page) + if type(revisions) is not list or any(type(r) is not dict for r in revisions): + raise ReportingOwnershipError() + ids = [r.get("reporting_revision_id") for r in revisions] + if ( + any(type(r) is not str for r in ids) + or len(set(ids)) != len(ids) + or set(ids) != set(bindings) + ): + raise ReportingOwnershipError() + return bindings + + +def _page_revisions(page: Mapping[str, Any]) -> Any: + names = {"revision", "reporting_revision", "revisions"}.intersection(page) + if len(names) > 1: + raise ReportingOwnershipError() + for name in ("revision", "reporting_revision"): + if name not in page: + continue + revision = page[name] + if type(revision) is not dict: + raise ReportingOwnershipError() + if "reporting_revision_binding" in page: + binding = page["reporting_revision_binding"] + if type(binding) is not dict or binding.get("reporting_revision_id") != revision.get( + "reporting_revision_id" + ): + raise ReportingOwnershipError() + return [revision] + return page.get("revisions", []) + + +def with_revision_ownership(page: Mapping[str, Any], bindings: Mapping[str, str]) -> dict[str, Any]: + """Merge the reserved namespace, rejecting an existing conflicting claim. + + The input is not mutated. Callers supply the already frozen ownership map; + only bindings for this page's revisions enter the response. + """ + result = deepcopy(dict(page)) + previous = page_revision_ownership(result) + revisions: Sequence[dict[str, Any]] = _page_revisions(result) + try: + local = { + r["reporting_revision_id"]: bindings[r["reporting_revision_id"]] for r in revisions + } + except (KeyError, TypeError): + raise ReportingOwnershipError() from None + if previous is not None and previous != local: + raise ReportingOwnershipError() + ext = result.setdefault("ext", {}) + adcp = ext.setdefault("adcp", {}) + adcp[_NAME] = { + "version": 1, + "bindings": [ + {"reporting_revision_id": revision, "reporting_obligation_id": owner} + for revision, owner in local.items() + ], + } + page_revision_ownership(result) + return result diff --git a/src/adcp/reporting/production/__init__.py b/src/adcp/reporting/production/__init__.py new file mode 100644 index 000000000..c68445505 --- /dev/null +++ b/src/adcp/reporting/production/__init__.py @@ -0,0 +1,67 @@ +"""Production reporting composition with explicit, frozen provider contracts. + +Mount one support's handler, start its owned lifecycle, then activate accounts +after migration and drain. The memory implementation is for conformance and +does not claim durability. PostgreSQL uses the shared optional driver guard. +""" + +from typing import TYPE_CHECKING, Any + +from adcp.reporting.production.configuration import ( + ConfigurationAdmission, + ConfigurationTask, + ReportingConfigurationAdmission, + ReportingProductionConfigurationTask, +) +from adcp.reporting.production.contracts import ( + ReportingProductionDestinationBinding, + ReportingProductionMethod, + ReportingProductionSource, + ReportingProductionSourceBinding, +) +from adcp.reporting.production.handler import ReportingProductionHandler +from adcp.reporting.production.memory import ( + InMemoryReportingProductionOutbox, + InMemoryReportingProductionStore, +) +from adcp.reporting.production.notifications import ( + ReportingProductionSigning, + production_notification_workers, +) +from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.production.service import ( + ReportingProductionDestination, + ReportingProductionSupport, +) + +if TYPE_CHECKING: + from adcp.reporting.production.pg import PgReportingProductionOutbox, PgReportingProductionStore + +__all__ = [ + "ConfigurationAdmission", + "ConfigurationTask", + "InMemoryReportingProductionOutbox", + "InMemoryReportingProductionStore", + "PgReportingProductionOutbox", + "PgReportingProductionStore", + "ReportingConfigurationAdmission", + "ReportingProductionConfigurationTask", + "ReportingProductionDestination", + "ReportingProductionDestinationBinding", + "ReportingProductionHandler", + "ReportingProductionMethod", + "ReportingProductionOffering", + "ReportingProductionSigning", + "ReportingProductionSource", + "ReportingProductionSourceBinding", + "ReportingProductionSupport", + "production_notification_workers", +] + + +def __getattr__(name: str) -> Any: + if name in {"PgReportingProductionOutbox", "PgReportingProductionStore"}: + from adcp.reporting.production import pg + + return getattr(pg, name) + raise AttributeError(name) diff --git a/src/adcp/reporting/production/configuration.py b/src/adcp/reporting/production/configuration.py new file mode 100644 index 000000000..fd81ee1af --- /dev/null +++ b/src/adcp/reporting/production/configuration.py @@ -0,0 +1,335 @@ +"""A typed admission boundary for the application's existing account task.""" + +from __future__ import annotations + +import json +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +from adcp.decisioning.capabilities import Account as AccountCapabilities +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, +) +from adcp.reporting.ledger.models import ReportingConfiguration +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.contracts import failure +from adcp.server.base import ToolContext + +if TYPE_CHECKING: + from adcp.reporting.production.service import ReportingProductionSupport + + +@dataclass(frozen=True) +class ReportingConfigurationAdmission: + """Resolved by trusted account/provider code, never decoded from buyer JSON. + + The bound references are opaque; credentials stay in destination sessions. + The SDK validates the complete frozen tuple before admitting configuration. + """ + + offering_id: str + configuration: ReportingConfiguration + binding: ReportingDestinationBinding + configuration_wire: Mapping[str, Any] = field(kw_only=True, repr=False, compare=False) + _wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.validation.schema_loader import get_named_validator + + wire = canonical_json_utf8_v1(dict(self.configuration_wire)) + validator = get_named_validator("core/reporting-delivery-config.json") + if validator is None or next(validator.iter_errors(json.loads(wire)), None) is not None: + raise ValueError("configuration must satisfy the complete public contract") + object.__setattr__(self, "_wire", wire) + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + def check(self, support: ReportingProductionSupport) -> None: + from adcp.reporting.ledger.store import reject_reserved_authoritative_party + + config, binding, raw = self.configuration, self.binding, self.wire() + reject_reserved_authoritative_party(config) + if raw.get("authoritative_party", "seller") != "seller": + raise LedgerConflictError("UNSUPPORTED_FEATURE", "consumer authority is reserved") + offering = support._configuration_offering(config, binding, offering_id=self.offering_id) + schedule = offering.configuration_schedule(config) + supplied_schedule = dict(raw["schedule"]) + if "period_anchor" in supplied_schedule: + supplied_schedule["period_anchor"] = aware_timestamp( + supplied_schedule["period_anchor"] + ).isoformat() + schedule["period_anchor"] = aware_timestamp(schedule["period_anchor"]).isoformat() + scope = raw["scope"] + # This producer freezes an explicit full media-buy denominator. An + # application's dynamic all-buy or partial-coverage implementation must + # not be represented as this installed exact-generation contract. + if ( + raw["delivery_config_id"] != config.delivery_config_id + or raw["delivery_config_version"] != config.delivery_config_version + or raw["offering_id"] != self.offering_id + or raw["feed_purpose"] != config.feed_purpose + or raw["report_definition_id"] != config.report_definition_id + or raw["reporting_profile"] != config.reporting_profile + or raw["required_finality"] != config.required_finality + or raw["reconciliation_mode"] != binding.reconciliation_mode + or set(scope) != {"media_buy_ids"} + or set(scope["media_buy_ids"]) != set(config.media_buy_ids) + or raw["coverage_requirement"] != "full" + or supplied_schedule != schedule + or raw.get("method") != support._destination_binding(binding, offering).wire() + or raw["active"] != (config.activated_at is not None and config.deactivated_at is None) + or ( + "revocation_effective_at" in raw + and aware_timestamp(raw["revocation_effective_at"]) != config.deactivated_at + ) + ): + raise failure("BINDING_MISMATCH") + + +ConfigurationAdmission = Callable[[ReportingConfigurationAdmission], Awaitable[None]] +ConfigurationTask = Callable[ + [dict[str, Any], ToolContext | None, ConfigurationAdmission], Awaitable[dict[str, Any]] +] + + +@dataclass(frozen=True) +class ReportingProductionConfigurationTask: + """Compose the account implementation with enforced SDK reporting admission. + + ``handle`` remains the application's authenticated, caller-owned desired + state account task. It resolves provider grants and opaque bindings, calls + the supplied ``admit`` for every accepted reporting generation, and returns + the normal account response. The wrapper verifies returned ready states + against the actual stored records and that call's validated admissions. + A task cannot return a successful unsupported promise without a matching + admitted configuration, including on exact replay. + """ + + handle: ConfigurationTask = field(repr=False) + account: AccountCapabilities = field(repr=False, compare=False) + _account_wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.validation.schema_loader import get_named_validator + + raw = self.account.model_dump(mode="json", exclude_none=True, exclude_unset=True) + validator = get_named_validator("protocol/get-adcp-capabilities-response.json") + if ( + validator is None + or next( + validator.evolve(schema=validator.schema["properties"]["account"]).iter_errors(raw), + None, + ) + is not None + or raw.get("account_financials") is True + or any( + raw.get(name, {}).get("supported") is True + for name in ("notifications", "change_feed", "identity_updates") + ) + ): + raise ValueError( + "configuration task requires its actual account capabilities and mounted operations" + ) + object.__setattr__(self, "_account_wire", canonical_json_utf8_v1(raw)) + + def account_capabilities(self) -> dict[str, Any]: + return dict(json.loads(self._account_wire)) + + async def execute( + self, + support: ReportingProductionSupport, + request: dict[str, Any], + context: ToolContext | None, + ) -> dict[str, Any]: + from adcp.validation.schema_loader import get_named_validator + + validator = get_named_validator("account/sync-accounts-request.json") + if validator is None or next(validator.iter_errors(request), None) is not None: + raise LedgerConflictError("INVALID_REQUEST", "account request is invalid") + for entry in request["accounts"]: + configurations = entry.get("reporting_delivery_configs", ()) + keys = [(c["delivery_config_id"], c["delivery_config_version"]) for c in configurations] + if len(set(keys)) != len(keys): + raise LedgerConflictError( + "INVALID_REQUEST", "configuration generations must be unique" + ) + admitted: dict[tuple[str, str, int], ReportingConfigurationAdmission] = {} + + async def admit(value: ReportingConfigurationAdmission) -> None: + if request.get("dry_run") is True: + raise LedgerConflictError( + "INVALID_REQUEST", "dry runs cannot admit reporting state" + ) + if type(value) is not ReportingConfigurationAdmission or context is None: + raise LedgerConflictError("UNAUTHORIZED", "configuration authority is unavailable") + who = await support.handler._authorize( + {"account": {"account_id": value.configuration.account_id}}, context + ) + if who != value.binding.principal: + raise LedgerConflictError("UNAUTHORIZED", "configuration authority is unavailable") + matched = False + for entry in request["accounts"]: + if "reporting_delivery_configs" not in entry or "account" not in entry: + continue + try: + requested_caller = await support.handler._authorize( + {"account": entry["account"]}, context + ) + # An unresolved account cannot authorize this admission. + except Exception: # nosec B112 + continue + if requested_caller != who: + continue + desired = entry["reporting_delivery_configs"] + matched = value.wire() in desired or ( + value.configuration.deactivated_at is not None + and not any( + (c["delivery_config_id"], c["delivery_config_version"]) + == ( + value.configuration.delivery_config_id, + value.configuration.delivery_config_version, + ) + for c in desired + ) + ) + if matched: + break + if not matched: + raise LedgerConflictError( + "INVALID_REQUEST", "configuration differs from requested account state" + ) + support.validate_configuration(value) + await support._check_notifications(who.account_id) + key = ( + who.account_id, + value.configuration.delivery_config_id, + value.configuration.delivery_config_version, + ) + if key in admitted and admitted[key] != value: + raise LedgerConflictError( + "CONFIGURATION_GENERATION_IMMUTABLE", "configuration identity conflicts" + ) + await support.store.admit_production_configuration( + value.configuration, value.binding, offering_id=value.offering_id + ) + # The caller already entered the migrated, drained production + # lifecycle. Complete this account's versioned baseline before + # echoing ready; adopters need no account-enumeration worker or + # hand-maintained readiness flag. A failed activation remains + # unready and can resume under the same durable identities. + await support.activate(account_id=who.account_id) + admitted[key] = value + + response = await self.handle(dict(request), context, admit) + if not isinstance(response, Mapping): + raise LedgerConflictError( + "INVALID_REQUEST", "configuration task returned an invalid response" + ) + if response.get("errors") or request.get("dry_run") is True: + return dict(response) + for account in response.get("accounts", ()): + if account.get("action") == "failed": + continue + for state in account.get("reporting_delivery_configs", ()): + if state.get("state") not in {"ready", "inactive"}: + continue + config = state.get("configuration", {}) + key = ( + account.get("account_id"), + config.get("delivery_config_id"), + config.get("delivery_config_version"), + ) + value = admitted.get(key) + if value is None: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration requires SDK admission" + ) + validator = get_named_validator("core/reporting-delivery-config-state.json") + if ( + validator is None + or next(validator.iter_errors(state), None) is not None + or config != value.wire() + ): + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", + "configuration state differs from admission", + ) + expected_state = ( + "inactive" if value.configuration.deactivated_at is not None else "ready" + ) + if state["state"] != expected_state: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration lifecycle differs" + ) + for name in ("activated_at", "deactivated_at"): + actual_time = getattr(value.configuration, name) + supplied = state.get(name) + if supplied is not None and aware_timestamp(supplied) != actual_time: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration lifecycle differs" + ) + coverage = state.get("current_coverage") + if state["state"] == "ready" and ( + not isinstance(coverage, dict) + or coverage["status"] != "full" + or set(coverage["media_buy_ids"]) != set(value.configuration.media_buy_ids) + or set(coverage["fully_covered_media_buy_ids"]) + != set(value.configuration.media_buy_ids) + or any( + coverage[k] + for k in ( + "partially_covered_media_buy_ids", + "unsupported_media_buy_ids", + "unknown_media_buy_ids", + "unsupported_package_ids", + "unknown_package_ids", + "limitations", + ) + ) + or set(coverage["package_ids"]) != set(coverage["covered_package_ids"]) + ): + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration coverage differs" + ) + if state["state"] == "inactive": + from adcp.reporting.ledger.models import derive_period, first_ordinal_after + + configuration = value.configuration + assert configuration.deactivated_at is not None + ordinal = first_ordinal_after( + configuration.schedule, + account_timezone=configuration.account_timezone, + activated_at=configuration.deactivated_at, + ) + cutoff = derive_period( + configuration.schedule, + account_timezone=configuration.account_timezone, + ordinal=ordinal, + ).start + if aware_timestamp(state["publication_stopped_at"]) != cutoff: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration cutoff differs" + ) + who = ReportingDeliveryPrincipal( + value.configuration.account_id, value.binding.consumer_id + ) + actual = await support.store.get_destination_binding( + caller=who, generation_key=value.configuration.generation_key + ) + configs = await support.store.list_configurations(account_id=who.account_id) + support.validate_configuration(value) + if actual != value.binding or value.configuration not in configs: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", + "configuration admission is unavailable", + ) + if state.get("destination_ref") != actual.destination_ref: + raise LedgerConflictError( + "REPORTING_CONFIGURATION_UNADMITTED", "configuration destination differs" + ) + return dict(response) diff --git a/src/adcp/reporting/production/contracts.py b/src/adcp/reporting/production/contracts.py new file mode 100644 index 000000000..3e2e1f6d9 --- /dev/null +++ b/src/adcp/reporting/production/contracts.py @@ -0,0 +1,221 @@ +"""Provider declarations and trusted, immutable source generation bindings.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any, Protocol, runtime_checkable + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding +from adcp.reporting.ledger.models import ReportingConfiguration, ReportingConfigurationGenerationKey +from adcp.reporting.ledger.store import _config_payload +from adcp.reporting.materializer.contracts import ReportingWriterCapability, failure +from adcp.reporting.source import ( + MediaBuyConstituentV1, + ReportingConstituent, + ReportingSourceCapabilitiesV1, + ReportingSourceExecutor, +) + + +@dataclass(frozen=True) +class ReportingProductionMethod: + """The actual provider's complete public method for one writer capability. + + This includes the provider, destination modes, access/producer identity and + reader requirements. A method advertised by an offering must equal this + declaration. The copied bytes cannot change through an adopter's mapping. + Runtime writer, resolver, verifier and authorization checks remain required. + """ + + capability: ReportingWriterCapability + method: Mapping[str, Any] = field(repr=False, compare=False) + _wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.validation.schema_loader import get_named_validator + + raw = json.loads(canonical_json_utf8_v1(dict(self.method))) + validator = get_named_validator("core/reporting-delivery-offering.json") + if ( + validator is None + or next( + validator.evolve(schema=validator.schema["properties"]["method"]).iter_errors(raw), + None, + ) + is not None + or raw.get("orchestration") != "producer_managed" + or (raw.get("pattern"), raw.get("transport"), raw.get("format")) + != (self.capability.method, self.capability.transport, self.capability.format) + ): + raise ValueError("production method must match the provider's exact writer capability") + object.__setattr__(self, "_wire", canonical_json_utf8_v1(raw)) + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + +@dataclass(frozen=True) +class ReportingProductionDestinationBinding: + """The provider's authorized, immutable destination contract for a caller. + + ``configuration`` is the complete secret-free public delivery method, + including the selected destination. The provider resolves it from its + trusted binding, not from a buyer's claim. Credentials remain in the + independently authorized write/readback sessions. The SDK freezes these + bytes at admission and compares them again before materialization. + """ + + binding: ReportingDestinationBinding = field(repr=False) + method: ReportingProductionMethod + configuration: Mapping[str, Any] = field(repr=False, compare=False) + _wire: bytes = field(init=False, repr=False) + + def __post_init__(self) -> None: + from adcp.reporting.evidence import consumer_reference, resource_location + from adcp.validation.schema_loader import get_named_validator + + if ( + type(self.binding) is not ReportingDestinationBinding + or type(self.method) is not ReportingProductionMethod + ): + raise ValueError("destination requires the exact provider binding and method") + raw = json.loads(canonical_json_utf8_v1(dict(self.configuration))) + validator = get_named_validator("core/reporting-delivery-method.json") + offered = self.method.wire() + if ( + validator is None + or next(validator.iter_errors(raw), None) is not None + or any(raw.get(k) != offered.get(k) for k in ("pattern", "transport", "orchestration")) + or (raw["pattern"] == "file_transfer" and raw["format"] != offered.get("format")) + or raw["destination"]["mode"] not in offered["destination_modes"] + ): + raise ValueError("destination must match the provider's complete method") + destination = raw["destination"] + if destination["mode"] == "existing": + if destination["destination_ref"] != self.binding.destination_ref: + raise ValueError("destination must match the immutable binding") + elif destination.get("provider") != offered.get("provider") or destination.get( + "access_mode" + ) != offered.get("access_mode"): + raise ValueError("destination must match the provider's complete method") + if "location" in destination: + resource_location(destination["location"]) + if "recipient" in destination: + consumer_reference(destination["recipient"]["identity"]) + object.__setattr__(self, "_wire", canonical_json_utf8_v1(raw)) + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + +@dataclass(frozen=True) +class ReportingProductionSourceBinding: + """Trusted account-to-source mapping, fixed for a configuration generation. + + ``media_buy_products`` comes from the authenticated source/account mapping, + never from buyer JSON or a report-definition identifier. The SDK persists + it with admission, checks the effective capability digest on every source + turn, and refuses a changed mapping. Reauthorization can withdraw a binding; + it cannot silently change historical scope. No credentials belong here. + """ + + generation_key: ReportingConfigurationGenerationKey + capabilities_sha256: str + media_buy_products: tuple[tuple[str, str], ...] + configuration_sha256: str = field(kw_only=True) + + def __post_init__(self) -> None: + from adcp.reporting.evidence import reporting_identifier, sha256_value + + if type(self.generation_key) is not ReportingConfigurationGenerationKey: + raise ValueError("source binding requires an exact configuration generation") + sha256_value(self.capabilities_sha256) + sha256_value(self.configuration_sha256) + pairs = tuple(tuple(pair) for pair in self.media_buy_products) + if any(len(pair) != 2 for pair in pairs): + raise ValueError("source binding requires media-buy/product pairs") + for media_buy_id, product_id in pairs: + reporting_identifier(media_buy_id, maximum=255) + reporting_identifier(product_id, maximum=255) + if len({pair[0] for pair in pairs}) != len(pairs): + raise ValueError("source binding media buys must be unique") + object.__setattr__(self, "media_buy_products", tuple(sorted(pairs))) + + @classmethod + def for_configuration( + cls, + configuration: ReportingConfiguration, + *, + capabilities_sha256: str, + media_buy_products: tuple[tuple[str, str], ...], + ) -> ReportingProductionSourceBinding: + """Freeze the exact generation semantics and explicitly resolved products. + + Lifecycle changes retain the same semantic generation, matching the + ledger's immutable configuration contract. Captured projection inputs + independently retain each historical activation/deactivation boundary. + """ + return cls( + configuration.generation_key, + capabilities_sha256, + media_buy_products, + configuration_sha256=hashlib.sha256( + canonical_json_utf8_v1(_config_payload(configuration)) + ).hexdigest(), + ) + + def document(self) -> dict[str, Any]: + key = self.generation_key + return { + "account_id": key.account_id, + "delivery_config_id": key.delivery_config_id, + "delivery_config_version": key.delivery_config_version, + "capabilities_sha256": self.capabilities_sha256, + "configuration_sha256": self.configuration_sha256, + "media_buy_products": [list(pair) for pair in self.media_buy_products], + } + + def check( + self, + configuration: ReportingConfiguration, + capabilities: ReportingSourceCapabilitiesV1, + offering_id: str, + ) -> None: + offering = capabilities.offering(offering_id) + if ( + self.generation_key != configuration.generation_key + or self.configuration_sha256 + != hashlib.sha256(canonical_json_utf8_v1(_config_payload(configuration))).hexdigest() + or capabilities.scope != "effective_account" + or self.capabilities_sha256 != capabilities.capabilities_sha256 + or {pair[0] for pair in self.media_buy_products} != set(configuration.media_buy_ids) + or (self.media_buy_products and "media_buy" not in offering.constituent_kinds) + or any(pair[1] not in offering.product_ids for pair in self.media_buy_products) + ): + raise failure("BINDING_MISMATCH") + + def constituents(self) -> tuple[ReportingConstituent, ...]: + return tuple( + MediaBuyConstituentV1( + constituent_id=media_buy_id, media_buy_id=media_buy_id, product_id=product_id + ) + for media_buy_id, product_id in self.media_buy_products + ) + + +@runtime_checkable +class ReportingProductionSource(ReportingSourceExecutor, Protocol): + """A source with an authenticated generation mapping available before I/O. + + Discovery may precede any account binding. Admission and every source turn + require the applicable binding, obtained from the source's trusted account + configuration. Returning ``None`` withdraws authorization for new work. + """ + + def configuration_binding( + self, configuration: ReportingConfiguration + ) -> ReportingProductionSourceBinding | None: ... diff --git a/src/adcp/reporting/production/delivery_window.py b/src/adcp/reporting/production/delivery_window.py new file mode 100644 index 000000000..244e0cd1e --- /dev/null +++ b/src/adcp/reporting/production/delivery_window.py @@ -0,0 +1,160 @@ +"""Immutable first-attempt deadlines for the three production delivery queues.""" + +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timedelta +from typing import TYPE_CHECKING + +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox.activity import ActivityRequest, ReportingActivityStore, WebhookAttempt +from adcp.reporting.outbox.memory import InMemoryReportingOutbox +from adcp.reporting.outbox.models import DeliveryLease +from adcp.reporting.production.memory import InMemoryReportingProductionStore + +if TYPE_CHECKING: + from adcp.reporting.production.pg import PgReportingProductionStore + +RETRY_HORIZON_SECONDS = 86400 + + +@dataclass(frozen=True) +class ProductionDeliveryWindow: + store: InMemoryReportingProductionStore | PgReportingProductionStore + queue: str + + @staticmethod + def _inspect( + saved: tuple[str, str, datetime, datetime] | None, + expected: tuple[str, str], + moment: datetime, + ) -> tuple[datetime | None, bool]: + if saved is None: + return None, False + if saved[:2] != expected or moment < saved[2]: + raise ReportingNotificationError("notification_retry_unready") + return saved[3], moment >= saved[3] + + async def inspect(self, lease: DeliveryLease, *, now: datetime) -> tuple[datetime | None, bool]: + binding = lease.delivery.binding + key = (binding.account_id, binding.idempotency_key) + expected = (self.queue, binding.body_sha256) + if isinstance(self.store, InMemoryReportingProductionStore): + async with self.store._lock: + return self._inspect( + self.store._production_delivery_windows.get(key), expected, aware_utc(now) + ) + from adcp.reporting.outbox.pg import database_now + + try: + async with self.store._connection() as connection, connection.transaction(): + saved = await ( + await connection.execute( + "SELECT queue,body_sha256,started_at,expires_at" + " FROM reporting_production_delivery_windows" + " WHERE account_id=%s AND idempotency_key=%s", + key, + ) + ).fetchone() + return self._inspect( + saved, expected, await database_now(connection, self.store._clock) + ) + except ReportingNotificationError: + raise + except Exception: + raise ReportingNotificationError("notification_retry_unready") from None + + async def reserve_attempt( + self, + activity: ReportingActivityStore, + lease: DeliveryLease, + *, + request: ActivityRequest, + now: datetime, + ) -> tuple[WebhookAttempt | None, datetime | None, bool]: + """Commit the immutable deadline with the original SDK HTTP reservation. + + False admission never produces an activity ordinal. A transaction + failing after either insertion rolls back the window, ordinal head and + attempt together. Unknown commit outcomes leave the original identity. + """ + binding = lease.delivery.binding + key = (binding.account_id, binding.idempotency_key) + expected = (self.queue, binding.body_sha256) + if self.queue not in {"core", "status", "ready"}: + raise ReportingNotificationError("notification_retry_unready") + if isinstance(self.store, InMemoryReportingProductionStore): + if ( + not isinstance(activity, InMemoryReportingOutbox) + or activity._store is not self.store + ): + raise ReportingNotificationError("notification_retry_unready") + async with self.store._mutation(): + at = aware_utc(now) + saved = self.store._production_delivery_windows.get(key) + deadline, expired = self._inspect(saved, expected, at) + if expired: + return None, deadline, True + attempt = activity._reserve_attempt_locked(lease, request=request, now=at) + if attempt is None: + return None, deadline, False + retained = self.store._production_delivery_windows.setdefault( + key, + ( + *expected, + attempt.fired_at, + attempt.fired_at + timedelta(seconds=RETRY_HORIZON_SECONDS), + ), + ) + return attempt, retained[3], False + + from adcp.reporting.outbox.pg import PgReportingOutbox, database_now + + if not isinstance(activity, PgReportingOutbox) or activity._pool is not self.store._pool: + raise ReportingNotificationError("notification_retry_unready") + try: + # The activity participant supplies its exact queue adapter and + # connection. Take the inherited account lock before its row lock. + async with activity._activity_transaction() as connection: + await self.store._lock_account(connection, binding.account_id) + moment = await database_now(connection, self.store._clock) + saved = await ( + await connection.execute( + "SELECT queue,body_sha256,started_at,expires_at" + " FROM reporting_production_delivery_windows" + " WHERE account_id=%s AND idempotency_key=%s", + key, + ) + ).fetchone() + deadline, expired = self._inspect(saved, expected, moment) + if expired: + return None, deadline, True + attempt = await activity._reserve_attempt_on(connection, lease, request=request) + if attempt is None: + return None, deadline, False + deadline, expired = self._inspect(saved, expected, attempt.fired_at) + if expired: + # Time can cross the deadline while reserving an ordinal. + # Roll back that provisional head and attempt, too. + raise ReportingNotificationError("notification_retry_expired") + await connection.execute( + "INSERT INTO reporting_production_delivery_windows" + " (account_id,idempotency_key,queue,body_sha256,started_at,expires_at)" + " VALUES(%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + ( + *key, + *expected, + attempt.fired_at, + attempt.fired_at + timedelta(seconds=RETRY_HORIZON_SECONDS), + ), + ) + return ( + attempt, + deadline or attempt.fired_at + timedelta(seconds=RETRY_HORIZON_SECONDS), + False, + ) + except ReportingNotificationError as error: + if error.code == "notification_retry_expired": + return None, deadline, True + raise diff --git a/src/adcp/reporting/production/handler.py b/src/adcp/reporting/production/handler.py new file mode 100644 index 000000000..75de9eabd --- /dev/null +++ b/src/adcp/reporting/production/handler.py @@ -0,0 +1,316 @@ +"""Authenticated production routes with private, revision-bound exact reads.""" + +from __future__ import annotations + +import hashlib +from typing import TYPE_CHECKING, Any + +from adcp.decisioning.context import RequestContext +from adcp.exceptions import ADCPTaskError +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.consumer_status import ConsumerStatusIngest +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.store import LedgerConflictError, decode_cursor, encode_cursor +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.receipts.handler import ( + ReceiptAccountResolver, + ReportingReceiptHandler, + _consumer, +) +from adcp.server.base import NotImplementedResponse, ToolContext +from adcp.server.responses import capabilities_response +from adcp.types import ( + Error, + GetAdcpCapabilitiesRequest, + GetMediaBuyDeliveryRequest, + GetReportingStatusRequest, + SyncAccountsRequest, + SyncReportingReceiptsRequest, + SyncReportingStatusRequest, +) + +if TYPE_CHECKING: + from adcp.decisioning.registry import BuyerAgentRegistry + from adcp.reporting.production.service import ReportingProductionSupport + + +def _request(value: Any) -> dict[str, Any]: + return ( + dict(value) + if isinstance(value, dict) + else dict(value.model_dump(mode="json", exclude_unset=True)) + ) + + +def _task_error(task: str, code: str, message: str) -> ADCPTaskError: + return ADCPTaskError(operation=task, errors=[Error(code=code, message=message)]) + + +class ReportingProductionHandler(ReportingReceiptHandler): + """Mount the same instance on MCP/A2A; the support owns its lifecycle.""" + + advertised_tools = { + "get_adcp_capabilities", + "get_reporting_status", + "get_media_buy_delivery", + "sync_accounts", + "sync_reporting_receipts", + "sync_reporting_status", + } + + def __init__( + self, + production: ReportingProductionSupport, + *, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + ) -> None: + self.production = production + super().__init__( + production.store, + resolve_account=resolve_account, + buyer_agents=buyer_agents, + consumer_status_enabled=production.projection.consumer_status_enabled, + ) + + def advertised_tools_for_instance(self) -> set[str]: + names = { + "get_adcp_capabilities", + "get_reporting_status", + "get_media_buy_delivery", + "sync_accounts", + } + if any(o.reconciled for o in self.production.offerings): + names.add("sync_reporting_receipts") + if self._feed_consumer_status_enabled: + names.add("sync_reporting_status") + return names + + async def get_reporting_status( + self, + params: GetReportingStatusRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any] | NotImplementedResponse: + return await super().get_reporting_status(params, context) + + async def sync_reporting_receipts( + self, + params: SyncReportingReceiptsRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + if not any(o.reconciled for o in self.production.offerings): + raise _task_error( + "sync_reporting_receipts", "NOT_SUPPORTED", "receipt task is unavailable" + ) + return await super().sync_reporting_receipts(params, context) + + async def _authorize( + self, request: dict[str, Any], context: ToolContext | None + ) -> ReportingDeliveryPrincipal: + try: + if context is None or not isinstance(request.get("account"), dict): + raise ReportingReceiptError("UNAUTHORIZED") + consumer = await _consumer(context, self._receipt_registry) + account = await self._receipt_account_resolver( + dict(request["account"]), context, consumer + ) + if isinstance(context, RequestContext) and context.account.id != account: + raise ReportingReceiptError("UNAUTHORIZED") + return ReportingDeliveryPrincipal(account, consumer) + except Exception: + raise ReportingReceiptError("UNAUTHORIZED") from None + + async def get_adcp_capabilities( + self, + params: GetAdcpCapabilitiesRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + response = capabilities_response( + ["media_buy"], sandbox=False, idempotency={"supported": False} + ) + response["account"] = self.production.configuration_task.account_capabilities() + reporting = await self.production.reporting_delivery() + if reporting: + response["media_buy"] = {"reporting_delivery": reporting} + response["experimental_features"] = ["media_buy.reporting_delivery"] + if any( + reporting.get(k) + for k in ("ledger_notification", "status_notification", "readiness_notification") + ): + from adcp.reporting.production.notifications import ( + signing_capabilities, + signing_identity, + ) + + response["webhook_signing"] = signing_capabilities(self.production) + response["identity"] = signing_identity(self.production) + return response + + async def sync_accounts( + self, + params: SyncAccountsRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any]: + try: + self.production._assert_components() + return await self.production.configuration_task.execute( + self.production, _request(params), context + ) + except LedgerConflictError as error: + code, message = error.code, str(error) + except ReportingReceiptError as error: + code, message = error.code, str(error) + except Exception: + code, message = ( + "REPORTING_CONFIGURATION_UNAVAILABLE", + "reporting configuration is unavailable", + ) + raise _task_error("sync_accounts", code, message) + + async def sync_reporting_status( + self, + params: SyncReportingStatusRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any] | NotImplementedResponse: + if not self._feed_consumer_status_enabled: + return self._not_supported("sync_reporting_status") + try: + request = _request(params) + caller = await self._authorize(request, context) + result = await ConsumerStatusIngest(self.production.store, enabled=True).handle( + request, + account_id=caller.account_id, + consumer_id=caller.consumer_id, + ) + if await self._authorize(request, context) != caller: + raise ReportingReceiptError("UNAUTHORIZED") + return result + except (ReportingReceiptError, LedgerConflictError) as error: + code, message = error.code, str(error) + except Exception: + code, message = "REPORTING_STATUS_UNAVAILABLE", "reporting status is unavailable" + raise _task_error("sync_reporting_status", code, message) + + async def get_media_buy_delivery( + self, + params: GetMediaBuyDeliveryRequest | dict[str, Any], + context: ToolContext | None = None, + ) -> dict[str, Any] | NotImplementedResponse: + request = _request(params) + if "reporting_revision_id" not in request: + return self._not_supported("get_media_buy_delivery") + try: + from adcp.validation.schema_loader import get_named_validator + + validator = get_named_validator("media-buy/get-media-buy-delivery-request.json") + if validator is None or next(validator.iter_errors(request), None) is not None: + raise LedgerConflictError("INVALID_REQUEST", "exact revision request is invalid") + caller = await self._authorize(request, context) + revision_id = request["reporting_revision_id"] + status_request = { + "account": request["account"], + "view": "revision", + "reporting_revision_id": revision_id, + } + # This is the actual private mounted status path, including exact + # ownership/visibility and complete captured reconciliation checks. + projected = await self.get_reporting_status(status_request, context) + if not isinstance(projected, dict) or "revision" not in projected: + raise LedgerConflictError("LOOKUP_UNAVAILABLE", "no such revision is available") + store = self.production.store + revision = await store.get_revision( + account_id=caller.account_id, reporting_revision_id=revision_id + ) + if revision is None or not revision.readable: + raise LedgerConflictError("LOOKUP_UNAVAILABLE", "no such revision is available") + pagination = request.get("pagination") or {} + limit = pagination.get("max_results", 50) + if type(limit) is not int or not 1 <= limit <= 100: + raise LedgerConflictError("INVALID_REQUEST", "exact revision page size is invalid") + binding_hash = hashlib.sha256( + canonical_json_utf8_v1( + { + "account": caller.account_id, + "consumer": caller.consumer_id, + "revision": revision_id, + "digest": revision.revision_content_sha256, + "count": revision.row_count, + "limit": limit, + } + ) + ).hexdigest() + token = pagination.get("cursor") + offset = 0 + if token is not None: + if not isinstance(token, str) or not token.startswith("rpr2.") or len(token) > 2048: + raise LedgerConflictError( + "INVALID_CURSOR", "exact revision cursor is unavailable" + ) + cursor = decode_cursor(token[5:]) + if ( + set(cursor) != {"v", "h", "p"} + or cursor["v"] != 2 + or cursor["h"] != binding_hash + or type(cursor["p"]) is not int + or not 0 <= cursor["p"] <= revision.row_count + ): + raise LedgerConflictError( + "INVALID_CURSOR", "exact revision cursor is unavailable" + ) + offset = cursor["p"] + page = await store.read_revision_rows( + account_id=caller.account_id, + reporting_revision_id=revision_id, + limit=limit, + cursor=( + encode_cursor({"revision": revision_id, "offset": offset}) if offset else None + ), + ) + if page.total_count != revision.row_count or page.reporting_revision_id != revision_id: + raise ReportingNotificationError("reporting_revision_content_unavailable") + if await self._authorize(request, context) != caller: + raise ReportingReceiptError("UNAUTHORIZED") + after = await store.get_revision( + account_id=caller.account_id, reporting_revision_id=revision_id + ) + if after != revision: + raise LedgerConflictError("LOOKUP_UNAVAILABLE", "no such revision is available") + wire = projected["revision"] + totals = ( + [r.to_wire() for r in revision.managed_control_totals] + if revision.managed_control_totals is not None + else [{"name": n, "value": v} for n, v in revision.control_totals] + ) + position: dict[str, Any] = {"total_count": page.total_count, "has_more": page.has_more} + if page.has_more: + position["cursor"] = "rpr2." + encode_cursor( + {"v": 2, "h": binding_hash, "p": offset + len(page.rows)} + ) + result = { + "status": "completed", + "reporting_period": { + "start": wire["period"]["start"], + "end": wire["period"]["end"], + }, + "media_buy_deliveries": [], + "reporting_revision": wire, + "reporting_revision_binding": { + "reporting_revision_id": revision_id, + "row_count": revision.row_count, + "control_totals": totals, + "content_sha256": revision.revision_content_sha256, + }, + "reporting_rows": list(page.rows), + "pagination": position, + } + if "ext" in projected: + result["ext"] = projected["ext"] + return result + except ADCPTaskError: + raise + except (ReportingReceiptError, LedgerConflictError) as error: + code, message = error.code, str(error) + except Exception: + code, message = "REPORTING_CONTENT_UNAVAILABLE", "reporting content is unavailable" + raise _task_error("get_media_buy_delivery", code, message) diff --git a/src/adcp/reporting/production/memory.py b/src/adcp/reporting/production/memory.py new file mode 100644 index 000000000..b804a23bc --- /dev/null +++ b/src/adcp/reporting/production/memory.py @@ -0,0 +1,380 @@ +"""Reference production transactions for shared state-machine conformance.""" + +from __future__ import annotations + +import json +import weakref +from dataclasses import dataclass +from datetime import datetime +from typing import TYPE_CHECKING, Any + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDestinationBinding, + ReportingMaterializationRecord, +) +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + ReportingConfigurationGenerationKey, + ReportingObligationRecord, +) +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.producer_progress import acquisition_state, check_next_period +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import ReportingMaterializerBoundary +from adcp.reporting.materializer.contracts import ReportingVerificationKey, failure +from adcp.reporting.materializer.work import MaterializerContext, ReportingMaterializerLease +from adcp.reporting.outbox.memory import InMemoryReportingOutbox, NotificationState +from adcp.reporting.production.contracts import ReportingProductionSourceBinding +from adcp.reporting.projection.memory import InMemoryReportingProjectionStore +from adcp.reporting.source import ReportingConstituent + +if TYPE_CHECKING: + from adcp.reporting.materializer.memory import _Work + from adcp.reporting.production.service import ReportingProductionSupport + + +@dataclass(frozen=True) +class _Admission: + activated_at: datetime + policy: dict[str, Any] + + +@dataclass(frozen=True) +class _SourceWork: + obligation: ReportingObligationRecord + turn: int = 0 + state: str = "pending" + + +class InMemoryReportingProductionOutbox(InMemoryReportingOutbox): + """The ordinary SDK expansion/delivery state machine, in a separate collection.""" + + _store: InMemoryReportingProductionStore + + def __init__(self, store: InMemoryReportingProductionStore) -> None: + if store._production_outbox is None: + raise ReportingNotificationError("notifications_disabled") + self._store = store + + @property + def _state(self) -> NotificationState: + assert self._store._production_outbox is not None + return self._store._production_outbox + + +class InMemoryReportingProductionStore(InMemoryReportingProjectionStore): + """Matches production atomicity and epochs, without claiming durable storage.""" + + _materializer_writer_epoch = 2 + _production_support: weakref.ReferenceType[ReportingProductionSupport] | None = None + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._production_accounts: dict[str, _Admission] = {} + self._production_delivery_windows: dict[ + tuple[str, str], tuple[str, str, datetime, datetime] + ] = {} + self._production_generations: dict[ReportingConfigurationGenerationKey, str] = {} + self._production_source_bindings: dict[ + ReportingConfigurationGenerationKey, ReportingProductionSourceBinding + ] = {} + self._production_destination_bindings: dict[ + tuple[ReportingConfigurationGenerationKey, str], bytes + ] = {} + self._production_outbox = ( + NotificationState() if self._notification_state is not None else None + ) + self._production_status_heads: dict[ReportingDeliveryPrincipal, int] = {} + self._production_boundaries: list[ReportingMaterializerBoundary] = [] + self._production_closed: dict[ReportingConfigurationGenerationKey, datetime] = {} + self._production_source_turns: dict[ReportingConfigurationGenerationKey, int] = {} + self._production_source_work: dict[str, _SourceWork] = {} + + def _owner(self) -> ReportingProductionSupport: + from adcp.reporting.production.service import production_owner + + return production_owner(self) + + async def admit_production_configuration( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + offering_id: str, + ) -> None: + offering = self._owner()._configuration_offering( + configuration, binding, offering_id=offering_id + ) + async with self._mutation(): + await self.put_configuration(configuration) + await self.put_destination_binding(binding) + self._enroll(configuration, offering._producer_key, binding) + + def _enroll( + self, + configuration: ReportingConfiguration, + producer_key: str, + destination: ReportingDestinationBinding, + ) -> None: + binding = self._owner()._source_binding(configuration, producer_key) + previous = self._production_generations.setdefault( + configuration.generation_key, producer_key + ) + previous_binding = self._production_source_bindings.setdefault( + configuration.generation_key, binding + ) + if previous != producer_key or previous_binding != binding: + raise ReportingNotificationError("reporting_production_source_conflict") + owner = self._owner() + offering = owner._configuration_offering( + configuration, destination, producer_key=producer_key + ) + document = canonical_json_utf8_v1(owner._destination_binding(destination, offering).wire()) + original = self._production_destination_bindings.setdefault( + (configuration.generation_key, destination.consumer_id), document + ) + if original != document: + raise ReportingNotificationError("reporting_production_destination_conflict") + + def _destination_document(self, binding: ReportingDestinationBinding) -> dict[str, Any] | None: + raw = self._production_destination_bindings.get( + (binding.generation_key, binding.consumer_id) + ) + return dict(json.loads(raw)) if raw is not None else None + + def _configuration_lease_eligible(self, configuration: ReportingConfiguration) -> bool: + if configuration.account_id not in self._production_accounts: + return False + producer_key = self._production_generations.get(configuration.generation_key) + binding = self._production_source_bindings.get(configuration.generation_key) + if producer_key not in self._owner()._producer_keys() or binding is None: + return False + try: + self._owner()._check_source_binding(configuration, producer_key, binding.document()) + except Exception: + return False + return True + + def _check_source_generation(self, configuration: ReportingConfiguration) -> None: + if ( + not self._configuration_lease_eligible(configuration) + or self._configurations.get(configuration.generation_key) != configuration + ): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation is unavailable") + + def _wake_obligation(self, account_id: str, obligation_id: str) -> None: + super()._wake_obligation(account_id, obligation_id) + obligation = self._obligations.get(obligation_id) + if ( + obligation is not None + and obligation.account_id == account_id + and obligation.generation_key in self._production_generations + ): + previous = self._production_source_work.get(obligation_id) + self._production_source_work[obligation_id] = _SourceWork( + obligation, previous.turn if previous else 0 + ) + + async def producer_closed_through( + self, configuration: ReportingConfiguration + ) -> datetime | None: + async with self._lock: + self._check_source_generation(configuration) + return self._production_closed.get(configuration.generation_key) + + async def producer_constituents( + self, configuration: ReportingConfiguration, obligation: ReportingObligationRecord + ) -> tuple[ReportingConstituent, ...]: + async with self._lock: + self._check_source_generation(configuration) + if obligation.generation_key != configuration.generation_key or set( + obligation.media_buy_ids + ) != set(configuration.media_buy_ids): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "source denominator differs") + return self._production_source_bindings[configuration.generation_key].constituents() + + async def commit_producer_period( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + previous_end: datetime | None, + ) -> ReportingObligationRecord: + check_next_period(configuration, obligation, previous_end) + async with self._mutation(): + self._check_source_generation(configuration) + current = self._production_closed.get(configuration.generation_key) + if current != previous_end and (current is None or current < obligation.period.end): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer progress changed") + stored = await self.commit_obligation(obligation) + self._production_source_work.setdefault( + stored.reporting_obligation_id, _SourceWork(stored) + ) + self._production_closed[configuration.generation_key] = max( + obligation.period.end, current or obligation.period.end + ) + return stored + + async def next_producer_obligations( + self, configuration: ReportingConfiguration, *, now: datetime, limit: int + ) -> tuple[str, ...]: + if type(limit) is not int or not 1 <= limit <= 64: + raise ValueError("production acquisition limit must be in 1..64") + async with self._lock: + self._check_source_generation(configuration) + candidates = sorted( + (work.turn, work.obligation.reporting_obligation_id) + for work in self._production_source_work.values() + if work.obligation.generation_key == configuration.generation_key + and work.obligation.period.end <= now + and work.state == "pending" + )[:limit] + turn = self._production_source_turns.get(configuration.generation_key, 0) + for offset, (_, identifier) in enumerate(candidates, 1): + work = self._production_source_work[identifier] + self._production_source_work[identifier] = _SourceWork( + work.obligation, turn + offset + ) + self._production_source_turns[configuration.generation_key] = turn + len(candidates) + return tuple(identifier for _, identifier in candidates) + + async def finish_producer_acquisition( + self, configuration: ReportingConfiguration, *, reporting_obligation_id: str + ) -> None: + async with self._lock: + self._check_source_generation(configuration) + work = self._production_source_work[reporting_obligation_id] + if work.obligation.generation_key != configuration.generation_key: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation differs") + revisions = await self.list_revisions( + account_id=configuration.account_id, + reporting_obligation_id=reporting_obligation_id, + ) + self._production_source_work[reporting_obligation_id] = _SourceWork( + work.obligation, work.turn, acquisition_state(work.obligation, revisions) + ) + + async def _activate_production(self, *, account_id: str) -> bool: + owner = self._owner() + async with self._mutation(): + projection = self._projection_accounts.get(account_id) + if ( + projection is None + or not projection.ready + or projection.policy != owner.projection.policy + ): + raise ReportingNotificationError("status_projection_activation_required") + policy = owner._admission_policy() + current = self._production_accounts.get(account_id) + if current is not None: + if current.policy != policy: + raise ReportingNotificationError("reporting_production_policy_conflict") + return False + self._production_accounts[account_id] = _Admission(self._clock(), policy) + for _, _, record in self._retained_delivery_records(): + if ( + isinstance(record, ReportingDestinationBinding) + and record.principal.account_id == account_id + ): + configuration = self._configurations[record.generation_key] + try: + offering = owner._configuration_offering(configuration, record) + # Unsupported or unavailable bindings remain unadmitted. + except Exception: # nosec B112 + continue + self._enroll(configuration, offering._producer_key, record) + return True + + def _check_admission(self, lease: ReportingMaterializerLease) -> None: + if lease.admission_epoch != 2: + return + admission = self._production_accounts.get(lease.scope.principal.account_id) + if admission is None or lease.attempt.created_at < admission.activated_at: + raise failure("BINDING_MISMATCH") + self._owner()._check_context( + self._context(lease.scope), + lease.request.verification_key, + admission.policy, + self._production_generations.get(lease.scope.generation_key), + ( + self._production_source_bindings[lease.scope.generation_key].document() + if lease.scope.generation_key in self._production_source_bindings + else None + ), + self._destination_document(self._context(lease.scope).binding), + ) + + def _new_admission_epoch( + self, context: MaterializerContext, key: ReportingVerificationKey + ) -> int: + admission = self._production_accounts.get(context.scope.principal.account_id) + if admission is None: + raise ReportingNotificationError("reporting_production_activation_required") + self._owner()._check_context( + context, + key, + admission.policy, + self._production_generations.get(context.configuration.generation_key), + ( + self._production_source_bindings[context.configuration.generation_key].document() + if context.configuration.generation_key in self._production_source_bindings + else None + ), + self._destination_document(context.binding), + ) + return 2 + + def _materializer_candidate_enabled(self, account_id: str) -> bool: + return account_id in self._production_accounts + + def _held(self, lease: ReportingMaterializerLease) -> _Work | None: + work = super()._held(lease) + if work is not None: + self._check_admission(lease) + return work + + def _materializer_capture_collections( + self, outcome: ReportingMaterializationRecord + ) -> tuple[dict[ReportingDeliveryPrincipal, int], list[ReportingMaterializerBoundary]]: + work = self._materializer_work.get( + ( + outcome.scope.principal.account_id, + outcome.scope.consumer_id, + outcome.reporting_materialization_id, + ) + ) + if work is not None and work.admission_epoch == 2: + return self._production_status_heads, self._production_boundaries + return super()._materializer_capture_collections(outcome) + + def _enqueue_materializer( + self, event: ReportingDomainEvent, lease: ReportingMaterializerLease + ) -> None: + if lease.admission_epoch == 0: + return super()._enqueue_materializer(event, lease) + self._check_admission(lease) + if self._production_outbox is None: + raise failure("BINDING_MISMATCH") + self._production_outbox.enqueue(event) + if ( + self._production_outbox.events.get( + (event.account_id, event.consumer_namespace, event.notification_id) + ) + != event + ): + raise failure("BINDING_MISMATCH") + + async def read_production_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + if type(after) is not int or after < 0 or type(limit) is not int or not 1 <= limit <= 100: + raise ValueError("production boundary reads require bounded positions") + async with self._lock: + return tuple( + b for b in self._production_boundaries if b.caller == caller and b.sequence > after + )[:limit] diff --git a/src/adcp/reporting/production/notifications.py b/src/adcp/reporting/production/notifications.py new file mode 100644 index 000000000..1b2224cb3 --- /dev/null +++ b/src/adcp/reporting/production/notifications.py @@ -0,0 +1,364 @@ +"""Optional owned notification delivery, independent of complete polling.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any + +import httpx + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox.memory import InMemoryReportingOutbox +from adcp.reporting.outbox.routing import ( + ReportingEnvelopeCipher, + ReportingNotificationSubscription, + ReportingSigningMaterial, + ReportingSigningResolver, + ReportingSubscriptionResolver, +) +from adcp.reporting.outbox.worker import ReportingNotificationWorker +from adcp.reporting.production.delivery_window import ( + RETRY_HORIZON_SECONDS, + ProductionDeliveryWindow, +) +from adcp.reporting.production.memory import ( + InMemoryReportingProductionOutbox, + InMemoryReportingProductionStore, +) +from adcp.reporting.projection.memory import InMemoryReportingStatusProjection +from adcp.signing.crypto import ALLOWED_ALGS + +if TYPE_CHECKING: + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.production.service import ReportingProductionSupport + from adcp.reporting.projection.pg import PgReportingStatusProjection + + +@dataclass(frozen=True) +class ReportingProductionSigning: + """A declared RFC 9421 algorithm contract checked on every resolved key. + + Key material remains in the trusted resolver. The public declaration and + actual sender share this immutable contract; rotation cannot switch to an + unadvertised algorithm or a legacy authentication path. + """ + + resolver: ReportingSigningResolver = field(repr=False) + algorithms: tuple[str, ...] + brand_json_url: str = field(kw_only=True) + + def __post_init__(self) -> None: + object.__setattr__(self, "algorithms", tuple(self.algorithms)) + if ( + not self.algorithms + or len(set(self.algorithms)) != len(self.algorithms) + or not set(self.algorithms) <= ALLOWED_ALGS + or not callable(getattr(self.resolver, "resolve", None)) + ): + raise ReportingNotificationError("notification_signing_unready") + try: + if type(self.brand_json_url) is not str: + raise ValueError + identity = httpx.URL(self.brand_json_url) + valid = ( + identity.scheme == "https" + and bool(identity.host) + and not identity.userinfo + and not identity.query + and not identity.fragment + and identity.port in (None, 443) + ) + except (TypeError, ValueError, httpx.InvalidURL): + valid = False + if not valid: + raise ReportingNotificationError("notification_signing_unready") + + async def resolve( + self, *, account_id: str, principal_id: str, signing_scope_id: str + ) -> ReportingSigningMaterial: + material = await self.resolver.resolve( + account_id=account_id, principal_id=principal_id, signing_scope_id=signing_scope_id + ) + if type( + material + ) is not ReportingSigningMaterial or material.advertised_algorithms != frozenset( + self.algorithms + ): + raise ReportingNotificationError("notification_signing_unready") + ReportingSigningMaterial.__post_init__(material) + return material + + def wire(self) -> dict[str, Any]: + return { + "supported": True, + "profile": "adcp/webhook-signing/v1", + "algorithms": list(self.algorithms), + "legacy_hmac_fallback": False, + "delivery_retry_horizon_seconds": RETRY_HORIZON_SECONDS, + } + + +@dataclass(frozen=True) +class _SignedSubscriptions: + resolver: ReportingSubscriptionResolver = field(repr=False) + + @staticmethod + def _check(value: ReportingNotificationSubscription) -> ReportingNotificationSubscription: + if type(value) is not ReportingNotificationSubscription or value.authentication is not None: + raise ReportingNotificationError("notification_signing_unready") + return value + + async def list_active( + self, *, account_id: str, notification_type: str + ) -> tuple[ReportingNotificationSubscription, ...]: + values = await self.resolver.list_active( + account_id=account_id, notification_type=notification_type + ) + return tuple(self._check(v) for v in values) + + async def get_active( + self, *, account_id: str, subscriber_id: str, notification_type: str + ) -> ReportingNotificationSubscription | None: + value = await self.resolver.get_active( + account_id=account_id, subscriber_id=subscriber_id, notification_type=notification_type + ) + return self._check(value) if value is not None else None + + +def production_notification_workers( + store: InMemoryReportingProductionStore | PgReportingProductionStore, + projection: InMemoryReportingStatusProjection | PgReportingStatusProjection, + *, + subscriptions: ReportingSubscriptionResolver, + cipher: ReportingEnvelopeCipher, + signing: ReportingProductionSigning, +) -> tuple[ReportingNotificationWorker, ...]: + """Build all three real SDK queue workers for ``ReportingProductionSupport``. + + The support schedules these workers itself. Omitting them keeps polling + and enabled atomic logical enqueue available, without advertising push. + Retained epoch-zero readiness queues are never among these participants. + """ + from adcp.reporting.outbox.pg import PgReportingOutbox + from adcp.reporting.production.pg import PgReportingProductionOutbox, PgReportingProductionStore + + if ( + projection.ledger is not store + or not projection.policy["notifications_enabled"] + or type(signing) is not ReportingProductionSigning + ): + raise ReportingNotificationError("notification_chain_unready") + ReportingProductionSigning.__post_init__(signing) + signed_subscriptions = _SignedSubscriptions(subscriptions) + outboxes: tuple[Any, ...] + if type(store) is InMemoryReportingProductionStore: + outboxes = ( + InMemoryReportingOutbox(store), + projection.outbox, + InMemoryReportingProductionOutbox(store), + ) + elif type(store) is PgReportingProductionStore: + outboxes = ( + PgReportingOutbox(pool=store._pool, clock=store._clock), + projection.outbox, + PgReportingProductionOutbox(pool=store._pool, clock=store._clock), + ) + else: + raise ReportingNotificationError("notification_chain_unready") + return tuple( + ReportingNotificationWorker( + outbox=outbox, + subscriptions=signed_subscriptions, + cipher=cipher, + signing=signing, + activity=outbox, + clock=store._clock, + delivery_window=ProductionDeliveryWindow(store, queue), + ) + for outbox, queue in zip(outboxes, ("core", "status", "ready")) + ) + + +def worker_identity(worker: ReportingNotificationWorker) -> tuple[int, ...]: + return tuple( + id(component) + for component in ( + worker, + worker.outbox, + worker.subscriptions, + worker.cipher, + worker.signing, + worker.activity, + worker.delivery_window, + getattr(worker.signing, "resolver", None), + getattr(worker.signing, "algorithms", None), + getattr(worker.signing, "brand_json_url", None), + getattr(worker.subscriptions, "resolver", None), + ) + ) + + +def check_workers(support: ReportingProductionSupport) -> None: + workers = support.notification_workers + if not workers: + return + if ( + type(workers) is not tuple + or len(workers) != 3 + or not support.notifications_enabled + or any(type(w) is not ReportingNotificationWorker for w in workers) + or any(type(w.cipher) is not ReportingEnvelopeCipher for w in workers) + or any(type(w.signing) is not ReportingProductionSigning for w in workers) + or any(type(w.subscriptions) is not _SignedSubscriptions for w in workers) + or any(type(w.delivery_window) is not ProductionDeliveryWindow for w in workers) + or any(id(w.activity) != id(w.outbox) for w in workers) + or any( + w.subscriptions is not workers[0].subscriptions + or w.signing is not workers[0].signing + or w.cipher is not workers[0].cipher + for w in workers + ) + or workers[1].outbox is not support.projection.outbox + or tuple(worker_identity(w) for w in workers) != support._notification_identity + ): + raise ReportingNotificationError("notification_chain_unready") + expected: tuple[type[Any], ...] + if isinstance(support.store, InMemoryReportingProductionStore): + from adcp.reporting.projection.memory import InMemoryReportingProjectionOutbox + + expected = ( + InMemoryReportingOutbox, + InMemoryReportingProjectionOutbox, + InMemoryReportingProductionOutbox, + ) + linked = all(getattr(w.outbox, "_store", None) is support.store for w in workers) + else: + from adcp.reporting.outbox.pg import PgReportingOutbox + from adcp.reporting.production.pg import PgReportingProductionOutbox + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + + expected = (PgReportingOutbox, PgReportingProjectionOutbox, PgReportingProductionOutbox) + linked = all(getattr(w.outbox, "_pool", None) is support.store._pool for w in workers) + if not linked or tuple(type(w.outbox) for w in workers) != expected: + raise ReportingNotificationError("notification_chain_unready") + for worker, queue in zip(workers, ("core", "status", "ready")): + window = worker.delivery_window + if ( + not isinstance(window, ProductionDeliveryWindow) + or window.store is not support.store + or window.queue != queue + ): + raise ReportingNotificationError("notification_chain_unready") + signing = workers[0].signing + assert isinstance(signing, ReportingProductionSigning) + ReportingProductionSigning.__post_init__(signing) + subscriptions = workers[0].subscriptions + assert isinstance(subscriptions, _SignedSubscriptions) + if not all( + callable(getattr(subscriptions.resolver, method, None)) + for method in ("list_active", "get_active") + ): + raise ReportingNotificationError("notification_chain_unready") + + +def signing_capabilities(support: ReportingProductionSupport) -> dict[str, Any]: + check_workers(support) + signing = support.notification_workers[0].signing + assert isinstance(signing, ReportingProductionSigning) + return signing.wire() + + +def signing_identity(support: ReportingProductionSupport) -> dict[str, str]: + check_workers(support) + signing = support.notification_workers[0].signing + assert isinstance(signing, ReportingProductionSigning) + return {"brand_json_url": signing.brand_json_url} + + +async def check_account_notifications(support: ReportingProductionSupport, account_id: str) -> None: + """Resolve trusted registrations/signing before admitting this account. + + This check does not replace dispatch's authorization/revocation checks. + A supported empty seller needs no invented account to advertise discovery. + """ + import asyncio + + check_workers(support) + if not support.notification_workers: + return + worker = support.notification_workers[0] + for event_type in ( + "reporting.ledger_changed", + "reporting.status_changed", + "reporting.delivery_ready", + ): + subscriptions = await asyncio.wait_for( + worker.subscriptions.list_active(account_id=account_id, notification_type=event_type), + timeout=worker.lease_seconds * 0.8, + ) + if not isinstance(subscriptions, (tuple, list)): + raise ReportingNotificationError("notification_chain_unready") + identifiers = set() + for subscription in subscriptions: + if ( + type(subscription) is not ReportingNotificationSubscription + or subscription.account_id != account_id + or subscription.subscriber_id in identifiers + or event_type not in subscription.event_types + or not ( + subscription.active and subscription.authorized and subscription.proof_valid + ) + ): + raise ReportingNotificationError("notification_chain_unready") + ReportingNotificationSubscription.__post_init__(subscription) + identifiers.add(subscription.subscriber_id) + sender = await asyncio.wait_for( + worker._sender(subscription), timeout=worker.lease_seconds * 0.8 + ) + await sender.aclose() + check_workers(support) + + +async def next_account(worker: ReportingNotificationWorker, *, delivery: bool) -> str | None: + """Sample one indexed due queue row; never enumerate adopter accounts.""" + from adcp.reporting.outbox.pg import PgReportingOutbox, database_now + + outbox = worker.outbox + if isinstance(outbox, InMemoryReportingOutbox): + now = worker._clock() + async with outbox._store._lock: + pending = ( + ((key[0], work) for key, (_, work) in outbox._state.deliveries.items()) + if delivery + else ((key[0], work) for key, work in outbox._state.expansions.items()) + ) + values = [(work.due_at, account) for account, work in pending if work.available(now)] + return min(values)[1] if values else None + if not isinstance(outbox, PgReportingOutbox): + raise ReportingNotificationError("notification_chain_unready") + # Both identifiers are closed SDK literals. The concrete queue adapter + # selects the matching isolated table on this same actual connection. + table = "reporting_notification_deliveries" if delivery else "reporting_notification_expansions" + async with outbox._connection() as connection: + now = await database_now(connection, outbox._clock) + row = await ( + await connection.execute( + f"SELECT account_id FROM {table}" # nosec B608 + " WHERE due_at<=%s AND (state='pending' OR" + " (state='leased' AND lease_expires_at<=%s))" + " ORDER BY due_at,account_id LIMIT 1", + (now, now), + ) + ).fetchone() + return str(row[0]) if row is not None else None + + +async def notification_turn(support: ReportingProductionSupport) -> None: + for worker in support.notification_workers: + for delivery in (False, True): + support._assert_components() + account_id = await next_account(worker, delivery=delivery) + if account_id is not None: + support._assert_components() + operation = worker.deliver_one if delivery else worker.expand_one + await operation(account_id=account_id) diff --git a/src/adcp/reporting/production/offerings.py b/src/adcp/reporting/production/offerings.py new file mode 100644 index 000000000..932ffc8f6 --- /dev/null +++ b/src/adcp/reporting/production/offerings.py @@ -0,0 +1,352 @@ +"""Atomic offerings bind public promises to the actual producer and verifier.""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any + +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding +from adcp.reporting.ledger.models import ( + ReportingConfiguration, + _schedule_clock, + iso_duration_to_timedelta, +) +from adcp.reporting.ledger.producer import ReportingProducer +from adcp.reporting.materializer.contracts import ReportingVerificationKey, failure +from adcp.reporting.materializer.verification import _same_definition +from adcp.reporting.production.contracts import ( + ReportingProductionSource, + ReportingProductionSourceBinding, +) +from adcp.reporting.source import ( + AuthoritativeOfferingV1, + ProvisionalSnapshotOfferingV1, + ReportingSourceCapabilitiesV1, + ReportingSourceStagedObjectReader, + iso_duration_milliseconds_v1, +) +from adcp.types import ReportingDeliveryOffering + + +@dataclass(frozen=True) +class ReportingProductionOffering: + """One public offering and the exact running components that implement it. + + The wire model is copied into immutable bytes so mutating an adopter's + Pydantic model after construction cannot change the advertised contract. + Capability availability is still checked against live components. + """ + + offering: ReportingDeliveryOffering = field(repr=False, compare=False) + producer: ReportingProducer = field(repr=False, compare=False) + verification_key: ReportingVerificationKey + source_offering_id: str + _wire: bytes = field(init=False, repr=False) + _producer_key: str = field(init=False, repr=False) + _source_identity: int = field(init=False, repr=False) + _reader_identity: int = field(init=False, repr=False) + + def __post_init__(self) -> None: + checked = ReportingDeliveryOffering.model_validate( + self.offering.model_dump(mode="json", exclude_none=True) + ) + raw = checked.model_dump(mode="json", exclude_none=True) + from adcp.validation.schema_loader import get_named_validator + + validator = get_named_validator("core/reporting-delivery-offering.json") + if validator is None or next(validator.iter_errors(raw), None) is not None: + raise ValueError("offering must satisfy the complete public contract") + profile, definition, key = ( + raw["reporting_profile"], + self.verification_key.definition, + self.verification_key, + ) + method = raw.get("method") + if ( + method is None + or method.get("orchestration") != "producer_managed" + or (method["pattern"], method["transport"], method.get("format")) + != (key.capability.method, key.capability.transport, key.capability.format) + or (raw["report_definition_id"], profile["id"]) + != (key.report_definition_id, key.reporting_profile) + or (raw["report_definition_uri"], raw["report_definition_sha256"].lower()) + != (definition.report_definition_uri, definition.report_definition_sha256) + or ( + profile["version"], + profile["schema_uri"], + profile["schema_sha256"].lower(), + profile["schema_dialect"], + profile["schema_ref_policy"], + ) + != ( + definition.schema_version, + definition.schema_uri, + definition.schema_sha256, + definition.schema_dialect, + definition.schema_ref_policy, + ) + or not raw["supported_finality"] + or len(set(raw["supported_finality"])) != len(raw["supported_finality"]) + ): + raise ValueError("offering must match its installed producer and exact verifier") + if raw["reconciliation_mode"] == "consumer_receipt": + if ( + raw["supported_finality"] != ["official"] + or key.capability.verification_profile != "canonical_digest" + or ( + profile.get("canonicalization_id"), + profile.get("canonicalization_uri"), + str(profile.get("canonicalization_sha256", "")).lower(), + ) + != ( + key.canonicalization.canonicalization_id, + key.canonicalization.canonicalization_uri, + key.canonicalization.canonicalization_sha256, + ) + ): + raise ValueError("reconciled offerings require official canonical receipt evidence") + elif raw["feed_purpose"] == "billing": + raise ValueError("billing offerings require consumer receipts") + object.__setattr__(self, "_wire", canonical_json_utf8_v1(raw)) + object.__setattr__(self, "_source_identity", id(self.producer._source)) + object.__setattr__(self, "_reader_identity", id(self.producer._object_reader)) + object.__setattr__( + self, + "_producer_key", + hashlib.sha256( + canonical_json_utf8_v1( + { + "offering": raw, + "source_offering_id": self.source_offering_id, + "publication_namespace": self.producer._offerings.publication_namespace, + "source_scope": dict(self.producer._offerings.source_scope), + } + ) + ).hexdigest(), + ) + self.check_source() + + @property + def offering_id(self) -> str: + return str(self.wire()["offering_id"]) + + @property + def reconciled(self) -> bool: + return bool(self.wire()["reconciliation_mode"] == "consumer_receipt") + + def wire(self) -> dict[str, Any]: + return dict(json.loads(self._wire)) + + def check_source(self, *, effective: bool = False) -> ReportingSourceCapabilitiesV1: + source = self.producer._source + if ( + id(source) != self._source_identity + or not isinstance(source, ReportingProductionSource) + or id(self.producer._object_reader) != self._reader_identity + or not isinstance(self.producer._object_reader, ReportingSourceStagedObjectReader) + ): + raise failure("BINDING_MISMATCH") + capabilities = ReportingSourceCapabilitiesV1.model_validate( + source.capabilities.model_dump(mode="json", exclude_none=True) + ) + if effective and capabilities.scope != "effective_account": + raise failure("BINDING_MISMATCH") + raw = self.wire() + source_offering = capabilities.offering(self.source_offering_id) + contract, key = source_offering.contract, self.verification_key + source_values = contract.model_dump(mode="json") + expected = { + "report_definition_id": key.report_definition_id, + "reporting_profile": key.reporting_profile, + } + expected.update( + { + name: getattr(key.definition, name) + for name in ( + "report_definition_uri", + "report_definition_sha256", + "schema_version", + "schema_uri", + "schema_sha256", + "schema_dialect", + "schema_ref_policy", + ) + } + ) + finality = raw["supported_finality"] + configured = self.producer._offerings + if ( + any(source_values.get(name) != value for name, value in expected.items()) + or source_offering.publication_namespace != configured.publication_namespace + or dict(configured.source_scope) != capabilities.source_scope + or not source_offering.provider_execution.supports_cancellation + or ( + "official" in finality + and ( + not isinstance(source_offering, AuthoritativeOfferingV1) + or configured.official_offering_id != self.source_offering_id + or ( + self.reconciled + and source_offering.correction_policy != "immutable_correction" + ) + ) + ) + or ( + "snapshot" in finality + and ( + not isinstance(source_offering, ProvisionalSnapshotOfferingV1) + or configured.snapshot_offering_id != self.source_offering_id + ) + ) + ): + raise failure("UNSUPPORTED_VERIFICATION") + schedule = raw["schedule"] + duration = iso_duration_milliseconds_v1(schedule["period_duration"]) + if not ( + iso_duration_milliseconds_v1(source_offering.windowing.minimum_window) + <= duration + <= iso_duration_milliseconds_v1(source_offering.windowing.maximum_window) + ) or iso_duration_milliseconds_v1(schedule["delivery_sla"]) < iso_duration_milliseconds_v1( + source_offering.worst_case_availability_lag + ): + raise failure("UNSUPPORTED_VERIFICATION") + if isinstance( + source_offering, ProvisionalSnapshotOfferingV1 + ) and duration < iso_duration_milliseconds_v1(source_offering.fastest_safe_cadence): + raise failure("UNSUPPORTED_VERIFICATION") + if ( + schedule["alignment"] == "source_timezone" + and schedule.get("period_timezone") != source_offering.source_timezone + ): + raise failure("UNSUPPORTED_VERIFICATION") + return capabilities + + def source_binding( + self, configuration: ReportingConfiguration + ) -> ReportingProductionSourceBinding: + try: + capabilities = self.check_source(effective=True) + source = self.producer._source + assert isinstance(source, ReportingProductionSource) + binding = source.configuration_binding(configuration) + if type(binding) is not ReportingProductionSourceBinding: + raise failure("BINDING_MISMATCH") + binding.check(configuration, capabilities, self.source_offering_id) + return binding + except Exception: + raise failure("BINDING_MISMATCH") from None + + def configuration_schedule(self, configuration: ReportingConfiguration) -> dict[str, Any]: + """Project a proven legacy clock into the public schedule vocabulary. + + Existing captured clocks and their closed storage decoder stay intact. + New production admission requires the normative public phase; an old + explicit anchor is compatible only when it produces the same periods. + Calendar-month/year clocks unsupported by that decoder are refused. + """ + offered = self.wire()["schedule"] + schedule = configuration.schedule + alignment = offered["alignment"] + expected_alignment = ( + alignment if alignment in {"utc", "account_timezone"} else "custom_timezone" + ) + zone, duration, anchor = _schedule_clock(schedule, configuration.account_timezone) + if ( + schedule.alignment != expected_alignment + or schedule.period_duration != offered["period_duration"] + or schedule.delivery_sla != offered["delivery_sla"] + or (alignment != "billing_cycle" and (anchor - datetime(1970, 1, 1)) % duration) + or (alignment == "billing_cycle" and schedule.period_anchor is None) + ): + raise failure("BINDING_MISMATCH") + result: dict[str, Any] = { + "period_duration": schedule.period_duration, + "delivery_sla": schedule.delivery_sla, + "alignment": alignment, + } + if alignment in {"source_timezone", "billing_cycle"}: + result["period_timezone"] = zone.key + if alignment == "billing_cycle": + assert schedule.period_anchor is not None + result["period_anchor"] = schedule.period_anchor.isoformat() + if ( + offered.get("period_timezone_policy") == "fixed" + or offered.get("period_anchor_policy") == "fixed" + ) and result.get("period_timezone") != offered.get("period_timezone"): + raise failure("BINDING_MISMATCH") + if offered.get( + "period_anchor_policy" + ) == "fixed" and schedule.period_anchor != aware_timestamp(offered["period_anchor"]): + raise failure("BINDING_MISMATCH") + if ( + alignment == "source_timezone" + and zone.key + != self.check_source(effective=True).offering(self.source_offering_id).source_timezone + ): + raise failure("BINDING_MISMATCH") + return result + + def check_configuration( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + retention_days: int, + ) -> None: + capabilities = self.check_source(effective=True) + self.source_binding(configuration) + self.configuration_schedule(configuration) + raw, key = self.wire(), self.verification_key + schedule, offered = configuration.schedule, raw["schedule"] + if ( + binding.generation_key != configuration.generation_key + or ( + configuration.report_definition_id, + configuration.reporting_profile, + configuration.feed_purpose, + configuration.required_finality, + ) + != ( + key.report_definition_id, + key.reporting_profile, + raw["feed_purpose"], + raw["supported_finality"][0], + ) + or not _same_definition(key, configuration.definition) + or configuration.authoritative_party != "seller" + or binding.reconciliation_mode != raw["reconciliation_mode"] + or binding.feed_purpose != raw["feed_purpose"] + or (binding.method, binding.transport, binding.format, binding.verification_profile) + != ( + key.capability.method, + key.capability.transport, + key.capability.format, + key.capability.verification_profile, + ) + or binding.resource_retention_days != retention_days + or binding.reader_compatibility != tuple(raw["method"].get("reader_compatibility", ())) + or schedule.period_duration != offered["period_duration"] + or iso_duration_to_timedelta(schedule.delivery_sla) + != iso_duration_to_timedelta(offered["delivery_sla"]) + or iso_duration_milliseconds_v1(schedule.delivery_sla) + < iso_duration_milliseconds_v1( + capabilities.offering(self.source_offering_id).worst_case_availability_lag + ) + or ( + offered.get("period_anchor_policy") == "fixed" + and ( + schedule.period_anchor is None + or schedule.period_anchor != aware_timestamp(str(offered.get("period_anchor"))) + ) + ) + or ( + offered.get("period_timezone_policy") == "fixed" + and schedule.period_timezone != offered.get("period_timezone") + ) + ): + raise failure("BINDING_MISMATCH") diff --git a/src/adcp/reporting/production/pg.py b/src/adcp/reporting/production/pg.py new file mode 100644 index 000000000..9d25821e0 --- /dev/null +++ b/src/adcp/reporting/production/pg.py @@ -0,0 +1,755 @@ +"""Production admission selects new participants in the single B2.1 transaction.""" + +from __future__ import annotations + +import asyncio +import json +import weakref +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextvars import ContextVar +from datetime import datetime, timedelta +from importlib.resources import files +from typing import TYPE_CHECKING, Any + +from adcp.reporting.ledger._delivery_state import decode_record +from adcp.reporting.ledger.delivery_models import ( + ReportingDeliveryPrincipal, + ReportingDeliveryScope, + ReportingDestinationBinding, +) +from adcp.reporting.ledger.models import ReportingConfiguration, ReportingObligationRecord +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.pg import _configuration_from_row +from adcp.reporting.ledger.producer_progress import acquisition_state, check_next_period +from adcp.reporting.ledger.store import LeasedConfiguration, LedgerConflictError, _utc +from adcp.reporting.materializer.capture import ReportingMaterializerBoundary +from adcp.reporting.materializer.contracts import ( + ReportingPreparedRevision, + ReportingVerificationKey, + ReportingWriterFailure, +) +from adcp.reporting.materializer.verification import ReportingVerifiedDestination +from adcp.reporting.materializer.work import ( + MaterializerContext, + ReportingMaterializerLease, + ReportingMaterializerTurn, + key_for, +) +from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox +from adcp.reporting.projection.pg import PgReportingProjectionStore +from adcp.reporting.source import ReportingConstituent + +if TYPE_CHECKING: + from adcp.reporting.production.service import ReportingProductionSupport + +_EPOCH: ContextVar[tuple[int, int] | None] = ContextVar( + "reporting_production_operation", default=None +) +_ADMISSION_CONNECTION: ContextVar[tuple[int, object, Any] | None] = ContextVar( + "reporting_production_configuration_connection", default=None +) + +_SOURCE_CONFIGURATION = ( + "SELECT c.delivery_config_id,c.delivery_config_version,c.account_id," + " c.report_definition_id,c.reporting_profile,c.feed_purpose," + " c.required_finality,c.account_timezone,c.schedule,c.media_buy_ids," + " c.activated_at,c.deactivated_at," + " c.automated_recovery_seconds,c.status_retention_days,c.definition," + " c.authoritative_party,g.producer_key,g.source_binding FROM reporting_configurations c" + " JOIN reporting_production_generations g" + " USING(account_id,delivery_config_id,delivery_config_version)" + " JOIN reporting_production_accounts a ON a.account_id=g.account_id" + " WHERE c.account_id=%s AND c.delivery_config_id=%s" + " AND c.delivery_config_version=%s AND g.producer_key=ANY(%s)" +) + +_OWNED = ( + "(SELECT account_id,consumer_id,delivery_config_id,delivery_config_version," + "reporting_obligation_id,reporting_materialization_id,state,retry_allowed" + " FROM reporting_materializer_work UNION ALL" + " SELECT account_id,consumer_id,delivery_config_id,delivery_config_version," + "reporting_obligation_id,reporting_materialization_id,state,retry_allowed" + " FROM reporting_production_work)" +) + +_ProducerSample = tuple[int, datetime | None, str, str, int] + + +class _ProductionConnection: + """Only fixed SDK identifiers are selected; account/writer ordering stays intact.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + def transaction(self) -> Any: + return self.connection.transaction() + + async def execute(self, query: str, params: Any = None) -> Any: + # A known terminal B2.1 failure remains eligible for an N+1 retry, but + # pending/unknown effects in either epoch prohibit a new reservation. + if query.startswith("SELECT retry_allowed FROM reporting_materializer_work"): + query = query.replace("reporting_materializer_work", _OWNED + " owned", 1) + elif "SELECT 1 FROM reporting_materializer_work w WHERE" in query: + query = query.replace( + "FROM reporting_materializer_work w WHERE", "FROM " + _OWNED + " w WHERE" + ) + else: + for old, new in ( + ("reporting_materializer_notification_", "reporting_production_notification_"), + ("reporting_materializer_status_", "reporting_production_status_"), + ("reporting_materializer_work", "reporting_production_work"), + ): + query = query.replace(old, new) + return await self.connection.execute(query, params) + + +class _ProductionQueueConnection: + def __init__(self, connection: Any) -> None: + self.connection = connection + + def transaction(self) -> Any: + return self.connection.transaction() + + async def execute(self, query: str, params: Any = None) -> Any: + for old, new in ( + ("reporting_notification_", "reporting_production_notification_"), + ("reporting_webhook_", "reporting_production_webhook_"), + ): + query = query.replace(old, new) + return await self.connection.execute(query, params) + + +class PgReportingProductionOutbox(PgReportingStatusOutbox): + """Generic crash-safe delivery/fanout and private activity on the admitted queue.""" + + @asynccontextmanager + async def _connection(self) -> AsyncIterator[Any]: + async with self._pool.connection() as connection: + yield _ProductionQueueConnection(connection) + + async def create_schema(self) -> None: + await PgReportingProductionStore(pool=self._pool, notifications=True).create_schema() + + +class PgReportingProductionStore(PgReportingProjectionStore): + """The production store retains all ordinary legacy reader/writer APIs. + + New work requires the live SDK composition and its mounted applicable + routes. Pending epoch-zero work uses its original tables, identity and + permanently quarantined enqueue. Installation alone admits nothing. + """ + + _production_support: weakref.ReferenceType[ReportingProductionSupport] | None = None + _production_lease_samples: dict[tuple[str, ...], _ProducerSample] | None = None + + def _owner(self) -> ReportingProductionSupport: + from adcp.reporting.production.service import production_owner + + return production_owner(self) + + @asynccontextmanager + async def _connection(self) -> AsyncIterator[Any]: + admission = _ADMISSION_CONNECTION.get() + if admission is not None and admission[:2] == (id(self), asyncio.current_task()): + yield admission[2] + return + async with super()._connection() as connection: + if _EPOCH.get() == (id(self), 2): + yield _ProductionConnection(connection) + else: + yield connection + + async def admit_production_configuration( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + offering_id: str, + ) -> None: + offering = self._owner()._configuration_offering( + configuration, binding, offering_id=offering_id + ) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, configuration.account_id) + token = _ADMISSION_CONNECTION.set((id(self), asyncio.current_task(), connection)) + try: + await self.put_configuration(configuration) + await self.put_destination_binding(binding) + await self._enroll_on(connection, configuration, offering._producer_key, binding) + finally: + _ADMISSION_CONNECTION.reset(token) + + async def _enroll_on( + self, + connection: Any, + configuration: ReportingConfiguration, + producer_key: str, + destination: ReportingDestinationBinding, + ) -> None: + key = configuration.generation_key + identity = (key.account_id, key.delivery_config_id, key.delivery_config_version) + binding = self._owner()._source_binding(configuration, producer_key).document() + await connection.execute( + "INSERT INTO reporting_production_generations VALUES(%s,%s,%s,%s,%s::jsonb)" + " ON CONFLICT DO NOTHING", + (*identity, producer_key, json.dumps(binding)), + ) + row = await ( + await connection.execute( + "SELECT producer_key,source_binding FROM reporting_production_generations" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + if row is None or row != (producer_key, binding): + raise ReportingNotificationError("reporting_production_source_conflict") + owner = self._owner() + offering = owner._configuration_offering( + configuration, destination, producer_key=producer_key + ) + document = owner._destination_binding(destination, offering).wire() + destination_identity = ( + key.account_id, + destination.consumer_id, + key.delivery_config_id, + key.delivery_config_version, + ) + await connection.execute( + "INSERT INTO reporting_production_destination_bindings VALUES(%s,%s,%s,%s,%s::jsonb)" + " ON CONFLICT DO NOTHING", + (*destination_identity, json.dumps(document)), + ) + original = await ( + await connection.execute( + "SELECT method FROM reporting_production_destination_bindings" + " WHERE account_id=%s AND consumer_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s", + destination_identity, + ) + ).fetchone() + if original is None or original[0] != document: + raise ReportingNotificationError("reporting_production_destination_conflict") + + async def lease_period_close( + self, *, worker_id: str, now: datetime, lease_seconds: float + ) -> LeasedConfiguration | None: + keys = self._owner()._producer_keys() + if not keys: + return None + moment = _utc(now) + expires = moment + timedelta(seconds=lease_seconds) + if expires <= moment: + raise ValueError("producer lease duration must be positive") + if self._production_lease_samples is None: + self._production_lease_samples = {} + after = self._production_lease_samples.get(keys) + following: _ProducerSample | None = None + result = None + async with self._connection() as connection, connection.transaction(): + # Discover a bounded set without locking configuration rows. The + # inherited configuration trigger takes the account lock, so that + # lock must precede the row lock here, just as it does in activation. + # A busy account cannot advance a durable rank while another + # transaction holds its lock. Continue a read-only sample instead + # of repeatedly trying the same prefix. At most one nonempty + # 32-row window is examined; an empty tail may wrap once. + for _ in range(2): + continuation = ( + " AND (greatest(coalesce(t.lease_turn,0),coalesce(p.probe_turn,0))," + " coalesce(c.lease_expires_at,'-infinity'::timestamptz)," + " c.account_id,c.delivery_config_id,c.delivery_config_version)" + " > (%s,coalesce(%s::timestamptz,'-infinity'::timestamptz),%s,%s,%s)" + if after is not None + else "" + ) + query = ( + "SELECT c.account_id,c.delivery_config_id,c.delivery_config_version," # nosec B608 + " greatest(coalesce(t.lease_turn,0),coalesce(p.probe_turn,0))," + " c.lease_expires_at" + " FROM reporting_production_generations g" + " JOIN reporting_production_accounts a ON a.account_id=g.account_id" + " JOIN reporting_configurations c" + " ON (c.account_id,c.delivery_config_id,c.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " LEFT JOIN adcp_reporting_configuration_lease_turns t" + " ON (t.account_id,t.delivery_config_id,t.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " LEFT JOIN reporting_production_source_probe_turns p" + " ON (p.account_id,p.delivery_config_id,p.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " WHERE g.producer_key=ANY(%s)" + " AND (c.lease_expires_at IS NULL OR c.lease_expires_at<=%s)" + + continuation + + " ORDER BY greatest(coalesce(t.lease_turn,0),coalesce(p.probe_turn,0))," + " c.lease_expires_at NULLS FIRST," + " c.account_id,c.delivery_config_id,c.delivery_config_version LIMIT 32" + ) + # Only the fixed SDK continuation above changes this SQL; + # every cursor value and source identity remains a parameter. + rows = await ( + await connection.execute(query, (list(keys), moment, *(after or ()))) + ).fetchall() + if rows or after is None: + break + after = None + for candidate in rows: + row = candidate[:3] + following = (candidate[3], candidate[4], row[0], row[1], row[2]) + locked = await ( + await connection.execute( + "SELECT pg_try_advisory_xact_lock(hashtext('adcp.reporting:' || %s))", + (row[0],), + ) + ).fetchone() + if not locked[0]: + continue + current = await ( + await connection.execute(_SOURCE_CONFIGURATION, (*row, list(keys))) + ).fetchone() + if current is None: + continue + configuration = _configuration_from_row(current[:16]) + try: + self._owner()._check_source_binding(configuration, current[16], current[17]) + except Exception: + # A permanently revoked generation must not occupy the + # first bounded window forever. This is a probe, not a + # lease: preserve ordinary lease ranks and all source and + # external identities. Its durable rank shares the same + # ordering clock as successful configuration acquisitions. + await connection.execute( + "INSERT INTO reporting_production_source_probe_turns" + " (account_id,delivery_config_id,delivery_config_version,probe_turn)" + " VALUES(%s,%s,%s,nextval('adcp_reporting_configuration_lease_turn_seq'))" + " ON CONFLICT(account_id,delivery_config_id,delivery_config_version)" + " DO UPDATE SET probe_turn=" + "nextval('adcp_reporting_configuration_lease_turn_seq')", + tuple(row), + ) + continue + acquired = await ( + await connection.execute( + "UPDATE reporting_configurations SET lease_worker_id=%s,lease_expires_at=%s" + " WHERE account_id=%s AND delivery_config_id=%s" + " AND delivery_config_version=%s" + " AND (lease_expires_at IS NULL OR lease_expires_at<=%s)" + " RETURNING account_id", + (worker_id, expires, *row, moment), + ) + ).fetchone() + if acquired is None: + continue + await self._retain_materializer_generation_on_lease_change(connection, tuple(row)) + await connection.execute( + "INSERT INTO adcp_reporting_configuration_lease_turns" + " (account_id,delivery_config_id,delivery_config_version,lease_turn)" + " VALUES(%s,%s,%s,nextval('adcp_reporting_configuration_lease_turn_seq'))" + " ON CONFLICT(account_id,delivery_config_id,delivery_config_version)" + " DO UPDATE SET lease_turn=" + "nextval('adcp_reporting_configuration_lease_turn_seq')", + tuple(row), + ) + result = LeasedConfiguration(row[0], row[1], row[2], expires) + break + # Hints are per store and selected producer keys, not durable work or + # leases. Publish a hint only after commit; a failed mutation retries + # the same window. Successful acquisition returns to the durable + # turn-primary order. A fresh store starts there too. Concurrent hints + # may cause a bounded revisit, but cannot authorize or fence any work. + if result is None and following is not None: + self._production_lease_samples[keys] = following + else: + self._production_lease_samples.pop(keys, None) + return result + + async def release_period_close(self, lease: LeasedConfiguration, *, worker_id: str) -> None: + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, lease.account_id) + identity = (lease.account_id, lease.delivery_config_id, lease.delivery_config_version) + released = await ( + await connection.execute( + "UPDATE reporting_configurations SET lease_worker_id=NULL,lease_expires_at=NULL" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " AND lease_worker_id=%s AND lease_expires_at=%s RETURNING account_id", + (*identity, worker_id, lease.lease_expires_at), + ) + ).fetchone() + if released is not None: + await self._retain_materializer_generation_on_lease_change(connection, identity) + + async def _retain_materializer_generation_on_lease_change( + self, connection: Any, identity: tuple[str, str, int] + ) -> None: + # The immutable inherited trigger treats *every* configuration UPDATE + # as source invalidation, including lease-only bookkeeping. These two + # SDK statements change only lease fields, under the account lock. + # Cancel just their one trigger increment in the same transaction; + # the wakeup remains harmless. No intervening source write can be + # hidden, no committed generation goes backwards, and pending work's + # original generation/epoch/external identity is never rewritten. + # A real configuration, revision or readability change still executes + # the original trigger without this correction and fences old work. + await connection.execute( + "UPDATE reporting_materializer_candidates SET generation=generation-1" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + + @asynccontextmanager + async def _source_connection( + self, configuration: ReportingConfiguration + ) -> AsyncIterator[tuple[Any, tuple[str, str, int]]]: + keys = self._owner()._producer_keys() + key = configuration.generation_key + identity = (key.account_id, key.delivery_config_id, key.delivery_config_version) + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, key.account_id) + row = await ( + await connection.execute( + _SOURCE_CONFIGURATION, + (*identity, list(keys)), + ) + ).fetchone() + if row is None or _configuration_from_row(row[:16]) != configuration: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation unavailable") + self._owner()._check_source_binding(configuration, row[16], row[17]) + await connection.execute( + "INSERT INTO reporting_production_source_progress" + " (account_id,delivery_config_id,delivery_config_version) VALUES(%s,%s,%s)" + " ON CONFLICT DO NOTHING", + identity, + ) + token = _ADMISSION_CONNECTION.set((id(self), asyncio.current_task(), connection)) + try: + yield connection, identity + finally: + _ADMISSION_CONNECTION.reset(token) + + async def producer_constituents( + self, configuration: ReportingConfiguration, obligation: ReportingObligationRecord + ) -> tuple[ReportingConstituent, ...]: + async with self._source_connection(configuration) as (connection, identity): + if obligation.generation_key != configuration.generation_key or set( + obligation.media_buy_ids + ) != set(configuration.media_buy_ids): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "source denominator differs") + row = await ( + await connection.execute( + "SELECT producer_key,source_binding FROM reporting_production_generations" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + binding = self._owner()._check_source_binding(configuration, row[0], row[1]) + return binding.constituents() + + async def producer_closed_through( + self, configuration: ReportingConfiguration + ) -> datetime | None: + async with self._source_connection(configuration) as (connection, identity): + row = await ( + await connection.execute( + "SELECT closed_through FROM reporting_production_source_progress" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + return row[0] if row is not None else None + + async def commit_producer_period( + self, + configuration: ReportingConfiguration, + obligation: ReportingObligationRecord, + *, + previous_end: datetime | None, + ) -> ReportingObligationRecord: + check_next_period(configuration, obligation, previous_end) + async with self._source_connection(configuration) as (connection, identity): + row = await ( + await connection.execute( + "SELECT closed_through FROM reporting_production_source_progress" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + identity, + ) + ).fetchone() + current = row[0] + if current != previous_end and (current is None or current < obligation.period.end): + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer progress changed") + stored = await self.commit_obligation(obligation) + await connection.execute( + "INSERT INTO reporting_production_source_work" + " (account_id,delivery_config_id,delivery_config_version,reporting_obligation_id," + " period_end) VALUES(%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING", + (*identity, stored.reporting_obligation_id, stored.period.end), + ) + await connection.execute( + "UPDATE reporting_production_source_progress" + " SET closed_through=greatest(closed_through,%s)" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s", + (stored.period.end, *identity), + ) + return stored + + async def next_producer_obligations( + self, configuration: ReportingConfiguration, *, now: datetime, limit: int + ) -> tuple[str, ...]: + if type(limit) is not int or not 1 <= limit <= 64: + raise ValueError("production acquisition limit must be in 1..64") + async with self._source_connection(configuration) as (connection, identity): + rows = await ( + await connection.execute( + "SELECT reporting_obligation_id FROM reporting_production_source_work" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " AND state='pending' AND period_end<=%s" + " ORDER BY acquisition_turn,reporting_obligation_id LIMIT %s FOR UPDATE", + (*identity, now, limit), + ) + ).fetchall() + if not rows: + return () + head = await ( + await connection.execute( + "UPDATE reporting_production_source_progress" + " SET acquisition_turn=acquisition_turn+%s" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " RETURNING acquisition_turn", + (len(rows), *identity), + ) + ).fetchone() + for offset, row in enumerate(rows, 1): + await connection.execute( + "UPDATE reporting_production_source_work SET acquisition_turn=%s" + " WHERE account_id=%s AND reporting_obligation_id=%s", + (head[0] - len(rows) + offset, identity[0], row[0]), + ) + return tuple(row[0] for row in rows) + + async def finish_producer_acquisition( + self, configuration: ReportingConfiguration, *, reporting_obligation_id: str + ) -> None: + async with self._source_connection(configuration) as (connection, identity): + obligation = await self.get_obligation( + account_id=identity[0], reporting_obligation_id=reporting_obligation_id + ) + if obligation is None or obligation.generation_key != configuration.generation_key: + raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer generation differs") + revisions = await self.list_revisions( + account_id=identity[0], reporting_obligation_id=reporting_obligation_id + ) + await connection.execute( + "UPDATE reporting_production_source_work SET state=%s" + " WHERE account_id=%s AND delivery_config_id=%s AND delivery_config_version=%s" + " AND reporting_obligation_id=%s", + (acquisition_state(obligation, revisions), *identity, reporting_obligation_id), + ) + + @asynccontextmanager + async def _lease_epoch(self, lease: ReportingMaterializerLease) -> AsyncIterator[None]: + token = _EPOCH.set((id(self), lease.admission_epoch)) + try: + yield + finally: + _EPOCH.reset(token) + + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + root = files("adcp.reporting.ledger") + for name in ( + "reporting_materializer.sql", + "reporting_receipt_ingestion.sql", + "reporting_feed.sql", + "reporting_status_notifications.sql", + "reporting_status_selector_version.sql", + "reporting_projection.sql", + "reporting_projection_notifications.sql", + "reporting_projection_feed.sql", + "reporting_production.sql", + ): + await connection.execute(root.joinpath(name).read_text()) + + async def materializer_ready(self) -> bool: + owner = self._owner() + if not await owner._schema_ready(): + raise ReportingNotificationError("reporting_production_schema_unready") + owner._assert_components() + return True + + async def _activate_production(self, *, account_id: str) -> bool: + owner = self._owner() + await self.materializer_ready() + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, account_id) + owner._assert_components() + projection = await ( + await connection.execute( + "SELECT policy FROM reporting_projection_accounts" + " WHERE account_id=%s AND current_input IS NOT NULL", + (account_id,), + ) + ).fetchone() + if projection is None or projection[0] != owner.projection.policy: + raise ReportingNotificationError("status_projection_activation_required") + policy = owner._admission_policy() + current = await ( + await connection.execute( + "SELECT policy FROM reporting_production_accounts WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + if current is not None: + if current[0] != policy: + raise ReportingNotificationError("reporting_production_policy_conflict") + return False + await connection.execute( + "INSERT INTO reporting_production_accounts(account_id,policy) VALUES(%s,%s::jsonb)", + (account_id, json.dumps(policy)), + ) + configurations = { + c.generation_key: c + for c in await self._list_configurations_on(connection, account_id=account_id) + } + bindings = await ( + await connection.execute( + "SELECT payload FROM reporting_reconciliation_records" + " WHERE account_id=%s AND namespace='destination_binding'", + (account_id,), + ) + ).fetchall() + for (document,) in bindings: + binding = decode_record(document) + if not isinstance(binding, ReportingDestinationBinding): + raise ReportingNotificationError("reporting_production_history_corrupt") + configuration = configurations[binding.generation_key] + try: + offering = owner._configuration_offering(configuration, binding) + # Unsupported or unavailable bindings remain unadmitted. + except Exception: # nosec B112 + continue + await self._enroll_on(connection, configuration, offering._producer_key, binding) + return True + + async def _materializer_context_on( + self, connection: Any, scope: ReportingDeliveryScope + ) -> MaterializerContext: + context = await super()._materializer_context_on(connection, scope) + if isinstance(connection, _ProductionConnection): + owner = self._owner() + row = await ( + await connection.execute( + "SELECT a.policy,g.producer_key,g.source_binding,d.method" + " FROM reporting_production_accounts a" + " LEFT JOIN reporting_production_generations g" + " ON g.account_id=a.account_id AND g.delivery_config_id=%s" + " AND g.delivery_config_version=%s" + " LEFT JOIN reporting_production_destination_bindings d" + " ON (d.account_id,d.delivery_config_id,d.delivery_config_version)=" + " (g.account_id,g.delivery_config_id,g.delivery_config_version)" + " AND d.consumer_id=%s WHERE a.account_id=%s", + ( + scope.generation_key.delivery_config_id, + scope.generation_key.delivery_config_version, + scope.consumer_id, + scope.principal.account_id, + ), + ) + ).fetchone() + if row is None: + raise ReportingNotificationError("reporting_production_activation_required") + owner._check_context( + context, + key_for(context.binding, context.obligation, owner.keys), + row[0], + row[1], + row[2], + row[3], + ) + return context + + async def _claim_account_on( + self, + connection: Any, + account_id: str, + keys: tuple[ReportingVerificationKey, ...], + lease_seconds: int, + ) -> ReportingMaterializerLease | ReportingMaterializerTurn: + activated = await ( + await connection.execute( + "SELECT 1 FROM reporting_production_accounts WHERE account_id=%s", (account_id,) + ) + ).fetchone() + if activated is None: + return ReportingMaterializerTurn("idle") + old = await ( + await connection.execute( + "SELECT to_jsonb(w) FROM reporting_materializer_work w WHERE account_id=%s" + " AND state='pending' AND due_at<=clock_timestamp()" + " AND (lease_until IS NULL OR lease_until<=clock_timestamp())" + " ORDER BY due_at,reporting_materialization_id LIMIT 1 FOR UPDATE", + (account_id,), + ) + ).fetchone() + if old is not None: + return await super()._lease_on(connection, old[0], keys, lease_seconds) + return await super()._claim_account_on( + _ProductionConnection(connection), account_id, keys, lease_seconds + ) + + async def _schedule_account_on(self, connection: Any, account_id: str) -> None: + if isinstance(connection, _ProductionConnection): + connection = connection.connection + # _OWNED contains only fixed SDK tables/columns; account values are bound. + await connection.execute( + "UPDATE reporting_materializer_accounts SET due_at=(SELECT min(due) FROM (" # nosec B608 + " SELECT due_at AS due FROM reporting_materializer_work" + " WHERE account_id=%s AND state='pending'" + " UNION ALL SELECT due_at FROM reporting_production_work" + " WHERE account_id=%s AND state='pending'" + " UNION ALL SELECT c.due_at FROM reporting_materializer_candidates c" + " WHERE c.account_id=%s AND c.due_at IS NOT NULL AND NOT EXISTS (SELECT 1 FROM " + + _OWNED + + " w WHERE w.account_id=c.account_id AND w.consumer_id=c.consumer_id" # nosec B608 + " AND w.delivery_config_id=c.delivery_config_id" + " AND w.delivery_config_version=c.delivery_config_version" + " AND w.reporting_obligation_id=c.reporting_obligation_id AND w.state='pending')" + " UNION ALL SELECT clock_timestamp() FROM reporting_materializer_discovery" + " WHERE account_id=%s AND NOT complete) ready) WHERE account_id=%s", + (account_id,) * 5, + ) + + async def renew_materialization( + self, lease: ReportingMaterializerLease, *, lease_seconds: int = 30 + ) -> bool: + async with self._lease_epoch(lease): + return await super().renew_materialization(lease, lease_seconds=lease_seconds) + + async def authorize_materialization(self, lease: ReportingMaterializerLease) -> None: + async with self._lease_epoch(lease): + await super().authorize_materialization(lease) + + async def finish_materialization( + self, + lease: ReportingMaterializerLease, + *, + prepared: ReportingPreparedRevision | None = None, + verified: ReportingVerifiedDestination | None = None, + error: ReportingWriterFailure | None = None, + ) -> ReportingMaterializerTurn: + if lease.admission_epoch == 2: + self._owner()._assert_components() + async with self._lease_epoch(lease): + return await super().finish_materialization( + lease, prepared=prepared, verified=verified, error=error + ) + + async def read_production_boundaries( + self, *, caller: ReportingDeliveryPrincipal, after: int = 0, limit: int = 100 + ) -> tuple[ReportingMaterializerBoundary, ...]: + token = _EPOCH.set((id(self), 2)) + try: + return await super().read_materializer_boundaries( + caller=caller, after=after, limit=limit + ) + finally: + _EPOCH.reset(token) diff --git a/src/adcp/reporting/production/required_schema.json b/src/adcp/reporting/production/required_schema.json new file mode 100644 index 000000000..587a22303 --- /dev/null +++ b/src/adcp/reporting/production/required_schema.json @@ -0,0 +1,1326 @@ +{ + "column:reporting_production_accounts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_accounts.activated_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_production_accounts.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_production_accounts.policy": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_delivery_windows.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_delivery_windows.body_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_delivery_windows.expires_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_delivery_windows.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_delivery_windows.queue": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_delivery_windows.started_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_destination_bindings.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_destination_bindings.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_destination_bindings.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_destination_bindings.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_destination_bindings.method": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_generations.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_generations.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_generations.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_generations.producer_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_generations.source_binding": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_notification_deliveries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.auth_mode": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_deliveries.body_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_deliveries.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_deliveries.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_notification_deliveries.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.destination_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_deliveries.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_notification_deliveries.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_deliveries.envelope": { + "enabled": true, + "fingerprint": "554c34e416bd4546469b42d5773bc7c59b2104366ffa5259a2838c64cd826e57" + }, + "column:reporting_production_notification_deliveries.envelope_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_notification_deliveries.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_deliveries.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.key_version": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_notification_deliveries.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_deliveries.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.signing_scope_id": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_production_notification_deliveries.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_production_notification_deliveries.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_deliveries.subscription_fingerprint": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_notification_events.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_production_notification_events.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_events.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_notification_events.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_events.reporting_materialization_id": { + "enabled": true, + "fingerprint": "a0d17631e6e95ba976e4615a034020bc089fa63601fb91205ebb35798a39c0de" + }, + "column:reporting_production_notification_events.snapshot": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_notification_expansions.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_expansions.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_notification_expansions.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_expansions.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_notification_expansions.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_notification_expansions.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_expansions.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_notification_expansions.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_production_notification_expansions.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_notification_expansions.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_production_source_probe_turns.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_probe_turns.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_probe_turns.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_source_probe_turns.probe_turn": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_source_progress.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_progress.acquisition_turn": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_source_progress.closed_through": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_source_progress.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_progress.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_source_work.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_work.acquisition_turn": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_production_source_work.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_work.delivery_config_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_production_source_work.period_end": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_source_work.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_source_work.state": { + "enabled": true, + "fingerprint": "675d9766cc6787c63f91ee16167a1de38069be7ad44f6d8362bd498885ae1355" + }, + "column:reporting_production_status_boundaries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_status_boundaries.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_status_boundaries.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_status_boundaries.outcome_namespace": { + "enabled": true, + "fingerprint": "37f064bd049ffa20c99bccfcb2ddd77b4e7b65fc6471d7683087ed0beec2098e" + }, + "column:reporting_production_status_boundaries.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_boundaries.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_status_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_heads.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_status_heads.max_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempt_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.last_attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempt_heads.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempt_heads.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempts.binding": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_production_webhook_attempts.completed_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_webhook_attempts.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_production_webhook_attempts.http_status_code": { + "enabled": true, + "fingerprint": "4340876cb26818ac55a8d51cfbc7047e90fc4d7e44f570ee454b4552beb351a1" + }, + "column:reporting_production_webhook_attempts.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.lease_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_webhook_attempts.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.payload_size_bytes": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_webhook_attempts.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.reservation_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_webhook_attempts.response_time_ms": { + "enabled": true, + "fingerprint": "992336704a95e12ec6e959825c59fa2e51cddc5f1568af6bebaf12f03ac5655f" + }, + "column:reporting_production_webhook_attempts.status": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_production_webhook_attempts.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_webhook_attempts.url": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_production_work.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.acknowledged_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_work.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_production_work.attempt_namespace": { + "enabled": true, + "fingerprint": "48602ba37bcbee2d9c8104042cc7868262df4d288eaf1543e387b9a059a08117" + }, + "column:reporting_production_work.binding_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.completion_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_production_work.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.created_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_production_work.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.delivery_config_version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_work.due_at": { + "enabled": true, + "fingerprint": "336df3243b293695d8321965e92f26d3f132e11514ed9678fb5e64e0015fa580" + }, + "column:reporting_production_work.external_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_production_work.imported": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_production_work.lease_token": { + "enabled": true, + "fingerprint": "88223bccb5aae7ddfe4b3feacd193586f6699cbfaa3507ae6c21250775392e25" + }, + "column:reporting_production_work.lease_until": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_production_work.notifications_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_production_work.reason": { + "enabled": true, + "fingerprint": "806b02ec486fb1a02f57be76cf01aba1287baf2d669233d70849bcb0df7c558c" + }, + "column:reporting_production_work.reporting_materialization_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.reporting_obligation_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.reporting_revision_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_production_work.retry_allowed": { + "enabled": true, + "fingerprint": "981e469ff869d932309f9e6aab9b07ff394c7439fe2431281320e25b41c4198d" + }, + "column:reporting_production_work.state": { + "enabled": true, + "fingerprint": "675d9766cc6787c63f91ee16167a1de38069be7ad44f6d8362bd498885ae1355" + }, + "column:reporting_production_work.verification_key_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_pkey": { + "enabled": true, + "fingerprint": "e65d70e61c89a93d66c4c4f4c59ef755d0ff526b0fcdd1d9ae6d830d2abea913" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check": { + "enabled": true, + "fingerprint": "80f89f47761fc22c72aabd7083963e72e4113bcfe00f0923f45be306e540b837" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check1": { + "enabled": true, + "fingerprint": "e83adfb83c455b7b7141555b4fc6c67370d787abf3c77c87001d3aa622239510" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check2": { + "enabled": true, + "fingerprint": "5c2314fc1a16a0761359a7885fc31c52f3a91e1a80ad7ea305a6cb3807503502" + }, + "constraint:reporting_production_accounts.reporting_production_accounts_policy_check3": { + "enabled": true, + "fingerprint": "39bd47a78f6779fe1881995a88ce9ee48123e77e936ef12b30ed231a44019c11" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_body_sha256_check": { + "enabled": true, + "fingerprint": "08bcbac24dab1de32cebf5ea27d21c0762ef54f2fd47ce6045a5815ee395b1ab" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_check": { + "enabled": true, + "fingerprint": "f675507fe20dae95352895867c06a71166f3889144d28389539d43a345a358f5" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_pkey": { + "enabled": true, + "fingerprint": "c49724177245a89c200c46c61b54b60e7988cddb577d3ba58c67f771cf2ba204" + }, + "constraint:reporting_production_delivery_windows.reporting_production_delivery_windows_queue_check": { + "enabled": true, + "fingerprint": "d4ad040b32660abca511e5442252d8b9e1be8be6e7db8a009c464b51d0a56345" + }, + "constraint:reporting_production_destination_bindings.reporting_production_destinat_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_destination_bindings.reporting_production_destination_bindings_method_check": { + "enabled": true, + "fingerprint": "a1d15c03d0e951caea89b55170544a51d76bac60017539348d3f55a73caca80b" + }, + "constraint:reporting_production_destination_bindings.reporting_production_destination_bindings_pkey": { + "enabled": true, + "fingerprint": "cb0658b07aa5249a6934cca6248293967db4ab1a848d706c3d5310fa98f08785" + }, + "constraint:reporting_production_generations.reporting_production_generati_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "4761471309c35947c8d3928dce4595b4d455e7de6166aeb78a5de220465cb64a" + }, + "constraint:reporting_production_generations.reporting_production_generations_pkey": { + "enabled": true, + "fingerprint": "d9ba22090ae520014ae689fbeb3206c705853c9227e68c533acd51317287e944" + }, + "constraint:reporting_production_generations.reporting_production_generations_producer_key_check": { + "enabled": true, + "fingerprint": "50fbb9f237cebdab6477e2b7e6a469e50607d1fb5d4ad21505f9fe5f0e8f8fc8" + }, + "constraint:reporting_production_generations.reporting_production_generations_source_binding_check": { + "enabled": true, + "fingerprint": "4fdc2c43438889e1348918f91fb9793d2241bc40abfef17de4b9cc822f6c39ca" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notific_account_id_consumer_namespac_fkey2": { + "enabled": true, + "fingerprint": "290b23049e9cd59676d28813c39b0b7ee4319a728220816e72ebaee448450298" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notific_account_id_consumer_namespac_fkey3": { + "enabled": true, + "fingerprint": "8673e8804fc529b047821250805cf1f6350f041bf84c658933c53e64c2a5e8aa" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "91982910079d78e1b6162e69e1e40d0f98ab055b811726cd8a9357c4aa9cfd22" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "bece3b7de1c22f3a600c89c175f942f83ecff078142418072b1bc1b42a670c18" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_del_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliv_notification_type_check": { + "enabled": true, + "fingerprint": "4dc2b2e5d5fe003a3b98e1832dd453c9115cc3d03d59d3cb23258e90605548e3" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_delive_cause_generation_check": { + "enabled": true, + "fingerprint": "d93961696665129cf254c93dae4377741c5cef5fe9d373c6a5cb36226e5a69e2" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_cause_kind_check": { + "enabled": true, + "fingerprint": "4d213a62ba48d8d428c2811ddc5039b1349d3a41943509a85062b43235ab669d" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_check": { + "enabled": true, + "fingerprint": "a61e7698dd266b3a4abee2c2716b00919f047e85bcd888a90c1a5bc5350c4d6e" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "126da0131155e9b09e48d24466c5f61b7b6dc1540a68c465c52b59769637820d" + }, + "constraint:reporting_production_notification_deliveries.reporting_production_notification_deliveries_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "39075eefddb9b204eef01670600946cd8e67c770bc795b66256ab4974afdef81" + }, + "constraint:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "1d1f2cee412feec063fd1340b0d7773adfc55cad1f740e21bb45f75acf185b8c" + }, + "constraint:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "7156cd6623f45f181906d2886b53dfba82bfdfc6cdcbdd0017c1f1cd7ceae407" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_even_consumer_namespace_check": { + "enabled": true, + "fingerprint": "69951e411156e1739417ca750e81d5d806b2fcccea7e1914674f730ca6433b58" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_event_notification_type_check": { + "enabled": true, + "fingerprint": "4dc2b2e5d5fe003a3b98e1832dd453c9115cc3d03d59d3cb23258e90605548e3" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_cause_generation_check": { + "enabled": true, + "fingerprint": "b43ce8b2f74254027bcc0d382aa0bda21801bd6539f4ea3365679718c5cd80ba" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_cause_kind_check": { + "enabled": true, + "fingerprint": "4d213a62ba48d8d428c2811ddc5039b1349d3a41943509a85062b43235ab669d" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check": { + "enabled": true, + "fingerprint": "bc5fdbb84edccb22f49869598e550f0f72ee45323f0852be3438c8c5d5a387fc" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check1": { + "enabled": true, + "fingerprint": "3d95e12acbdfe40cfd7c36ea0c4cceddc8106395e3165b6c8034dd14783ba596" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check2": { + "enabled": true, + "fingerprint": "abcd49a900cec31d6a71175389de38cf44ffe6d16fa9083e45fe82967313db78" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_check3": { + "enabled": true, + "fingerprint": "66bffc213553425f6a34852e7394536bdd550304c09cdb472a70223bcadb9f7b" + }, + "constraint:reporting_production_notification_events.reporting_production_notification_events_pkey": { + "enabled": true, + "fingerprint": "e8d63f44b25700915bcb3cb69e2e24b98cf21fd304f36a2e9c2992236309309d" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notific_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "543a59e86b26fff09da6144c4685bfd8e2489795644ec4e72668afd29fdcd324" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_exp_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_expansions_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "8ac920e8e1ed5a59417d5d1ffb9bea608bbeb64dbcd1a431240370fe555fe6f0" + }, + "constraint:reporting_production_notification_expansions.reporting_production_notification_expansions_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_production_source_probe_turns.reporting_production_source_p_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_source_probe_turns.reporting_production_source_probe_turns_pkey": { + "enabled": true, + "fingerprint": "d9ba22090ae520014ae689fbeb3206c705853c9227e68c533acd51317287e944" + }, + "constraint:reporting_production_source_probe_turns.reporting_production_source_probe_turns_probe_turn_check": { + "enabled": true, + "fingerprint": "58f1f55f151556fb2aef6678d8418605bef80f3141755502759c72efceaa7a62" + }, + "constraint:reporting_production_source_progress.reporting_production_source__account_id_delivery_config_i_fkey1": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_source_progress.reporting_production_source_progress_acquisition_turn_check": { + "enabled": true, + "fingerprint": "43f2b66f59d2dade9afd34a2fb5182710e40cd49ab12d2fed8b569ce4cdd5b9e" + }, + "constraint:reporting_production_source_progress.reporting_production_source_progress_pkey": { + "enabled": true, + "fingerprint": "d9ba22090ae520014ae689fbeb3206c705853c9227e68c533acd51317287e944" + }, + "constraint:reporting_production_source_work.reporting_production_source_w_account_id_delivery_config_i_fkey": { + "enabled": true, + "fingerprint": "75c62699e078bab9886b32e70b19f33a126b20459e13e71b7a1aab4b4a02e4fb" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_acquisition_turn_check": { + "enabled": true, + "fingerprint": "43f2b66f59d2dade9afd34a2fb5182710e40cd49ab12d2fed8b569ce4cdd5b9e" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_pkey": { + "enabled": true, + "fingerprint": "d355e8dc9dcf7840e1eb3d2f21f27c8423668327fe68c988c3df2f0918a2c454" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_reporting_obligation_id_fkey": { + "enabled": true, + "fingerprint": "52080b2461e9d04a0fc27011d8aa726299ff01b18975d49f00c9644137b78b06" + }, + "constraint:reporting_production_source_work.reporting_production_source_work_state_check": { + "enabled": true, + "fingerprint": "81b58b0df487102bccee7def2dc2f710efaa046347c5f0fd4c746d596459a5a3" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_outco_fkey": { + "enabled": true, + "fingerprint": "e3e10c3f3d574511ee5944b8bade5d927099809b0a45539ce8502caa9ebb05bf" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_repor_fkey": { + "enabled": true, + "fingerprint": "250741cd0a316028a3f4963b83b558e7daaeb0fc65a3aca9c610e7b7bf1c020d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "72945097549e77da2940c9cc5e54db645c12c0d5ef3bed0d4d7e7ba5cc93ffc1" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "460b11d8840751c3e894245e33c24e704407f4b3ca42bbd9e832cb48a0c3b24d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check1": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check2": { + "enabled": true, + "fingerprint": "307105bea1a76e214540a207ab8dddcffb26853845a89236454a6e9ecb0508d3" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check3": { + "enabled": true, + "fingerprint": "65d0dca50d7ca308123252fcc784fb427fd857d8bf3092bd85fc8fff5787d7ec" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check4": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check5": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_check6": { + "enabled": true, + "fingerprint": "9b7f5755a4d4093ab3618e85ff78ea92e931d1895d601ee48f781dcbca07155d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_content_sha256_check": { + "enabled": true, + "fingerprint": "2afe58d90df96e397cb8e6a941e52fcf8e01b7835024dd3b4ba67141a3578505" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_input_check": { + "enabled": true, + "fingerprint": "1403f246fdde17015118d212847ed5d8169df3adc0364598cbd5e82be066e1bf" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_input_check1": { + "enabled": true, + "fingerprint": "fadd88bcfddae6b78d4fc68c90018f5c21ba800b5beb4ddfb2d5805da7703a2b" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_outcome_namespace_check": { + "enabled": true, + "fingerprint": "60e879d7d85bbf368a4f30d08481dd59f5aa1d92eb6943b0e60a8e8e219c2ff2" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "00dd5f9ba4e0d0face7ab27fe5e52ffa9c4101943c95afb8ec1badb401c7010d" + }, + "constraint:reporting_production_status_boundaries.reporting_production_status_boundaries_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_production_status_heads.reporting_production_status_heads_max_sequence_check": { + "enabled": true, + "fingerprint": "ee47acec8d450eed7f99b637596847d21bc449a12317a605fd6b6cb0fad6aa79" + }, + "constraint:reporting_production_status_heads.reporting_production_status_heads_pkey": { + "enabled": true, + "fingerprint": "4d853add149814edf9eadd3f752bf43f1f7164cba2bd079f0349d4d540a7f20c" + }, + "constraint:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_last_attempt_check": { + "enabled": true, + "fingerprint": "5bf9c354d680faec2bf3155f8c9b27323d7893564b30688a039ba19fc8b8d5d3" + }, + "constraint:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "490860c80b1129b15ab44f458a1a967382fc8b073f694ac959bf938282a74af5" + }, + "constraint:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_principal_id_check": { + "enabled": true, + "fingerprint": "bfc70b4b01cc74b9c93c630928d7cde9b0134c3d6614e2cbb326604c865c2cac" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook__account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "37d7e7994745bab9c9bf7313f10cd46b9c94d08e615964b518e0aa3431f6d4f1" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "b8546a68def520db0e6139c898e3e2a76318e05548a3de1b6e568fbf3a75ee63" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "b8ce3a7a8a92fef844563b5e9850186bbf6b150861584017a7c764241b062db1" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_attempt_check": { + "enabled": true, + "fingerprint": "e05925f8a9fe41263902b6b2a72bca02958627e079787e99987820a52cb47d71" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_payload_size_bytes_check": { + "enabled": true, + "fingerprint": "95548ff5519cec8b0ff110ad66f7b77d77e6015ef8c54a52e09cd4d7d80a7118" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "5580d4f19f5ba257093ef71127d72dbba9669ac2db9b2f116f246d7f760522b4" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_response_time_ms_check": { + "enabled": true, + "fingerprint": "6698890e9bba25670f73e3a5a7a7a7f9b664ab2571637eba582f22863c92db4d" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_attempts_status_check": { + "enabled": true, + "fingerprint": "1c821d6bf41e5137a6263eae308d0a8648b7f0c95154724e5430e8c630aec30e" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_completion": { + "enabled": true, + "fingerprint": "adc02762a63dcbf72330ca318884e6236fb2418cb7e1111f861ee74f98b53316" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_identity": { + "enabled": true, + "fingerprint": "1e3f9725baa47d118c5b2efeeafbd5a2429400c1f667188fd14c4f2795114bf8" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_outcome": { + "enabled": true, + "fingerprint": "cc0cc61f029bb3b28629e12a42c6f6ddbfd943156e1b90e5eeb2346aed678e00" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_timestamps": { + "enabled": true, + "fingerprint": "0175d921479ed64020d88874f3b6ac822a9979f3ec8c3780d89bb9e0e3e3556c" + }, + "constraint:reporting_production_webhook_attempts.reporting_production_webhook_url_safe": { + "enabled": true, + "fingerprint": "3dd08bbc446317552434fb91bebd96e2a534395454b436910450f909ce2ab88e" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_consumer_id_attempt_n_fkey": { + "enabled": true, + "fingerprint": "e3a1d24fef8da0fbe5587a7f7887a4cb9adf0133071070a09c950db1474304c8" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_consumer_id_delivery__fkey": { + "enabled": true, + "fingerprint": "9390df97c6b965e9f16f700a4bd4e90392e60f89a15b6a9840d05682cdcc532f" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_consumer_id_external_i_key": { + "enabled": true, + "fingerprint": "78641c1ca086a76ec1a81540664d98811ffb2adb51d5dc27db4d307f428bd8cd" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_fkey": { + "enabled": true, + "fingerprint": "e6d7ea3bd4ffb8c9ad4e4c2307804a624d79764a63069a9a9762f49a2a092bee" + }, + "constraint:reporting_production_work.reporting_production_work_account_id_reporting_obligation__fkey": { + "enabled": true, + "fingerprint": "a0dca2cc4a27c9d4380eadb1513d30fe6383849657fc5e0d0f93c4b4c4ed4c08" + }, + "constraint:reporting_production_work.reporting_production_work_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_production_work.reporting_production_work_attempt_namespace_check": { + "enabled": true, + "fingerprint": "639b8bfe70df1945046d65f693e67c45a4015e0e318356d77a97f5e4b3a7b629" + }, + "constraint:reporting_production_work.reporting_production_work_binding_sha256_check": { + "enabled": true, + "fingerprint": "c9db9bf2eca41364cec2fc19983d43e23fedf6711e5c4deb5f1d80aed317fd2b" + }, + "constraint:reporting_production_work.reporting_production_work_check": { + "enabled": true, + "fingerprint": "a6826ba06594c4200bda46be2704e03cb9d234d36538780211c04c2b2392e61b" + }, + "constraint:reporting_production_work.reporting_production_work_check1": { + "enabled": true, + "fingerprint": "311f8f3d7b44fd93e0b4ed5945b86997d4c6fd3beea28eca693b1e506466676d" + }, + "constraint:reporting_production_work.reporting_production_work_check2": { + "enabled": true, + "fingerprint": "3b94ebc10b0f1c882069a518c6496ff87db17e6b3405aecdb2f9aca7b158c644" + }, + "constraint:reporting_production_work.reporting_production_work_check3": { + "enabled": true, + "fingerprint": "fc30826fbb5c547751d5b430e90d54631bc537ca3af856785d68bb4a350cf9cb" + }, + "constraint:reporting_production_work.reporting_production_work_check4": { + "enabled": true, + "fingerprint": "50525da46b6ed71b60656e35fbba42e4fcbb45719c4d2032c2eccf35b971fafc" + }, + "constraint:reporting_production_work.reporting_production_work_external_id_check": { + "enabled": true, + "fingerprint": "df053aff1b660dec5635dbcc1043c90a9b338c3d871b72d4ed26ec60d465d278" + }, + "constraint:reporting_production_work.reporting_production_work_generation_check": { + "enabled": true, + "fingerprint": "04ce269eb45cc51febf8cbf44b760185b1eb2fd4b1935d6b53ecae063496539e" + }, + "constraint:reporting_production_work.reporting_production_work_pkey": { + "enabled": true, + "fingerprint": "fb2ee6241427fec8a9ae91d2940abe9e20b25f47554bd2d0a7c4046eb7704c41" + }, + "constraint:reporting_production_work.reporting_production_work_reason_check": { + "enabled": true, + "fingerprint": "d2dc21e781a468c735dd8b305b5405e779c85c27657683c19570cb1ba4b94aa7" + }, + "constraint:reporting_production_work.reporting_production_work_state_check": { + "enabled": true, + "fingerprint": "2e00e4b40728462003ac3d7f891ca8354aaeee6e19d808f7809030ff4db7758a" + }, + "constraint:reporting_production_work.reporting_production_work_verification_key_sha256_check": { + "enabled": true, + "fingerprint": "5442191b821cf162c8dfe50da9181b02acbf254eeb477c27b98e69d1fd26134b" + }, + "function:reporting_production_old_reservation_guard()": { + "enabled": true, + "fingerprint": "a46a6daef7fc2a07be432cf5415abf3df043a0a3f6097b7644f1f3eb956a87b7" + }, + "function:reporting_production_retained_guard()": { + "enabled": true, + "fingerprint": "f3a494b6d28a4e5340a3f4707da937a765122016eb68d93572ca05c982bae49b" + }, + "function:reporting_production_source_dirty()": { + "enabled": true, + "fingerprint": "248a91ace6ab30967841ad2cfed33b7554e1b9524ef9ebbcc833d42eccaa956f" + }, + "function:reporting_production_source_progress_guard()": { + "enabled": true, + "fingerprint": "c1a4a8e578deb4bab1f4a277c23b7605e77dd6b327f88c2c3b038d54fe1ee315" + }, + "function:reporting_production_source_work_guard()": { + "enabled": true, + "fingerprint": "34fb03589c1ce3021d94f650b881df24b18789d78b187a9a2e4966b73f4f7bec" + }, + "function:reporting_production_webhook_attempt_guard()": { + "enabled": true, + "fingerprint": "6b996c7e32c7afe0804181f1966812f834aca9653235e7d3ec4ed18590447a63" + }, + "function:reporting_production_webhook_head_guard()": { + "enabled": true, + "fingerprint": "4e448e0df3d56a5b04bc01ca41a4c5afbb8e601a4699d2e6dd0e3922ed2f0bf8" + }, + "function:reporting_production_work_guard()": { + "enabled": true, + "fingerprint": "89e538b8d4aa47ed74d8d789d0d49a7f66b9360bac7455b21708a227eced1d33" + }, + "index:reporting_notification_deliveries.reporting_production_core_deliveries_due": { + "enabled": true, + "fingerprint": "faeefdc68a8158d06454340a165077ae8ae8c3c281afe6644ad60df092615bcc" + }, + "index:reporting_notification_expansions.reporting_production_core_expansions_due": { + "enabled": true, + "fingerprint": "2b408cc42bd605749529aba6df2ce2833ea6c973787b09e32feab68494cdb7e5" + }, + "index:reporting_production_accounts.reporting_production_accounts_pkey": { + "enabled": true, + "fingerprint": "151886856aad940481552b97892b6ec86cb2418479fbffa8fcde29bd3c26968f" + }, + "index:reporting_production_delivery_windows.reporting_production_delivery_windows_pkey": { + "enabled": true, + "fingerprint": "ba988fbea01cd07a302653122095f8ebfb30e0e5498d2c4877c08dafa9e407ca" + }, + "index:reporting_production_destination_bindings.reporting_production_destination_bindings_pkey": { + "enabled": true, + "fingerprint": "24b1631ba99eb166db21d1d6f4804cfcce250ba559d6934fb952bc8d7c512ae1" + }, + "index:reporting_production_generations.reporting_production_generations_pkey": { + "enabled": true, + "fingerprint": "52db7741f2374442bbbcc253d7376bf7af21b4f65e8e01dbd395486b024c1657" + }, + "index:reporting_production_generations.reporting_production_source_generations": { + "enabled": true, + "fingerprint": "df6bb3c5045c17bd05895151a79d292a4f57b06d82d6a9001b124b9490658370" + }, + "index:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "4762eb7ec4d24b1b248ea600d2421d675827fd683569f48515e04794a939578f" + }, + "index:reporting_production_notification_deliveries.reporting_production_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "86594fa7c0fb8e3d56b87ad0d41d555874649ef6e7bb0b9f299b70e9d9fe0f85" + }, + "index:reporting_production_notification_deliveries.reporting_production_notification_deliveries_due": { + "enabled": true, + "fingerprint": "b169ff747e8e54a0f0977b5bbef0f9f2b206edd2aeba44b47c91e963d52bc422" + }, + "index:reporting_production_notification_deliveries.reporting_production_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "61d1e3319019b1c0b5471a8c099e5bdfa0ff46df98e51a89ab4f2f4c2791c208" + }, + "index:reporting_production_notification_deliveries.reporting_production_ready_deliveries_due": { + "enabled": true, + "fingerprint": "0f8de7ab8666254f2c4b74246708c2dd59666ed9ec59fd2d6523116724988ed6" + }, + "index:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "627f488a483bbc20f6adcb9c6f8217efdf665915ff07e9f046cf8d95495230d7" + }, + "index:reporting_production_notification_events.reporting_production_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "ad2bfee21400b1b027e96a27d57a6b72261071e0772a548fd8aa356bf7cfcaca" + }, + "index:reporting_production_notification_events.reporting_production_notification_events_pkey": { + "enabled": true, + "fingerprint": "011f2487b90e534412c03aa5c5b506cb819f6f0268471ae76377cab257a9ec12" + }, + "index:reporting_production_notification_expansions.reporting_production_notification_due": { + "enabled": true, + "fingerprint": "bc56cbd2460e73c3f1d1f17157d1443b4640e1d4e836b453f632b6c54bb863ea" + }, + "index:reporting_production_notification_expansions.reporting_production_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "3f7ade1e36b9b44bdc4c05dddbf220bf121af7317b28febcf1dc5bb5236fa98b" + }, + "index:reporting_production_notification_expansions.reporting_production_ready_expansions_due": { + "enabled": true, + "fingerprint": "85b0655e19a3764f6eb71d8d0963056c844e09f5daae7f64ed821501e1c712b4" + }, + "index:reporting_production_source_probe_turns.reporting_production_source_probe_turns_pkey": { + "enabled": true, + "fingerprint": "9ef04a971b13db799196deb2251b35586a66ce783cc8c0b28c66165544ea9dac" + }, + "index:reporting_production_source_progress.reporting_production_source_progress_pkey": { + "enabled": true, + "fingerprint": "af80661aff369fc8a0a6d226ce94059e7ae95d8823fa5e34d1f3c47e4c3cdfd8" + }, + "index:reporting_production_source_work.reporting_production_source_pending": { + "enabled": true, + "fingerprint": "7e903d0a69de84d0763092ef6974055b4a7ce74884169456f5db72108998c59c" + }, + "index:reporting_production_source_work.reporting_production_source_work_pkey": { + "enabled": true, + "fingerprint": "b2bbf3fefe4e804ed8ea85ad6a0d9dcb5cb9845794b7e9d120e41781f6d693cf" + }, + "index:reporting_production_status_boundaries.reporting_production_status_b_account_id_consumer_id_report_key": { + "enabled": true, + "fingerprint": "31d4a0205b67c703c2329137212a899eebe9701a81f9bc4e6996481a156497da" + }, + "index:reporting_production_status_boundaries.reporting_production_status_bou_account_id_account_sequence_key": { + "enabled": true, + "fingerprint": "d9e0b980715b67bdcafc91339ba799c18815c4e19835796d40343976a5d0e5c1" + }, + "index:reporting_production_status_boundaries.reporting_production_status_boundaries_pkey": { + "enabled": true, + "fingerprint": "f60fc5fec06ce7bced70c766321327a6cbf96449546b0af9f7187971b67f0e7c" + }, + "index:reporting_production_status_heads.reporting_production_status_heads_pkey": { + "enabled": true, + "fingerprint": "df3abee748f8edaa4ddac8806a21c37a3a7b623fcb489f0fb3f581344fee1830" + }, + "index:reporting_production_webhook_attempt_heads.reporting_production_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "da0aa49ff5efb1becacef3f40bde178be69bff369146e22d36f0bdbd9de2853d" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "27fde93ad0125cfe9be4389b0a0fa9c399d172ee5775ed8cdb0f84b0d88d9396" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook_activity_newest": { + "enabled": true, + "fingerprint": "240921000941d702fa20c6457c0381d975c2b748935951653fab1bfc74172ff4" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook_activity_retention": { + "enabled": true, + "fingerprint": "a0b89634c6fdf96ecba7d7f7c6afdf64f61faa96d1c57fc32c6dba89e6561792" + }, + "index:reporting_production_webhook_attempts.reporting_production_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "4579c6dc33d4f2b51a51c12dc2002ce98476d5ff4c075d481a98fd27da9bb998" + }, + "index:reporting_production_work.reporting_production_one_pending": { + "enabled": true, + "fingerprint": "2f959d945d4dfaf5ec73b3407f4d7694470582cb7991a3fa9aa3f4b6877a366f" + }, + "index:reporting_production_work.reporting_production_work_account_id_consumer_id_external_i_key": { + "enabled": true, + "fingerprint": "c5a14351456c141c94f42726b9375e570b74cca97cdc85f4d8426b1139b9fe22" + }, + "index:reporting_production_work.reporting_production_work_due": { + "enabled": true, + "fingerprint": "ef3ca445274e81f60764e4a7130dc16f01c4297ad37a2fce495e249096b6314c" + }, + "index:reporting_production_work.reporting_production_work_pkey": { + "enabled": true, + "fingerprint": "deec29bab9f4b09d6f060b62e705f64b0e9a48e654dffe2597fb3b9b1ba7037f" + }, + "index:reporting_projection_notification_deliveries.reporting_production_status_deliveries_due": { + "enabled": true, + "fingerprint": "b5f0b4ffc23790dbe584e52c95b9fdb353d8926738ba95cf3829e9d89fba4572" + }, + "index:reporting_projection_notification_expansions.reporting_production_status_expansions_due": { + "enabled": true, + "fingerprint": "6f6175f294b31bbf3c2425db8bdfefc68fba8f5aee7d254f5c3eed7cf5fa610f" + }, + "table:reporting_production_accounts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_delivery_windows": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_destination_bindings": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_generations": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_notification_deliveries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_notification_events": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_notification_expansions": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_source_probe_turns": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_source_progress": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_source_work": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_status_boundaries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_status_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_webhook_attempt_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_webhook_attempts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_production_work": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_materializer_work.reporting_production_old_reservation_guard": { + "enabled": true, + "fingerprint": "e5f11febe88890b585f56ea796477656255225842e3abdcc42f158e3a3a10d7c" + }, + "trigger:reporting_obligations.reporting_production_source_obligation": { + "enabled": true, + "fingerprint": "9c3f01405aa80debf4b965494b6f2dff91f74b82f7b2242cb9df1d676821a656" + }, + "trigger:reporting_production_accounts.reporting_production_activation_immutable": { + "enabled": true, + "fingerprint": "f9426c100b6504a95c9bc1920ef446ff125d44b29bcd0ccf624872babd526d64" + }, + "trigger:reporting_production_delivery_windows.reporting_production_delivery_window_immutable": { + "enabled": true, + "fingerprint": "00babf492b6604846ede741a81f3a9cc270c0287df22a487c2de983205aa30b7" + }, + "trigger:reporting_production_destination_bindings.reporting_production_destination_immutable": { + "enabled": true, + "fingerprint": "0bd06a51562e44e8344367738c6a37b45c3b4fd6b9afca8678cacb55a434ed99" + }, + "trigger:reporting_production_generations.reporting_production_generation_immutable": { + "enabled": true, + "fingerprint": "d25848af82ae00800061fc9aaa9f2d64c0088527d228e6be033fff363451840b" + }, + "trigger:reporting_production_notification_events.reporting_production_event_immutable": { + "enabled": true, + "fingerprint": "8c3f2542634b9faca85f1266fbc6310eeb7f561e13572d5857c44a36044d37e1" + }, + "trigger:reporting_production_source_progress.reporting_production_source_progress_guard": { + "enabled": true, + "fingerprint": "5ea39da121c62e94e8d7c9c60844018872b94feab2671d461943f063efcbd90c" + }, + "trigger:reporting_production_source_work.reporting_production_source_work_guard": { + "enabled": true, + "fingerprint": "e03a1bcb6f6226a9bcce2e966227a0f238c0a333dcc53b582c727b1f712aa761" + }, + "trigger:reporting_production_status_boundaries.reporting_production_boundary_immutable": { + "enabled": true, + "fingerprint": "b744d5024657cf26ee4dd93e0e7ed4d2b1180c70479ffeefbf2ac2b8f7dcf15e" + }, + "trigger:reporting_production_webhook_attempt_heads.reporting_production_webhook_head_guard": { + "enabled": true, + "fingerprint": "42a7f06e3a7162412afe91173067f5bf85f99d7dac0ccc4d0035ddec97355599" + }, + "trigger:reporting_production_webhook_attempts.reporting_production_webhook_attempt_guard": { + "enabled": true, + "fingerprint": "0ef9aa027bdcd90909ed611dba2a25b353cbadf1d7b880309b25abba09102f53" + }, + "trigger:reporting_production_work.reporting_production_guard": { + "enabled": true, + "fingerprint": "71ae8906dce02251a41e4924e9b2160772d9526bbde775f1fd881808a9422832" + }, + "trigger:reporting_revisions.reporting_production_source_revision": { + "enabled": true, + "fingerprint": "d0e846596c4a33cf4ba19de7688bdcf21a2de3771af81d8e0098cbe2468d3a2d" + } +} diff --git a/src/adcp/reporting/production/schema.py b/src/adcp/reporting/production/schema.py new file mode 100644 index 000000000..b3cf06712 --- /dev/null +++ b/src/adcp/reporting/production/schema.py @@ -0,0 +1,24 @@ +"""Separate production participants; no inherited mandatory manifest is widened.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.projection.schema import validate_projection_schema + + +async def validate_production_schema(connection: Any, *, notifications: bool = False) -> None: + try: + required = json.loads( + files("adcp.reporting.production").joinpath("required_schema.json").read_text() + ) + actual = await schema_objects(connection) + if not required or any(actual.get(k) != v for k, v in required.items()): + raise ValueError + await validate_projection_schema(connection, notifications=notifications) + except Exception: + raise ReportingNotificationError("reporting_production_schema_unready") from None diff --git a/src/adcp/reporting/production/service.py b/src/adcp/reporting/production/service.py new file mode 100644 index 000000000..99ce6aba7 --- /dev/null +++ b/src/adcp/reporting/production/service.py @@ -0,0 +1,689 @@ +"""One live SDK composition for admitted reporting work and truthful discovery.""" + +from __future__ import annotations + +import asyncio +import hashlib +import json +import weakref +from collections.abc import Mapping +from contextvars import ContextVar +from dataclasses import dataclass +from datetime import timedelta +from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable + +from adcp.reporting.ledger.delivery_models import ReportingDestinationBinding +from adcp.reporting.ledger.models import ReportingConfiguration +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.materializer.contracts import ( + ReportingDestinationResolver, + ReportingDestinationWriter, + ReportingVerificationKey, + failure, +) +from adcp.reporting.materializer.reference import ( + ReferenceReportingDestinationWriter, + ReferenceReportingResolver, +) +from adcp.reporting.materializer.service import ReportingMaterializerService +from adcp.reporting.materializer.verification import ReportingDestinationIO +from adcp.reporting.materializer.work import MaterializerContext, verification_key_id +from adcp.reporting.production.configuration import ( + ReportingConfigurationAdmission, + ReportingProductionConfigurationTask, +) +from adcp.reporting.production.contracts import ( + ReportingProductionDestinationBinding, + ReportingProductionMethod, + ReportingProductionSourceBinding, +) +from adcp.reporting.production.memory import InMemoryReportingProductionStore +from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.projection.memory import InMemoryReportingStatusProjection +from adcp.reporting.receipts.handler import ReceiptAccountResolver + +if TYPE_CHECKING: + from adcp.decisioning.registry import BuyerAgentRegistry + from adcp.reporting.ledger.producer import ReportingProducer + from adcp.reporting.outbox.worker import ReportingNotificationWorker + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.projection.pg import PgReportingStatusProjection + from adcp.server.base import ADCPHandler + + +@runtime_checkable +class ReportingProductionDestination( + ReportingDestinationWriter, ReportingDestinationResolver, Protocol +): + """An actual provider's bounded retention and revocation commitments. + + These promises accompany the writer's exact capabilities. Every write and + independent readback still opens a separately authorized SDK session. A + service cannot make the development writer eligible by decorating it. + """ + + @property + def resource_retention_days(self) -> int: ... + + @property + def authorization_revocation_seconds(self) -> int: ... + + @property + def delivery_methods(self) -> tuple[ReportingProductionMethod, ...]: ... + + def configuration_binding( + self, binding: ReportingDestinationBinding + ) -> ReportingProductionDestinationBinding | None: ... + + +def _method_ready( + writer: ReportingDestinationWriter, offering: ReportingProductionOffering +) -> bool: + return ( + isinstance(writer, ReportingProductionDestination) + and offering.verification_key.capability in writer.capabilities + and any( + type(method) is ReportingProductionMethod + and method.capability == offering.verification_key.capability + and method.wire() == offering.wire()["method"] + for method in writer.delivery_methods + ) + ) + + +def production_owner( + store: InMemoryReportingProductionStore | PgReportingProductionStore, +) -> ReportingProductionSupport: + reference = store._production_support + owner = reference() if reference is not None else None + if owner is None or owner.store is not store: + raise ReportingNotificationError("reporting_production_component_unready") + owner._assert_components() + return owner + + +@dataclass(frozen=True) +class _MountedProduction: + mount: weakref.ReferenceType[Any] + dispatcher: weakref.ReferenceType[Any] + transport: str + entries: dict[str, tuple[int, int]] + + def current(self) -> dict[str, tuple[int, int]]: + dispatcher = self.dispatcher() + if self.mount() is None or dispatcher is None: + return {} + try: + if self.transport == "mcp": + return { + name: (id(tool), id(tool.fn)) + for name, tool in dispatcher._tool_manager._tools.items() + } + if self.transport == "a2a": + return {name: (id(fn), id(fn)) for name, fn in dispatcher._tool_callers.items()} + except (AttributeError, TypeError): + return {} + return {} + + +def register_production_mount( + handler: ADCPHandler[Any], mount: object, *, transport: str, dispatcher: object +) -> None: + """Called only after the SDK has installed the transport's real dispatch map.""" + from adcp.reporting.production.handler import ReportingProductionHandler + + if type(handler) is ReportingProductionHandler: + proof = _MountedProduction(weakref.ref(mount), weakref.ref(dispatcher), transport, {}) + proof.entries.update(proof.current()) + handler.production._mounts.append(proof) + + +class ReportingProductionSupport: + """Compose, mount, start, then activate accounts after draining old workers. + + The owned worker performs indexed producer/materializer/projector turns. + Stopping it withdraws production claims and prevents fresh admission. An + optional HTTP notification worker is independent of polling readiness; + enabled logical enqueue is always part of the materializer transaction. + + Migrate with ``store.create_schema()`` before construction/start. For DDL, + drain and close this support, migrate, and construct fresh support. Schema + proofs do not detect arbitrary serving-time DDL or search-path changes. + """ + + def __init__( + self, + materializer: ReportingMaterializerService, + projection: InMemoryReportingStatusProjection | PgReportingStatusProjection, + *, + offerings: tuple[ReportingProductionOffering, ...], + configuration_task: ReportingProductionConfigurationTask, + resolve_account: ReceiptAccountResolver, + buyer_agents: BuyerAgentRegistry | None = None, + automated_recovery_window: timedelta = timedelta(hours=6), + status_retention_days: int = 400, + notification_workers: tuple[ReportingNotificationWorker, ...] = (), + poll_seconds: float = 0.25, + ) -> None: + from adcp.reporting.outbox.worker import ReportingNotificationWorker + from adcp.reporting.production.handler import ReportingProductionHandler + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.projection.pg import PgReportingStatusProjection + + if ( + type(materializer) is not ReportingMaterializerService + or type(materializer.store) + not in {InMemoryReportingProductionStore, PgReportingProductionStore} + or type(projection) + not in {InMemoryReportingStatusProjection, PgReportingStatusProjection} + or projection.ledger is not materializer.store + or type(configuration_task) is not ReportingProductionConfigurationTask + or type(offerings) is not tuple + or not offerings + or any(type(o) is not ReportingProductionOffering for o in offerings) + or len({o.offering_id for o in offerings}) != len(offerings) + or type(status_retention_days) is not int + or not 1 <= status_retention_days <= 36500 + or automated_recovery_window <= timedelta(0) + or not 0.01 <= poll_seconds <= 60 + or type(notification_workers) is not tuple + or any(type(w) is not ReportingNotificationWorker for w in notification_workers) + ): + raise ValueError( + "production support requires exact SDK components and bounded promises" + ) + assert isinstance( + materializer.store, (InMemoryReportingProductionStore, PgReportingProductionStore) + ) + self.store = materializer.store + previous = self.store._production_support + if previous is not None and previous() is not None: + raise ReportingNotificationError("reporting_production_owner_conflict") + self.materializer, self.projection = materializer, projection + self.offerings, self.configuration_task = offerings, configuration_task + self.automated_recovery_window, self.status_retention_days = ( + automated_recovery_window, + status_retention_days, + ) + self.notification_workers, self.poll_seconds = notification_workers, poll_seconds + from adcp.reporting.production.notifications import worker_identity + + self._notification_identity = tuple(worker_identity(w) for w in notification_workers) + self.handler = ReportingProductionHandler( + self, resolve_account=resolve_account, buyer_agents=buyer_agents + ) + self._mounts: list[_MountedProduction] = [] + self._task: asyncio.Task[None] | None = None + self._notification_task: asyncio.Task[None] | None = None + self._stop = asyncio.Event() + self._started = asyncio.Event() + self._notification_started = asyncio.Event() + self._closed = False + self._failed = False + self._producer_turn: ContextVar[ReportingProducer | None] = ContextVar( + "reporting_production_source_turn", default=None + ) + self._schema_task: asyncio.Task[bool] | None = None + self._schema_epoch = 0 + self._schema_positive: tuple[int, int] | None = None + self._components = self._component_ids() + self._methods = self._handler_methods() + self.store._production_support = weakref.ref(self) + self._assert_components(running=False, mounted=False) + + @property + def keys(self) -> tuple[ReportingVerificationKey, ...]: + return tuple(dict.fromkeys(o.verification_key for o in self.offerings)) + + @property + def notifications_enabled(self) -> bool: + return bool(self.projection.policy["notifications_enabled"]) + + def _component_ids(self) -> tuple[int, ...]: + return tuple( + id(v) + for v in ( + self.store, + getattr(self.store, "_pool", None), + self.materializer, + self.materializer.io, + self.materializer.io.registry, + self.materializer.io.resolver, + self.materializer.writer, + self.projection, + self.projection.outbox, + self.configuration_task, + *self.offerings, + *self.notification_workers, + ) + ) + + def _handler_methods(self) -> tuple[object, ...]: + return tuple( + getattr(getattr(self.handler, name), "__func__", None) + for name in ( + "get_reporting_status", + "get_media_buy_delivery", + "sync_reporting_receipts", + "sync_reporting_status", + "sync_accounts", + "get_adcp_capabilities", + ) + ) + + def _mounted(self, *, receipts: bool = False) -> bool: + required = { + "get_adcp_capabilities", + "get_reporting_status", + "get_media_buy_delivery", + "sync_accounts", + } + if receipts: + required.add("sync_reporting_receipts") + if self.projection.consumer_status_enabled: + required.add("sync_reporting_status") + live = [proof for proof in self._mounts if proof.mount() is not None] + return bool(live) and all( + required <= proof.entries.keys() + and all(proof.current().get(name) == proof.entries[name] for name in required) + for proof in live + ) + + def _assert_components(self, *, running: bool = True, mounted: bool = True) -> None: + from adcp.reporting.production.notifications import check_workers + + check_workers(self) + writer = self.materializer.writer + if ( + self._closed + or self._failed + or (running and (self._task is None or self._task.done())) + or ( + running + and self.notification_workers + and (self._notification_task is None or self._notification_task.done()) + ) + or self._components != self._component_ids() + or self._methods != self._handler_methods() + or ( + (running or mounted) + and not isinstance(self.store, InMemoryReportingProductionStore) + and self.store._pool.closed is not False + ) + or self.materializer.store is not self.store + or self.projection.ledger is not self.store + or not self._projection_outbox_linked() + or self.handler.receipt_store is not self.store + or self.handler.reporting_feed_store is not self.store + or self.handler._feed_consumer_status_enabled != self.projection.consumer_status_enabled + or self.store._projection_read_policy != self.projection.policy + or type(self.materializer.io) is not ReportingDestinationIO + or isinstance(writer, ReferenceReportingDestinationWriter) + or isinstance(self.materializer.io.resolver, ReferenceReportingResolver) + or not isinstance(writer, ReportingProductionDestination) + or writer is not self.materializer.io.resolver + or writer.production_eligible is not True + or type(writer.resource_retention_days) is not int + or not 1 <= writer.resource_retention_days <= 36500 + or type(writer.authorization_revocation_seconds) is not int + or not 0 <= writer.authorization_revocation_seconds <= 86400 + or (mounted and not self._mounted()) + ): + raise ReportingNotificationError("reporting_production_component_unready") + + def _projection_outbox_linked(self) -> bool: + # The projection queue participates even when optional HTTP delivery + # workers are absent. Its connection ownership cannot ride a cached + # catalog proof belonging to a different pool or in-memory ledger. + if isinstance(self.store, InMemoryReportingProductionStore): + from adcp.reporting.projection.memory import InMemoryReportingProjectionOutbox + + return ( + type(self.projection.outbox) is InMemoryReportingProjectionOutbox + and self.projection.outbox._store is self.store + ) + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + + return ( + type(self.projection.outbox) is PgReportingProjectionOutbox + and self.projection.outbox._pool is self.store._pool + ) + + def _offering_ready(self, offering: ReportingProductionOffering) -> bool: + try: + if ( + offering.producer.store is not self.store + or type(offering.producer._max_periods_per_turn) is not int + or not 1 <= offering.producer._max_periods_per_turn <= 64 + or offering.producer.escalation != self.projection.escalation + or not _method_ready(self.materializer.writer, offering) + or (offering.reconciled and not self._mounted(receipts=True)) + ): + return False + verifier = self.materializer.io.registry.require(offering.verification_key) + profile = offering.wire()["reporting_profile"] + contract = json.loads(verifier.canonicalization_bytes) + definition = json.loads(verifier.definition_bytes) + if ( + offering.producer._revision_verifier is not verifier + or profile["primary_keys"] != contract["primary_keys"] + or profile["grain"] != definition.get("grain") + ): + return False + offering.check_source() + return True + except Exception: + return False + + def _admission_policy(self) -> dict[str, Any]: + self._assert_components() + # This fixes the installed contract, not its current authorization. + # Each reservation separately proves its own offering; an unrelated + # provider outage must not rewrite this epoch or veto healthy offerings. + ready = self.offerings + return { + "version": 2, + "verification_keys": sorted({verification_key_id(o.verification_key) for o in ready}), + "offerings": sorted(hashlib.sha256(o._wire).hexdigest() for o in ready), + "producers": sorted(o._producer_key for o in ready), + "notifications_enabled": self.notifications_enabled, + "projection": self.projection.policy, + } + + def _configuration_offering( + self, + configuration: ReportingConfiguration, + binding: ReportingDestinationBinding, + *, + offering_id: str | None = None, + producer_key: str | None = None, + ) -> ReportingProductionOffering: + self._assert_components() + writer = self.materializer.writer + assert isinstance(writer, ReportingProductionDestination) + matches = [] + for offering in self.offerings: + if (offering_id is not None and offering.offering_id != offering_id) or ( + producer_key is not None and offering._producer_key != producer_key + ): + continue + if not self._offering_ready(offering): + continue + try: + offering.check_configuration( + configuration, binding, retention_days=writer.resource_retention_days + ) + self._destination_binding(binding, offering) + # A failed proof never qualifies this offering. + except Exception: # nosec B112 + continue + matches.append(offering) + if ( + len(matches) != 1 + or configuration.automated_recovery_window != self.automated_recovery_window + or configuration.status_retention_days < self.status_retention_days + ): + raise failure("BINDING_MISMATCH") + return matches[0] + + def _destination_binding( + self, binding: ReportingDestinationBinding, offering: ReportingProductionOffering + ) -> ReportingProductionDestinationBinding: + try: + writer = self.materializer.writer + assert isinstance(writer, ReportingProductionDestination) + resolved = writer.configuration_binding(binding) + if ( + type(resolved) is not ReportingProductionDestinationBinding + or resolved.binding != binding + or resolved.method.capability != offering.verification_key.capability + or resolved.method.wire() != offering.wire()["method"] + ): + raise failure("BINDING_MISMATCH") + return resolved + except Exception: + raise failure("BINDING_MISMATCH") from None + + def _producer_keys(self) -> tuple[str, ...]: + self._assert_components() + producer = self._producer_turn.get() + return tuple( + o._producer_key + for o in self.offerings + if o.producer is producer and self._offering_ready(o) + ) + + def validate_configuration(self, value: ReportingConfigurationAdmission) -> None: + value.check(self) + + def _source_binding( + self, configuration: ReportingConfiguration, producer_key: str + ) -> ReportingProductionSourceBinding: + self._assert_components() + matches = [ + offering + for offering in self.offerings + if offering._producer_key == producer_key and self._offering_ready(offering) + ] + if len(matches) != 1: + raise failure("BINDING_MISMATCH") + return matches[0].source_binding(configuration) + + def _check_source_binding( + self, + configuration: ReportingConfiguration, + producer_key: str, + document: dict[str, Any] | None, + ) -> ReportingProductionSourceBinding: + binding = self._source_binding(configuration, producer_key) + if binding.document() != document: + raise failure("BINDING_MISMATCH") + return binding + + def _check_context( + self, + context: MaterializerContext, + key: ReportingVerificationKey, + policy: dict[str, Any], + producer_key: str | None, + source_binding: dict[str, Any] | None, + destination_binding: dict[str, Any] | None, + ) -> None: + if producer_key is None: + raise failure("BINDING_MISMATCH") + offering = self._configuration_offering( + context.configuration, context.binding, producer_key=producer_key + ) + self._check_source_binding(context.configuration, producer_key, source_binding) + if self._destination_binding(context.binding, offering).wire() != destination_binding: + raise failure("BINDING_MISMATCH") + if ( + policy != self._admission_policy() + or offering.verification_key != key + or verification_key_id(key) not in policy["verification_keys"] + or context.obligation.generation_key != context.configuration.generation_key + ): + raise failure("BINDING_MISMATCH") + + def invalidate_schema_validation(self) -> None: + """Drain first; an older scan cannot publish a new epoch's proof.""" + self._schema_epoch += 1 + self._schema_positive = None + self._schema_task = None + + async def _schema_ready(self) -> bool: + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.production.schema import validate_production_schema + + if not isinstance(self.store, PgReportingProductionStore): + return False + store = self.store + pool, epoch = store._pool, self._schema_epoch + identity = (id(pool), epoch) + if self._schema_positive == identity: + return True + task = self._schema_task + if task is None: + + async def scan() -> bool: + try: + async with pool.connection() as connection: + await validate_production_schema( + connection, notifications=self.notifications_enabled + ) + if epoch == self._schema_epoch and pool is store._pool: + self._schema_positive = identity + return True + return False + finally: + if self._schema_task is asyncio.current_task(): + self._schema_task = None + + task = asyncio.create_task(scan()) + self._schema_task = task + task.add_done_callback(lambda done: None if done.cancelled() else done.exception()) + if task.get_loop() is not asyncio.get_running_loop(): + return False + return await asyncio.shield(task) + + async def start(self) -> None: + if self._task is not None: + self._assert_components() + return + self._assert_components(running=False) + if ( + not isinstance(self.store, InMemoryReportingProductionStore) + and not await self._schema_ready() + ): + raise ReportingNotificationError("reporting_production_schema_unready") + self._assert_components(running=False) + self._task = asyncio.create_task(self._run(), name="adcp-reporting-production") + if self.notification_workers: + self._notification_task = asyncio.create_task( + self._run_notifications(), name="adcp-reporting-production-notifications" + ) + await self._started.wait() + if self.notification_workers: + await self._notification_started.wait() + self._assert_components() + + async def _run(self) -> None: + try: + while not self._stop.is_set(): + # Each producer leases a generation through the reviewed fair + # indexed acquisition path; no account enumeration is required. + for producer in dict.fromkeys(o.producer for o in self.offerings): + token = self._producer_turn.set(producer) + try: + await producer.run_worker() + finally: + self._producer_turn.reset(token) + await self.materializer.run_once() + await self.projection.rebuild_one() + await self.projection.sweep_one() + self._started.set() + try: + await asyncio.wait_for(self._stop.wait(), self.poll_seconds) + except asyncio.TimeoutError: + pass + except asyncio.CancelledError: + raise + except Exception: + # Retain only the closed lifecycle state, never provider bodies. + self._failed = True + finally: + self._started.set() + + async def _run_notifications(self) -> None: + from adcp.reporting.production.notifications import notification_turn + + try: + while not self._stop.is_set(): + await notification_turn(self) + self._notification_started.set() + try: + await asyncio.wait_for(self._stop.wait(), self.poll_seconds) + except asyncio.TimeoutError: + pass + except asyncio.CancelledError: + raise + except Exception: + self._failed = True + self._stop.set() + finally: + self._notification_started.set() + + async def _check_notifications(self, account_id: str) -> None: + from adcp.reporting.production.notifications import check_account_notifications + + self._assert_components() + try: + await check_account_notifications(self, account_id) + except Exception: + raise ReportingNotificationError("notification_chain_unready") from None + self._assert_components() + + async def activate(self, *, account_id: str) -> bool: + await self._check_notifications(account_id) + await self.projection.activate(account_id=account_id) + return await self.store._activate_production(account_id=account_id) + + async def aclose(self) -> None: + self._stop.set() + task = self._task + if task is not None: + # Drain short database turns instead of interrupting psycopg's + # transaction entry/exit. Long I/O already has SDK-owned deadlines. + await asyncio.gather(task, return_exceptions=True) + if self._notification_task is not None: + await asyncio.gather(self._notification_task, return_exceptions=True) + self._notification_task = None + self._task = None + self._closed = True + proof = self._schema_task + if proof is not None: + await asyncio.gather(proof, return_exceptions=True) + if self.store._production_support is not None and self.store._production_support() is self: + self.store._production_support = None + + async def reporting_delivery(self, *, extra: Mapping[str, Any] | None = None) -> dict[str, Any]: + # Validate protected extra even when the concrete surface is unready. + producer = self.offerings[0].producer + payload = producer.advertised_reporting_delivery( + consumer_status_task=self.projection.consumer_status_enabled, + offerings=[], + automated_recovery_window=self.automated_recovery_window, + status_retention_days=self.status_retention_days, + extra=extra, + ) + try: + if self._stop.is_set(): + return {} + self._assert_components() + if not await self._schema_ready(): + return {} + # Catalog proof is cached; live component/mount ownership is not. + # It must still hold after an awaited first scan or invalidation. + self._assert_components() + offerings = [o for o in self.offerings if self._offering_ready(o)] + if not offerings: + return {} + writer = self.materializer.writer + assert isinstance(writer, ReportingProductionDestination) + payload.update( + offerings=[o.wire() for o in offerings], + managed_delivery=True, + resource_retention_days=writer.resource_retention_days, + authorization_revocation_seconds=writer.authorization_revocation_seconds, + ) + if any(o.reconciled for o in offerings): + payload.update(reconciled_billing=True, receipt_task="sync_reporting_receipts") + if self.notification_workers: + payload.update( + ledger_notification="reporting.ledger_changed", + readiness_notification="reporting.delivery_ready", + status_notification="reporting.status_changed", + ) + return payload + except Exception: + return {} diff --git a/src/adcp/reporting/projection/__init__.py b/src/adcp/reporting/projection/__init__.py new file mode 100644 index 000000000..22823c9b0 --- /dev/null +++ b/src/adcp/reporting/projection/__init__.py @@ -0,0 +1,36 @@ +"""Versioned private reporting projection, independent of optional delivery workers.""" + +from typing import TYPE_CHECKING, Any + +from adcp.reporting.projection.capture import ReportingProjectionInput +from adcp.reporting.projection.memory import ( + InMemoryReportingProjectionOutbox, + InMemoryReportingProjectionStore, + InMemoryReportingStatusProjection, +) + +if TYPE_CHECKING: + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + from adcp.reporting.projection.pg import PgReportingProjectionStore, PgReportingStatusProjection + +__all__ = [ + "InMemoryReportingProjectionOutbox", + "InMemoryReportingProjectionStore", + "InMemoryReportingStatusProjection", + "PgReportingProjectionOutbox", + "PgReportingProjectionStore", + "PgReportingStatusProjection", + "ReportingProjectionInput", +] + + +def __getattr__(name: str) -> Any: + if name in {"PgReportingProjectionStore", "PgReportingStatusProjection"}: + from adcp.reporting.projection import pg + + return getattr(pg, name) + if name == "PgReportingProjectionOutbox": + from adcp.reporting.projection.notifications import PgReportingProjectionOutbox + + return PgReportingProjectionOutbox + raise AttributeError(name) diff --git a/src/adcp/reporting/projection/capture.py b/src/adcp/reporting/projection/capture.py new file mode 100644 index 000000000..3d6978317 --- /dev/null +++ b/src/adcp/reporting/projection/capture.py @@ -0,0 +1,191 @@ +"""Closed immutable v2 inputs preserve legacy C/B2.1/B2.2 document shapes.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass, field, fields, is_dataclass, replace +from datetime import datetime, timedelta, timezone +from typing import Any +from zoneinfo import ZoneInfo + +from pydantic import TypeAdapter, ValidationError +from pydantic_core import TzInfo + +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.evidence import aware_utc +from adcp.reporting.ledger._delivery_state import decode_record, payload, principal, record_identity +from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.ledger.status_snapshot import snapshot_from_storage + +_CORE = TypeAdapter(ReportingStatusSnapshot) +_COLLECTIONS = ( + "configurations", + "obligations", + "revisions", + "statuses", + "lifecycles", + "issue_scopes", + "adjustments", + "changes", + "reconciliation", +) + + +def _require_frozen(value: Any) -> None: + if value is None or type(value) in {str, bytes, int, bool, float, timedelta}: + return + if type(value) is datetime and type(value.tzinfo) in {timezone, ZoneInfo, TzInfo}: + # Datetime itself is immutable; a user-defined tzinfo need not be. + # These closed decoder timezone types cannot mutate shared history. + return + if type(value) is tuple: + for item in value: + _require_frozen(item) + return + if is_dataclass(value) and getattr(type(value), "__dataclass_params__").frozen: + for item in fields(value): + _require_frozen(getattr(value, item.name)) + return + raise ReportingNotificationError("status_projection_history_corrupt") + + +@dataclass(frozen=True) +class ReportingProjectionInput: + core: ReportingStatusSnapshot = field(repr=False) + reconciliation: tuple[ReportingDeliveryRecord, ...] = field(repr=False) + document: bytes = field(repr=False) + + def __post_init__(self) -> None: + # Structural validation also protects direct internal construction: + # frozen outer dataclasses alone do not make nested lists/dicts safe. + _require_frozen(self.core) + _require_frozen(self.reconciliation) + if type(self.document) is not bytes: + raise ReportingNotificationError("status_projection_history_corrupt") + + def __deepcopy__(self, memo: dict[int, Any]) -> ReportingProjectionInput: + # Decoding closes every collection into tuples of frozen records. A + # rollback copies the mutable account cursor, queues and input list; + # the immutable historical values can safely remain shared. Copying + # every earlier full-account snapshot on each new source mutation made + # bounded production catch-up grow cubically in retained history. + memo[id(self)] = self + return self + + def at(self, at: datetime) -> ReportingProjectionInput: + """Evaluate a deadline on the same captured history, never today's records.""" + return replace(self, core=replace(self.core, as_of=aware_utc(at))) + + +def capture_memory_input( + core: ReportingStatusSnapshot, + changes: tuple[tuple[int, Any, ReportingDeliveryRecord], ...], +) -> ReportingProjectionInput: + raw = _CORE.dump_python(core, mode="json") + entries = [ + {"consumer_id": who.consumer_id, "sequence": seq, "record": payload(record)} + for seq, who, record in changes + if who.account_id == core.account_id + ] + document = { + "version": 2, + "account_id": core.account_id, + "as_of": core.as_of.isoformat(), + "core_format": "typed-v1", + "core": raw, + "reconciliation": entries, + "counts": {k: len(entries) if k == "reconciliation" else len(raw[k]) for k in _COLLECTIONS}, + } + return decode_projection_input(document) + + +def decode_projection_input(value: Any) -> ReportingProjectionInput: + try: + if ( + type(value) is not dict + or set(value) + != {"version", "account_id", "as_of", "core_format", "core", "reconciliation", "counts"} + or type(value["version"]) is not int + or value["version"] != 2 + or value["core_format"] not in {"typed-v1", "sql-v1"} + or type(value["core"]) is not dict + or type(value["reconciliation"]) is not list + or type(value["counts"]) is not dict + or set(value["counts"]) != set(_COLLECTIONS) + ): + raise ValueError + # Closure counts are checked before interpreting any financial evidence. + for name in _COLLECTIONS: + rows = value["reconciliation"] if name == "reconciliation" else value["core"][name] + count = value["counts"][name] + if type(rows) is not list or type(count) is not int or count != len(rows): + raise ValueError + core = ( + _CORE.validate_python(value["core"]) + if value["core_format"] == "typed-v1" + else snapshot_from_storage(value["core"]) + ) + if core.account_id != value["account_id"] or core.as_of != aware_timestamp(value["as_of"]): + raise ValueError + positions: dict[str, int] = {} + identities: set[tuple[str, tuple[str, str]]] = set() + records = [] + for row in value["reconciliation"]: + if type(row) is not dict or set(row) != {"consumer_id", "sequence", "record"}: + raise ValueError + record = decode_record(row["record"]) + who = principal(record) + if who.account_id != core.account_id or who.consumer_id != row["consumer_id"]: + raise ValueError + sequence = row["sequence"] + if type(sequence) is not int or sequence != positions.get(who.consumer_id, 0) + 1: + raise ValueError + positions[who.consumer_id] = sequence + identity = who.consumer_id, record_identity(record) + if identity in identities: + raise ValueError + identities.add(identity) + records.append(record) + if not set(positions).issubset(core.consumer_ids): + raise ValueError + for name in ( + "configurations", + "obligations", + "revisions", + "statuses", + "lifecycles", + "adjustments", + ): + if any(row.account_id != core.account_id for row in getattr(core, name)): + raise ValueError + return ReportingProjectionInput(core, tuple(records), canonical_json_utf8_v1(value)) + except (ValueError, TypeError, KeyError, AttributeError, ValidationError): + raise ReportingNotificationError("status_projection_history_corrupt") from None + + +def source_identity(value: ReportingProjectionInput) -> bytes: + """Memory transaction change detection excludes only the observation clock.""" + raw = json.loads(value.document) + raw.pop("as_of") + raw["core"].pop("as_of") + return canonical_json_utf8_v1(raw) + + +def with_projection_core( + value: ReportingProjectionInput, core: ReportingStatusSnapshot +) -> ReportingProjectionInput: + """Runtime replay state is separate from each immutable source boundary.""" + raw = json.loads(value.document) + raw.update( + core_format="typed-v1", + core=_CORE.dump_python(core, mode="json"), + as_of=core.as_of.isoformat(), + ) + raw["counts"] = { + k: len(raw["reconciliation"]) if k == "reconciliation" else len(raw["core"][k]) + for k in _COLLECTIONS + } + return decode_projection_input(raw) diff --git a/src/adcp/reporting/projection/history.py b/src/adcp/reporting/projection/history.py new file mode 100644 index 000000000..095f19109 --- /dev/null +++ b/src/adcp/reporting/projection/history.py @@ -0,0 +1,119 @@ +"""Replay retained private boundaries without creating active notifications. + +The old materializer and receipt captures are complete for their own caller. +They are never joined to today's configuration, artifacts or other consumers. +Their derived checkpoints use the same pure projector and generation primitive +as live projection. The original readiness queues and external identities are +not read, copied or modified here. +""" + +from __future__ import annotations + +import hashlib +from collections.abc import Mapping +from dataclasses import dataclass +from typing import Any, Literal + +from pydantic import TypeAdapter + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.notification_models import ( + ReportingNotificationError, + event_storage, +) +from adcp.reporting.ledger.status_projection import ( + StatusProjectionInput, + project_status_scope, + projection_scopes, +) +from adcp.reporting.materializer.capture import ( + ReportingMaterializerBoundary, + decode_materializer_boundary, +) +from adcp.reporting.outbox.status import StatusCheckpoint, advance_checkpoint, settled_replay +from adcp.reporting.receipts.capture import ReportingReceiptBoundary, decode_receipt_boundary + +HistoricalBoundary = ReportingMaterializerBoundary | ReportingReceiptBoundary +HistoricalKind = Literal["materializer", "receipt"] +_CHECKPOINT = TypeAdapter(StatusCheckpoint) + + +def decode_boundary(kind: HistoricalKind, value: dict[str, Any]) -> HistoricalBoundary: + if kind == "materializer": + return decode_materializer_boundary(value) + if kind == "receipt": + return decode_receipt_boundary(value) + raise ReportingNotificationError("status_projection_history_corrupt") + + +def checkpoint_key(checkpoint: StatusCheckpoint) -> str: + return hashlib.sha256(canonical_json_utf8_v1(checkpoint.scope.checkpoint_key)).hexdigest() + + +def checkpoint_document(checkpoint: StatusCheckpoint) -> dict[str, Any]: + result: dict[str, Any] = _CHECKPOINT.dump_python(checkpoint, mode="json") + return result + + +def decode_checkpoint(document: dict[str, Any]) -> StatusCheckpoint: + value = _CHECKPOINT.validate_python(document) + if checkpoint_document(value) != document: + raise ReportingNotificationError("status_projection_history_corrupt") + return value + + +@dataclass(frozen=True) +class HistoricalStep: + checkpoint: StatusCheckpoint + event: dict[str, Any] | None + + def document(self) -> dict[str, Any]: + return { + "admission_epoch": 0, + "checkpoint": checkpoint_document(self.checkpoint), + "event": self.event, + } + + +def project_boundary( + boundary: HistoricalBoundary, + previous: Mapping[str, StatusCheckpoint], + *, + baselines: Mapping[str, StatusCheckpoint], + source_sequence: int, + escalation: ReportingDeliveryEscalation | None, + consumer_status_enabled: bool, +) -> tuple[HistoricalStep, ...]: + core = settled_replay(boundary.core) + scopes = { + s.checkpoint_key: s + for s in projection_scopes(core) + if s.consumer_id == boundary.caller.consumer_id + } + scopes.update( + (c.scope.checkpoint_key, c.scope) + for c in (*baselines.values(), *previous.values()) + if c.scope.account_id == boundary.caller.account_id + and c.scope.consumer_id == boundary.caller.consumer_id + ) + results = [] + for scope in sorted(scopes.values(), key=lambda s: s.checkpoint_key): + result = project_status_scope( + StatusProjectionInput( + core, + scope, + escalation, + reconciliation=boundary.reconciliation, + consumer_status_enabled=consumer_status_enabled, + ) + ) + key = hashlib.sha256(canonical_json_utf8_v1(scope.checkpoint_key)).hexdigest() + checkpoint, event = advance_checkpoint( + previous.get(key, baselines.get(key)), + result, + fired_at=boundary.as_of, + source_sequence=source_sequence, + ) + results.append(HistoricalStep(checkpoint, event_storage(event) if event else None)) + return tuple(results) diff --git a/src/adcp/reporting/projection/memory.py b/src/adcp/reporting/projection/memory.py new file mode 100644 index 000000000..258b04734 --- /dev/null +++ b/src/adcp/reporting/projection/memory.py @@ -0,0 +1,411 @@ +"""Memory conformance participant with the same unconditional rollback boundary.""" + +from __future__ import annotations + +import asyncio +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from contextvars import ContextVar +from copy import deepcopy +from dataclasses import dataclass, field, replace +from typing import Any + +from adcp.reporting.feed.memory import InMemoryReportingFeedStore +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.status_projection import with_replay_lifecycles +from adcp.reporting.ledger.status_snapshot import memory_snapshot +from adcp.reporting.ledger.store import _MEMORY_TRANSACTION +from adcp.reporting.outbox.memory import NotificationState +from adcp.reporting.outbox.status import ( + StatusCheckpoint, + StatusDueLease, + StatusTurn, + escalation_identity, + settled_replay, +) +from adcp.reporting.outbox.status_memory import ( + InMemoryReportingStatusOutbox, + InMemoryStatusNotificationStore, + _StatusMemoryState, +) +from adcp.reporting.projection.capture import ( + ReportingProjectionInput, + capture_memory_input, + source_identity, + with_projection_core, +) +from adcp.reporting.projection.history import ( + HistoricalBoundary, + checkpoint_key, + project_boundary, +) + +_REPLAY: ContextVar[object | None] = ContextVar("reporting_projection_replay", default=None) + + +@dataclass +class _ProjectionAccount: + policy: dict[str, Any] + checkpoint_floor: int + current: ReportingProjectionInput + source: bytes + cursor: int = 0 + inputs: list[ReportingProjectionInput] = field(default_factory=list) + ready: bool = False + legacy: tuple[HistoricalBoundary, ...] = () + legacy_cursor: int = 0 + baselines: dict[str, StatusCheckpoint] = field(default_factory=dict) + historical_checkpoints: dict[str, StatusCheckpoint] = field(default_factory=dict) + historical_steps: list[tuple[int, dict[str, Any]]] = field(default_factory=list) + + +class InMemoryReportingProjectionOutbox(InMemoryReportingStatusOutbox): + _store: InMemoryReportingProjectionStore + + @property + def _state(self) -> NotificationState: + state = self._store._projection_outbox + if state is None: + raise ReportingNotificationError("notifications_disabled") + return state + + +class InMemoryReportingProjectionStore(InMemoryReportingFeedStore): + """Reference semantics, never a production durability claim.""" + + _projection_read_policy: dict[str, Any] + _projection_read_escalation: ReportingDeliveryEscalation | None + + def __init__(self, **kwargs: Any) -> None: + super().__init__(**kwargs) + self._projection_accounts: dict[str, _ProjectionAccount] = {} + self._projection_outbox = ( + NotificationState() if self._notification_state is not None else None + ) + + def _capture_projection(self, account_id: str) -> ReportingProjectionInput: + return capture_memory_input( + memory_snapshot(self, account_id), tuple(getattr(self, "_delivery_records", ())) + ) + + @asynccontextmanager + async def _mutation(self) -> AsyncIterator[None]: + nested = _MEMORY_TRANSACTION.get() == (id(self), asyncio.current_task()) + async with super()._mutation(): + yield + if not nested and _REPLAY.get() != id(self): + for account_id, state in self._projection_accounts.items(): + captured = self._capture_projection(account_id) + identity = source_identity(captured) + if identity != state.source: + state.inputs.append(captured) + state.source = identity + + def _feed_projection_options(self, caller: ReportingDeliveryPrincipal) -> dict[str, Any]: + state = self._projection_accounts.get(caller.account_id) + if state is None or not state.ready: + return {} + return { + "representation_version": 2, + "revision_ownership": state.policy["ownership_enabled"], + "activated_consumer_status_enabled": state.policy["consumer_status_enabled"], + } + + async def read_projection_input( + self, *, caller: ReportingDeliveryPrincipal + ) -> ReportingProjectionInput: + async with self._lock: + state = self._projection_accounts.get(caller.account_id) + if state is None or not state.ready: + raise ReportingNotificationError("status_projection_activation_required") + return state.inputs[-1].at(self._clock()) + + async def read_tier_status( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: + from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler + from adcp.reporting.projection.wire import render_tier_status + + async with self._lock: + state = self._projection_accounts.get(caller.account_id) + value = state.inputs[-1].at(self._clock()) if state and state.ready else None + policy = state.policy if state and state.ready else None + if value is None or policy is None: + return await ReportingStatusHandler( + self, consumer_status_enabled=consumer_status_enabled + ).handle(request, caller=ReportingStatusCaller(caller.account_id, caller.consumer_id)) + return render_tier_status(self, request, value, caller, policy, consumer_status_enabled) + + +class InMemoryReportingStatusProjection(InMemoryStatusNotificationStore): + _projection_version = 2 + ledger: InMemoryReportingProjectionStore + + def __init__( + self, + ledger: InMemoryReportingProjectionStore, + *, + consumer_status_enabled: bool = False, + revision_ownership: bool = False, + escalation: ReportingDeliveryEscalation | None = None, + ) -> None: + if not isinstance(ledger, InMemoryReportingProjectionStore): + raise ReportingNotificationError("status_projection_component_unready") + if type(consumer_status_enabled) is not bool or type(revision_ownership) is not bool: + raise ValueError("projection feature selections must be booleans") + self.ledger, self.escalation = ledger, escalation + self.consumer_status_enabled, self.revision_ownership = ( + consumer_status_enabled, + revision_ownership, + ) + if ledger._status_notification_state is None: + ledger._status_notification_state = _StatusMemoryState() + self.outbox = InMemoryReportingProjectionOutbox(ledger) + ledger._projection_read_policy = self.policy + ledger._projection_read_escalation = escalation + + def _enqueue_status(self, event: ReportingDomainEvent) -> None: + self.outbox._state.enqueue(event) + + async def checkpoints(self, *, account_id: str) -> tuple[StatusCheckpoint, ...]: + # A checkpoint contains a mutable JSON projection. Never lend that + # dictionary to a reader or retain it in a caller's published result. + return deepcopy(await super().checkpoints(account_id=account_id)) + + @property + def policy(self) -> dict[str, Any]: + return { + "version": 2, + "escalation": escalation_identity(self.escalation), + "consumer_status_enabled": self.consumer_status_enabled, + "ownership_enabled": self.revision_ownership, + "notifications_enabled": self.ledger._notification_state is not None, + } + + @asynccontextmanager + async def _transaction(self) -> AsyncIterator[None]: + token = _REPLAY.set(id(self.ledger)) + try: + async with self.ledger._mutation(): + yield + finally: + _REPLAY.reset(token) + + def _account(self, account_id: str) -> _ProjectionAccount: + state = self.ledger._projection_accounts.get(account_id) + if state is None: + raise ReportingNotificationError("status_projection_activation_required") + if state.policy != self.policy: + raise ReportingNotificationError("status_projection_policy_conflict") + return state + + def _cursor(self, account_id: str) -> int: + self._account(account_id) + return super()._cursor(account_id) + + async def baseline_ready(self, *, account_id: str) -> bool: + async with self.ledger._lock: + if account_id not in self.ledger._projection_accounts: + return False + return self._account(account_id).ready + + def _apply_value( + self, value: ReportingProjectionInput, *, through: int, silent: bool = False + ) -> int: + return self._apply( + settled_replay(value.core), + through=through, + reconciliation=value.reconciliation, + consumer_status_enabled=self.consumer_status_enabled, + enqueue=self.ledger._notification_state is not None and not silent, + ) + + async def activate(self, *, account_id: str) -> bool: + started = await self._begin_activation(account_id=account_id) + while True: + async with self._transaction(): + state = self._account(account_id) + if state.ready: + return started + self._activation_step(account_id) + + async def _begin_activation(self, *, account_id: str) -> bool: + async with self._transaction(): + if account_id in self.ledger._projection_accounts: + self._account(account_id) + return False + old = self._state.accounts.get(account_id) + notifications = self.ledger._notification_state + through = ( + max( + (d.sequence for d in notifications.dirty if d.scope.account_id == account_id), + default=0, + ) + if notifications + else 0 + ) + if old is not None: + if old[1] != escalation_identity(self.escalation): + raise ReportingNotificationError("status_policy_conflict") + if old[0] != through or self._needs_rebuild(account_id): + raise ReportingNotificationError("status_projection_legacy_drain_required") + checkpoints = [ + c for c in self._state.checkpoints.values() if c.scope.account_id == account_id + ] + now = self.ledger._clock() + if any( + (c.lease_expires_at and c.lease_expires_at > now) + or (c.next_due_at and c.next_due_at <= now) + for c in checkpoints + ): + raise ReportingNotificationError("status_projection_legacy_drain_required") + value = self.ledger._capture_projection(account_id) + floor = max((c.source_sequence for c in checkpoints), default=0) + legacy = tuple( + sorted( + ( + b + for b in ( + *self.ledger._materializer_boundaries, + *getattr(self.ledger, "_receipt_boundaries", ()), + ) + if b.caller.account_id == account_id + ), + key=lambda b: b.account_sequence, + ) + ) + expected = self.ledger._materializer_account_heads.get(account_id, 0) + if len(legacy) != expected or any( + b.account_sequence != n for n, b in enumerate(legacy, 1) + ): + raise ReportingNotificationError("status_projection_history_corrupt") + self.ledger._projection_accounts[account_id] = _ProjectionAccount( + self.policy, + floor + expected, + value, + source_identity(value), + inputs=[value], + legacy=legacy, + baselines={checkpoint_key(c): c for c in checkpoints}, + ) + self._state.accounts[account_id] = (through, escalation_identity(self.escalation)) + self._state.selector_accounts[account_id] = "complete" + return True + + def _activation_step(self, account_id: str) -> StatusTurn: + state = self._account(account_id) + if state.legacy_cursor < len(state.legacy): + boundary = state.legacy[state.legacy_cursor] + for step in project_boundary( + boundary, + state.historical_checkpoints, + baselines=state.baselines, + source_sequence=state.checkpoint_floor + - len(state.legacy) + + boundary.account_sequence, + escalation=self.escalation, + consumer_status_enabled=self.consumer_status_enabled, + ): + state.historical_checkpoints[checkpoint_key(step.checkpoint)] = step.checkpoint + state.historical_steps.append((boundary.account_sequence, step.document())) + self._state.checkpoints[step.checkpoint.scope.checkpoint_key] = step.checkpoint + state.legacy_cursor += 1 + return StatusTurn(True) + self._apply_value(state.inputs[0], through=state.checkpoint_floor + 1, silent=True) + state.cursor, state.ready = 1, True + return StatusTurn(True) + + async def baseline(self, *, account_id: str) -> bool: + return await self.activate(account_id=account_id) + + def _project_version(self, account_id: str) -> StatusTurn: + state = self._account(account_id) + if not state.ready: + return self._activation_step(account_id) + following = state.inputs[state.cursor] if state.cursor < len(state.inputs) else None + due = min( + ( + c.next_due_at + for c in self._state.checkpoints.values() + if c.scope.account_id == account_id and c.next_due_at is not None + ), + default=None, + ) + cursor = state.cursor + if ( + due is not None + and due <= self.ledger._clock() + and (following is None or due < following.core.as_of) + ): + value = state.current.at(due) + elif following is not None: + value, cursor = following, cursor + 1 + else: + return StatusTurn(False) + if value.core.as_of < state.current.core.as_of: + raise ReportingNotificationError("status_projection_clock_regressed") + value = with_projection_core( + value, settled_replay(with_replay_lifecycles(value.core, state.current.core)) + ) + count = self._apply_value(value, through=state.checkpoint_floor + cursor) + state.current, state.cursor = value, cursor + return StatusTurn(True, count) + + async def project_one(self, *, account_id: str) -> StatusTurn: + async with self._transaction(): + return self._project_version(account_id) + + async def rebuild_one(self) -> StatusTurn: + async with self._transaction(): + for account_id, state in sorted(self.ledger._projection_accounts.items()): + if state.cursor < len(state.inputs) and state.policy == self.policy: + return self._project_version(account_id) + return StatusTurn(False) + + async def complete_due(self, lease: StatusDueLease) -> StatusTurn: + async with self._transaction(): + self._account(lease.scope.account_id) + checkpoint = self._state.checkpoints.get(lease.scope.checkpoint_key) + if ( + checkpoint is None + or checkpoint.lease_token != lease.token + or checkpoint.lease_expires_at != lease.expires_at + or lease.expires_at <= self.ledger._clock() + ): + return StatusTurn(False) + result = self._project_version(lease.scope.account_id) + current = self._state.checkpoints[lease.scope.checkpoint_key] + if current.lease_token != lease.token or lease.expires_at <= self.ledger._clock(): + raise ReportingNotificationError("status_lease_lost") + self._state.checkpoints[lease.scope.checkpoint_key] = replace( + current, + lease_token=None, + lease_expires_at=None, + ) + return result + + async def sweep_one(self) -> StatusTurn: + async with self.ledger._lock: + at = self.ledger._clock() + due = sorted( + (c.next_due_at, c.scope.account_id) + for c in self._state.checkpoints.values() + if c.next_due_at is not None + and c.next_due_at <= at + and c.scope.account_id in self.ledger._projection_accounts + and self.ledger._projection_accounts[c.scope.account_id].policy == self.policy + and (c.lease_expires_at is None or c.lease_expires_at <= at) + ) + if not due: + return StatusTurn(False) + lease = await self.claim_due(account_id=due[0][1]) + return await self.complete_due(lease) if lease is not None else StatusTurn(False) diff --git a/src/adcp/reporting/projection/notifications.py b/src/adcp/reporting/projection/notifications.py new file mode 100644 index 000000000..8349cd2a8 --- /dev/null +++ b/src/adcp/reporting/projection/notifications.py @@ -0,0 +1,41 @@ +"""Isolated v2 status queues using the original SDK fanout/delivery transactions.""" + +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Any + +from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox + + +class _ProjectionQueueConnection: + """Closed identifiers only; shares the caller's actual connection/transaction.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + def transaction(self) -> Any: + return self.connection.transaction() + + async def execute(self, query: str, params: Any = None) -> Any: + for old, new in ( + ("reporting_status_notification_", "reporting_projection_notification_"), + ("reporting_status_webhook_", "reporting_projection_webhook_"), + ("reporting_notification_", "reporting_projection_notification_"), + ("reporting_webhook_", "reporting_projection_webhook_"), + ): + query = query.replace(old, new) + return await self.connection.execute(query, params) + + +class PgReportingProjectionOutbox(PgReportingStatusOutbox): + @asynccontextmanager + async def _connection(self) -> AsyncIterator[Any]: + async with self._pool.connection() as connection: + yield _ProjectionQueueConnection(connection) + + async def create_schema(self) -> None: + from adcp.reporting.projection.pg import PgReportingProjectionStore + + await PgReportingProjectionStore(pool=self._pool, notifications=True).create_schema() diff --git a/src/adcp/reporting/projection/pg.py b/src/adcp/reporting/projection/pg.py new file mode 100644 index 000000000..b24e723eb --- /dev/null +++ b/src/adcp/reporting/projection/pg.py @@ -0,0 +1,632 @@ +"""Connection-bound v2 capture over the reviewed feed and status participants.""" + +from __future__ import annotations + +import asyncio +import json +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import replace +from importlib.resources import files +from typing import Any + +from adcp.reporting.feed.pg import PgReportingFeedStore, _CapturedFeed, _PreparedFeed +from adcp.reporting.feed.snapshot import StoredFeedSnapshot +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.notification_models import ( + ReportingDomainEvent, + ReportingNotificationError, +) +from adcp.reporting.ledger.status_projection import with_replay_lifecycles +from adcp.reporting.ledger.status_snapshot import persist_replay_lifecycles_on +from adcp.reporting.outbox.pg import database_now +from adcp.reporting.outbox.status import ( + StatusDueLease, + StatusTurn, + escalation_identity, + settled_replay, +) +from adcp.reporting.outbox.status_pg import _CHECKPOINT as _LEGACY_CHECKPOINT +from adcp.reporting.outbox.status_pg import PgStatusNotificationStore, _enqueue_on +from adcp.reporting.outbox.status_pg import _checkpoint as _legacy_checkpoint +from adcp.reporting.projection.capture import ( + ReportingProjectionInput, + decode_projection_input, + with_projection_core, +) +from adcp.reporting.projection.history import ( + checkpoint_document, + checkpoint_key, + decode_boundary, + decode_checkpoint, + project_boundary, +) +from adcp.reporting.projection.notifications import ( + PgReportingProjectionOutbox, + _ProjectionQueueConnection, +) +from adcp.reporting.projection.schema import validate_projection_schema + + +class _ProjectionFeedConnection: + """Closed SDK table selection; no adopter-supplied SQL identifiers.""" + + def __init__(self, connection: Any) -> None: + self.connection = connection + + async def execute(self, query: str, params: Any = None) -> Any: + return await self.connection.execute( + query.replace("reporting_feed_snapshots", "reporting_projection_feed_snapshots"), params + ) + + +class PgReportingProjectionStore(PgReportingFeedStore): + """Optional v2 store. Installation alone neither activates tiers nor sends readiness.""" + + _projection_read_policy: dict[str, Any] + _projection_read_escalation: ReportingDeliveryEscalation | None + + async def create_schema(self) -> None: + async with self._connection() as connection, connection.transaction(): + await self._create_schema_on(connection) + root = files("adcp.reporting.ledger") + for name in ( + "reporting_materializer.sql", + "reporting_receipt_ingestion.sql", + "reporting_feed.sql", + "reporting_status_notifications.sql", + "reporting_status_selector_version.sql", + "reporting_projection.sql", + "reporting_projection_notifications.sql", + "reporting_projection_feed.sql", + ): + await connection.execute(root.joinpath(name).read_text()) + + async def _feed_snapshot_on( + self, connection: Any, snapshot_id: str, caller: ReportingDeliveryPrincipal + ) -> StoredFeedSnapshot | None: + legacy = await super()._feed_snapshot_on(connection, snapshot_id, caller) + if legacy is not None: + return legacy + return await super()._feed_snapshot_on( + _ProjectionFeedConnection(connection), snapshot_id, caller + ) + + async def _save_feed_snapshot_on(self, connection: Any, stored: _PreparedFeed) -> None: + if stored.snapshot.representation_version == 2: + connection = _ProjectionFeedConnection(connection) + await super()._save_feed_snapshot_on(connection, stored) + + async def _capture_feed_on( + self, connection: Any, caller: ReportingDeliveryPrincipal + ) -> _CapturedFeed: + captured = await super()._capture_feed_on(connection, caller) + row = await ( + await connection.execute( + "SELECT ownership_enabled,consumer_status_enabled" + " FROM reporting_projection_accounts WHERE account_id=%s" + " AND current_input IS NOT NULL", + (caller.account_id,), + ) + ).fetchone() + if row is None: + return captured + await validate_projection_schema(connection, notifications=self._notifications_enabled) + return replace( + captured, + representation_version=2, + revision_ownership=row[0], + activated_consumer_status_enabled=row[1], + ) + + async def read_projection_input( + self, *, caller: ReportingDeliveryPrincipal + ) -> ReportingProjectionInput: + async with self._connection() as connection, connection.transaction(): + await self._lock_account(connection, caller.account_id) + await validate_projection_schema(connection, notifications=self._notifications_enabled) + row = await ( + await connection.execute( + "SELECT input,content_sha256=reporting_receipt_ingestion_sha256(input)" + " FROM reporting_projection_inputs WHERE account_id=%s" + " AND EXISTS (SELECT 1 FROM reporting_projection_accounts a" + " WHERE a.account_id=reporting_projection_inputs.account_id" + " AND a.current_input IS NOT NULL)" + " ORDER BY sequence DESC LIMIT 1", + (caller.account_id,), + ) + ).fetchone() + if row is None: + raise ReportingNotificationError("status_projection_activation_required") + if not row[1]: + raise ReportingNotificationError("status_projection_history_corrupt") + value = decode_projection_input(row[0]) + return value.at(await database_now(connection, self._clock)) + + async def read_tier_status( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: + from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler + from adcp.reporting.projection.wire import render_tier_status + + async with self._connection() as connection: + row = await ( + await connection.execute( + "SELECT policy FROM reporting_projection_accounts WHERE account_id=%s" + " AND current_input IS NOT NULL", + (caller.account_id,), + ) + ).fetchone() + if row is None: + return await ReportingStatusHandler( + self, consumer_status_enabled=consumer_status_enabled + ).handle(request, caller=ReportingStatusCaller(caller.account_id, caller.consumer_id)) + value = await self.read_projection_input(caller=caller) + return await asyncio.to_thread( + render_tier_status, self, request, value, caller, row[0], consumer_status_enabled + ) + + +class PgReportingStatusProjection(PgStatusNotificationStore): + """v2 input/cursor lifecycle reusing C's pure projector, checkpoint and event transaction.""" + + _projection_version = 2 + ledger: PgReportingProjectionStore + + def __init__( + self, + ledger: PgReportingProjectionStore, + *, + consumer_status_enabled: bool = False, + revision_ownership: bool = False, + escalation: ReportingDeliveryEscalation | None = None, + ) -> None: + if not isinstance(ledger, PgReportingProjectionStore): + raise ReportingNotificationError("status_projection_component_unready") + if type(consumer_status_enabled) is not bool or type(revision_ownership) is not bool: + raise ValueError("projection feature selections must be booleans") + self.ledger, self.escalation = ledger, escalation + self.consumer_status_enabled, self.revision_ownership = ( + consumer_status_enabled, + revision_ownership, + ) + self.outbox = PgReportingProjectionOutbox(pool=ledger._pool, clock=ledger._clock) + ledger._projection_read_policy = self.policy + ledger._projection_read_escalation = escalation + + async def _enqueue_status_on(self, connection: Any, event: ReportingDomainEvent) -> None: + await _enqueue_on(_ProjectionQueueConnection(connection), event) + + @property + def policy(self) -> dict[str, Any]: + return { + "version": 2, + "escalation": escalation_identity(self.escalation), + "consumer_status_enabled": self.consumer_status_enabled, + "ownership_enabled": self.revision_ownership, + "notifications_enabled": self.ledger._notifications_enabled, + } + + async def create_schema(self) -> None: + await self.ledger.create_schema() + + @asynccontextmanager + async def _transaction(self, account_id: str) -> AsyncIterator[Any]: + async with self.ledger._connection() as connection, connection.transaction(): + await connection.execute( + "SELECT set_config('adcp.reporting.projection_version','2',true)" + ) + await connection.execute( + "SELECT set_config('adcp.reporting.selector_semantics_version','2',true)" + ) + await connection.execute( + "SELECT set_config('adcp.reporting.projection_internal','on',true)" + ) + await self.ledger._lock_account(connection, account_id) + if self.ledger._clock is not None: + await connection.execute( + "SELECT set_config('adcp.reporting.projection_clock',%s,true)", + (self.ledger._clock().isoformat(),), + ) + yield connection + + async def _state_on(self, connection: Any, account_id: str) -> tuple[Any, ...]: + row = await ( + await connection.execute( + "SELECT policy,cursor,max_sequence,checkpoint_floor,current_input,current_as_of" + " FROM reporting_projection_accounts WHERE account_id=%s FOR UPDATE", + (account_id,), + ) + ).fetchone() + if row is None: + raise ReportingNotificationError("status_projection_activation_required") + if row[0] != self.policy: + raise ReportingNotificationError("status_projection_policy_conflict") + return tuple(row) + + async def _account_on(self, connection: Any, account_id: str) -> int: + await self._state_on(connection, account_id) + return await super()._account_on(connection, account_id) + + async def baseline_ready(self, *, account_id: str) -> bool: + async with self._transaction(account_id) as connection: + await validate_projection_schema( + connection, notifications=self.ledger._notifications_enabled + ) + row = await ( + await connection.execute( + "SELECT policy,current_input IS NOT NULL FROM reporting_projection_accounts" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + if row is None: + return False + if row[0] != self.policy: + raise ReportingNotificationError("status_projection_policy_conflict") + return bool(row[1]) + + async def activate(self, *, account_id: str) -> bool: + """Fence, replay captured private history, then activate the frozen baseline. + + Each retained input advances in its own transaction. Cancellation or a + process crash resumes at that exact cursor. The derived historical + events stay in an immutable epoch-zero journal, never a delivery queue. + Old C projectors/sweepers must first be stopped and drained. + """ + started = await self._begin_activation(account_id=account_id) + while True: + async with self._transaction(account_id) as connection: + state = await self._state_on(connection, account_id) + if state[4] is not None: + return started + await self._activation_step_on(connection, account_id) + + async def _begin_activation(self, *, account_id: str) -> bool: + async with self._transaction(account_id) as connection: + await validate_projection_schema( + connection, notifications=self.ledger._notifications_enabled + ) + existing = await ( + await connection.execute( + "SELECT policy FROM reporting_projection_accounts WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + if existing is not None: + await self._state_on(connection, account_id) + return False + now = await database_now(connection, self.ledger._clock) + old = await ( + await connection.execute( + "SELECT dirty_sequence,baseline_complete,policy FROM reporting_status_accounts" + " WHERE account_id=%s FOR UPDATE", + (account_id,), + ) + ).fetchone() + dirty = await ( + await connection.execute( + "SELECT coalesce(max_sequence,0) FROM reporting_status_dirty_heads" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + through = int(dirty[0]) if dirty else 0 + if old is not None and old[1]: + if old[2] != escalation_identity(self.escalation): + raise ReportingNotificationError("status_policy_conflict") + if int(old[0]) != through or await self._needs_rebuild_on(connection, account_id): + raise ReportingNotificationError("status_projection_legacy_drain_required") + rows = await ( + await connection.execute( + "SELECT source_sequence,lease_expires_at,next_due_at FROM" + " reporting_status_scope_checkpoints" + " WHERE account_id=%s ORDER BY" + " consumer_namespace,delivery_config_id,version,scope_kind," + " obligation_namespace FOR UPDATE", + (account_id,), + ) + ).fetchall() + if any( + (r[1] is not None and r[1] > now) or (r[2] is not None and r[2] <= now) + for r in rows + ): + raise ReportingNotificationError("status_projection_legacy_drain_required") + floor = max((int(r[0]) for r in rows), default=0) + # The inherited SDK column list is fixed; the account is bound. + old_checkpoints = [ + _legacy_checkpoint(r) + for r in await ( + await connection.execute( + f"SELECT {_LEGACY_CHECKPOINT} FROM reporting_status_scope_checkpoints" # nosec B608 + " WHERE account_id=%s", + (account_id,), + ) + ).fetchall() + ] + generation_floor = max((c.generation for c in old_checkpoints if c), default=0) + legacy_head = await ( + await connection.execute( + "SELECT captured_sequence FROM reporting_materializer_accounts" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + legacy_through = int(legacy_head[0]) if legacy_head else 0 + await connection.execute( + "INSERT INTO" + " reporting_status_accounts(account_id,policy,baseline_complete,dirty_sequence," + "baseline_highwater,baseline_at,selector_target_version,selector_transition)" + " VALUES(%s,%s::jsonb,TRUE,%s,%s,%s,2,'complete')" + " ON CONFLICT(account_id) DO NOTHING", + ( + account_id, + json.dumps(escalation_identity(self.escalation)), + through, + through, + now, + ), + ) + await connection.execute( + "INSERT INTO" + " reporting_projection_accounts(account_id,activated_at,notifications_enabled," + "consumer_status_enabled,ownership_enabled,policy,legacy_through,checkpoint_floor," + "legacy_capture_through,legacy_generation_floor)" + " VALUES(%s,%s,%s,%s,%s,%s::jsonb,%s,%s,%s,%s)", + ( + account_id, + now, + self.ledger._notifications_enabled, + self.consumer_status_enabled, + self.revision_ownership, + json.dumps(self.policy), + through, + floor + legacy_through, + legacy_through, + generation_floor, + ), + ) + for checkpoint in old_checkpoints: + if checkpoint is None: + continue + document = json.dumps(checkpoint_document(checkpoint)) + await connection.execute( + "INSERT INTO reporting_projection_legacy_baselines VALUES" + " (%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))", + (account_id, checkpoint_key(checkpoint), document, document), + ) + await connection.execute( + "UPDATE reporting_status_scope_checkpoints SET projection_writer_floor=2," + "lease_token=NULL,lease_expires_at=NULL WHERE account_id=%s", + (account_id,), + ) + await connection.execute("SELECT reporting_projection_capture(%s)", (account_id,)) + return True + + async def _activation_step_on(self, connection: Any, account_id: str) -> StatusTurn: + metadata = await ( + await connection.execute( + "SELECT legacy_capture_through,legacy_capture_cursor," + "checkpoint_floor FROM reporting_projection_accounts WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + through, cursor, floor = map(int, metadata) + if cursor < through: + row = await ( + await connection.execute( + "SELECT kind,input,content_sha256=reporting_receipt_ingestion_sha256(input)," + " count(*) OVER (PARTITION BY account_sequence)" + " FROM (SELECT 'materializer' AS kind,account_sequence,input,content_sha256" + " FROM reporting_materializer_status_boundaries WHERE account_id=%s" + " AND account_sequence>%s AND account_sequence<=%s UNION ALL" + " SELECT 'receipt',account_sequence,input,content_sha256" + " FROM reporting_receipt_ingestion_boundaries WHERE account_id=%s" + " AND account_sequence>%s AND account_sequence<=%s) b" + " ORDER BY account_sequence LIMIT 1", + (account_id, cursor, through, account_id, cursor, through), + ) + ).fetchone() + if row is None or not row[2] or row[3] != 1: + raise ReportingNotificationError("status_projection_history_corrupt") + boundary = decode_boundary(row[0], row[1]) + if boundary.account_sequence != cursor + 1 or boundary.caller.account_id != account_id: + raise ReportingNotificationError("status_projection_history_corrupt") + previous_rows = await ( + await connection.execute( + "SELECT scope_key,checkpoint FROM reporting_projection_legacy_checkpoints" + " WHERE account_id=%s AND consumer_id=%s ORDER BY scope_key", + (account_id, boundary.caller.consumer_id), + ) + ).fetchall() + previous = {r[0]: decode_checkpoint(r[1]) for r in previous_rows} + baseline_rows = await ( + await connection.execute( + "SELECT scope_key,checkpoint FROM reporting_projection_legacy_baselines" + " WHERE account_id=%s ORDER BY scope_key", + (account_id,), + ) + ).fetchall() + baselines = {r[0]: decode_checkpoint(r[1]) for r in baseline_rows} + steps = project_boundary( + boundary, + previous, + baselines=baselines, + source_sequence=floor - through + boundary.account_sequence, + escalation=self.escalation, + consumer_status_enabled=self.consumer_status_enabled, + ) + raw = json.dumps(row[1]) + await connection.execute( + "INSERT INTO reporting_projection_legacy_inputs VALUES" + " (%s,%s,%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))", + ( + account_id, + boundary.account_sequence, + boundary.caller.consumer_id, + row[0], + raw, + raw, + ), + ) + await self._lock_scopes_on(connection, boundary.core) + for step in steps: + key = checkpoint_key(step.checkpoint) + document = json.dumps(step.document()) + await connection.execute( + "INSERT INTO reporting_projection_legacy_steps VALUES" + " (%s,%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))", + (account_id, boundary.account_sequence, key, document, document), + ) + checkpoint = json.dumps(checkpoint_document(step.checkpoint)) + await connection.execute( + "INSERT INTO reporting_projection_legacy_checkpoints VALUES" + " (%s,%s,%s,%s::jsonb,reporting_receipt_ingestion_sha256(%s::jsonb))" + " ON CONFLICT(account_id,scope_key) DO UPDATE SET" + " checkpoint=EXCLUDED.checkpoint,content_sha256=EXCLUDED.content_sha256", + (account_id, boundary.caller.consumer_id, key, checkpoint, checkpoint), + ) + # Advance the original guarded checkpoint once per retained + # boundary. Only the immutable epoch-zero journal receives the + # event; activation never releases it into an active queue. + await self._write_on(connection, step.checkpoint) + await connection.execute( + "UPDATE reporting_projection_accounts SET legacy_capture_cursor=%s" + " WHERE account_id=%s", + (boundary.account_sequence, account_id), + ) + return StatusTurn(True) + row = await ( + await connection.execute( + "SELECT input FROM reporting_projection_inputs WHERE account_id=%s AND sequence=1", + (account_id,), + ) + ).fetchone() + value = decode_projection_input(row[0]) + await self._lock_scopes_on(connection, value.core) + await self._apply_value_on(connection, value, through=floor + 1, silent=True) + await connection.execute( + "UPDATE reporting_projection_accounts SET" + " cursor=1,current_input=%s::jsonb,current_as_of=%s WHERE account_id=%s", + (value.document.decode(), value.core.as_of, account_id), + ) + return StatusTurn(True) + + async def baseline(self, *, account_id: str) -> bool: + return await self.activate(account_id=account_id) + + async def _apply_value_on( + self, + connection: Any, + value: ReportingProjectionInput, + *, + through: int, + silent: bool = False, + ) -> int: + return await self._apply_on( + connection, + settled_replay(value.core), + through=through, + reconciliation=value.reconciliation, + consumer_status_enabled=self.consumer_status_enabled, + enqueue=self.ledger._notifications_enabled and not silent, + ) + + async def _project_version_on(self, connection: Any, account_id: str) -> StatusTurn: + state = await self._state_on(connection, account_id) + if state[4] is None: + return await self._activation_step_on(connection, account_id) + cursor, floor = int(state[1]), int(state[3]) + row = await ( + await connection.execute( + "SELECT sequence,input,content_sha256=reporting_receipt_ingestion_sha256(input)" + " FROM reporting_projection_inputs WHERE account_id=%s AND sequence>%s" + " ORDER BY sequence LIMIT 1", + (account_id, cursor), + ) + ).fetchone() + next_value = None + if row is not None: + if row[0] != cursor + 1 or not row[2]: + raise ReportingNotificationError("status_projection_history_corrupt") + next_value = decode_projection_input(row[1]) + now = await database_now(connection, self.ledger._clock) + deadline = await ( + await connection.execute( + "SELECT min(next_due_at) FROM reporting_status_scope_checkpoints" + " WHERE account_id=%s", + (account_id,), + ) + ).fetchone() + due = deadline[0] if deadline else None + if due is not None and due <= now and (next_value is None or due < next_value.core.as_of): + value = decode_projection_input(state[4]).at(due) + if due < state[5]: + raise ReportingNotificationError("status_projection_clock_regressed") + elif next_value is not None: + value, cursor = next_value, int(row[0]) + if value.core.as_of < state[5]: + raise ReportingNotificationError("status_projection_clock_regressed") + else: + return StatusTurn(False) + prior = decode_projection_input(state[4]) + value = with_projection_core( + value, settled_replay(with_replay_lifecycles(value.core, prior.core)) + ) + await persist_replay_lifecycles_on(self.ledger, connection, value.core) + count = await self._apply_value_on(connection, value, through=floor + cursor) + await connection.execute( + "UPDATE reporting_projection_accounts SET" + " cursor=%s,current_input=%s::jsonb,current_as_of=%s" + " WHERE account_id=%s", + (cursor, value.document.decode(), value.core.as_of, account_id), + ) + return StatusTurn(True, count) + + async def project_one(self, *, account_id: str) -> StatusTurn: + async with self._transaction(account_id) as connection: + return await self._project_version_on(connection, account_id) + + async def rebuild_one(self) -> StatusTurn: + async with self.ledger._connection() as connection: + row = await ( + await connection.execute( + "SELECT account_id FROM reporting_projection_accounts WHERE cursor StatusTurn: + async with self._transaction(lease.scope.account_id) as connection: + await self._state_on(connection, lease.scope.account_id) + if not await self._held_on(connection, lease): + return StatusTurn(False) + result = await self._project_version_on(connection, lease.scope.account_id) + if not await self._ack_on(connection, lease): + raise ReportingNotificationError("status_lease_lost") + return result + + async def sweep_one(self) -> StatusTurn: + async with self.ledger._connection() as connection: + at = await database_now(connection, self.ledger._clock) + row = await ( + await connection.execute( + "SELECT c.account_id FROM reporting_status_scope_checkpoints c" + " JOIN reporting_projection_accounts a ON a.account_id=c.account_id" + " WHERE c.next_due_at<=%s AND a.policy=%s::jsonb" + " AND (c.lease_expires_at IS NULL OR c.lease_expires_at<=%s)" + " ORDER BY c.next_due_at,c.account_id LIMIT 1", + (at, json.dumps(self.policy), at), + ) + ).fetchone() + if row is None: + return StatusTurn(False) + lease = await self.claim_due(account_id=row[0]) + return await self.complete_due(lease) if lease is not None else StatusTurn(False) diff --git a/src/adcp/reporting/projection/required_schema.json b/src/adcp/reporting/projection/required_schema.json new file mode 100644 index 000000000..19d957b28 --- /dev/null +++ b/src/adcp/reporting/projection/required_schema.json @@ -0,0 +1,1270 @@ +{ + "column:reporting_projection_accounts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_accounts.activated_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_accounts.checkpoint_floor": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.consumer_status_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_projection_accounts.current_as_of": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_accounts.current_input": { + "enabled": true, + "fingerprint": "1f539b84d6adb1a8577add320aabb81fdd5c614ad01b46c7b77eb55b995a6556" + }, + "column:reporting_projection_accounts.cursor": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.legacy_capture_cursor": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.legacy_capture_through": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_accounts.legacy_generation_floor": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_accounts.legacy_through": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_accounts.max_sequence": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_accounts.notifications_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_projection_accounts.ownership_enabled": { + "enabled": true, + "fingerprint": "1abe3a1c570fbe885784dab5d979307c373d50f567ff4481ce3996869bf58fed" + }, + "column:reporting_projection_accounts.policy": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_accounts.version": { + "enabled": true, + "fingerprint": "cec0cb8bc536b9a98fca05facc40ec8bb053622ba9030f5cb1888fe47da07e39" + }, + "column:reporting_projection_feed_snapshots.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_feed_snapshots.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.document": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_feed_snapshots.ownership_mode": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_feed_snapshots.representation_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_projection_feed_snapshots.signing_key": { + "enabled": true, + "fingerprint": "554c34e416bd4546469b42d5773bc7c59b2104366ffa5259a2838c64cd826e57" + }, + "column:reporting_projection_feed_snapshots.snapshot_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_inputs.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_inputs.as_of": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_inputs.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_inputs.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_inputs.sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_inputs.transaction_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_baselines.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_baselines.checkpoint": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_baselines.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_baselines.scope_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.checkpoint": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_checkpoints.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_checkpoints.scope_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_legacy_inputs.consumer_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_inputs.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_inputs.kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_steps.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_steps.account_sequence": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_legacy_steps.content_sha256": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_legacy_steps.input": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_legacy_steps.scope_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.auth_mode": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_deliveries.body_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_deliveries.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_deliveries.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_notification_deliveries.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.destination_sha256": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_deliveries.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_notification_deliveries.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_deliveries.envelope": { + "enabled": true, + "fingerprint": "554c34e416bd4546469b42d5773bc7c59b2104366ffa5259a2838c64cd826e57" + }, + "column:reporting_projection_notification_deliveries.envelope_version": { + "enabled": true, + "fingerprint": "64be57437fdc0a07a97985c2aa058031f8082db7251bdb4d5afa1a9b088de97a" + }, + "column:reporting_projection_notification_deliveries.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_deliveries.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.key_version": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_notification_deliveries.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_deliveries.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.signing_scope_id": { + "enabled": true, + "fingerprint": "85b88fb407e112399f25b5dc8830dfdcb2a8271ecf00e70b662f6917b4a002cd" + }, + "column:reporting_projection_notification_deliveries.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_projection_notification_deliveries.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_deliveries.subscription_fingerprint": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_events.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.admission_epoch": { + "enabled": true, + "fingerprint": "b636d6e865960a7e2527ed11b86a9a63fff025d6106672308d86fc9630c328ab" + }, + "column:reporting_projection_notification_events.cause_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_events.cause_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.cause_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.delivery_config_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.fingerprint": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_notification_events.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_notification_events.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.notification_type": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.obligation_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.scope_kind": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_events.snapshot": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_notification_events.version": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_expansions.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_expansions.claim_count": { + "enabled": true, + "fingerprint": "42202005517b72e082eb22c9eceb2ac0252815e5df700c83eb50c54cfeb46297" + }, + "column:reporting_projection_notification_expansions.consumer_namespace": { + "enabled": true, + "fingerprint": "128b66e02f14946100273f112c144af7605d626124d20fe16c1ba30c6b19ae3a" + }, + "column:reporting_projection_notification_expansions.due_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_notification_expansions.emission_generation": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_notification_expansions.error_code": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_expansions.lease_expires_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_notification_expansions.lease_token": { + "enabled": true, + "fingerprint": "5b92595d0b54d473a3a3818455845f1fe0dc20b48cfe0faf63061a82d1a3cb02" + }, + "column:reporting_projection_notification_expansions.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_notification_expansions.state": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_projection_webhook_attempt_heads.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.last_attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_webhook_attempt_heads.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempt_heads.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.attempt": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_webhook_attempts.binding": { + "enabled": true, + "fingerprint": "ac355fc16c02b70cb0a24afee8214cdce5f5cbfdc7fd1630786d5101932ecfa4" + }, + "column:reporting_projection_webhook_attempts.completed_at": { + "enabled": true, + "fingerprint": "6f1466ce5d0aaac8471e39834b9c4b1f85d6f7169d9238a245ac035ff518e0fc" + }, + "column:reporting_projection_webhook_attempts.consumer_namespace": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.delivery_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.fired_at": { + "enabled": true, + "fingerprint": "1cac4e73af11a8ecafd646ef6a0ff6ecb087d46f150408dcfebc630fe1bf5e1e" + }, + "column:reporting_projection_webhook_attempts.http_status_code": { + "enabled": true, + "fingerprint": "4340876cb26818ac55a8d51cfbc7047e90fc4d7e44f570ee454b4552beb351a1" + }, + "column:reporting_projection_webhook_attempts.idempotency_key": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.lease_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_webhook_attempts.notification_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.payload_size_bytes": { + "enabled": true, + "fingerprint": "54a5f9962598caa8723435f8b55bd5e15823053af5dd63d531663997537dc6ba" + }, + "column:reporting_projection_webhook_attempts.principal_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.reservation_token": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_webhook_attempts.response_time_ms": { + "enabled": true, + "fingerprint": "992336704a95e12ec6e959825c59fa2e51cddc5f1568af6bebaf12f03ac5655f" + }, + "column:reporting_projection_webhook_attempts.status": { + "enabled": true, + "fingerprint": "1a1ab7c892bfef3ca42f00cf764453bc5e0578b3a82ecf81da90b56816df493f" + }, + "column:reporting_projection_webhook_attempts.subscriber_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_webhook_attempts.url": { + "enabled": true, + "fingerprint": "10ccaa0dc3b93d48a1f32c7ef2352a11676632e1e871d77ef9ef4393eea15d27" + }, + "column:reporting_projection_writes.account_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_projection_writes.transaction_id": { + "enabled": true, + "fingerprint": "fdc4b549138bf90a5af5ed00b87a86ff9a0cbea553e7f017b4d95510708f6f88" + }, + "column:reporting_status_scope_checkpoints.projection_writer_floor": { + "enabled": true, + "fingerprint": "7d6ce545a69d9f3e873875690ed9ccbec7114a9e32ad4820babca3f1c97b38ea" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_check": { + "enabled": true, + "fingerprint": "ea35b760e040eef6a722d0b1ecf8a6e9f8d9d57feb6ac045caa2037b6b048a9b" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_check1": { + "enabled": true, + "fingerprint": "77fe4d1cb567d39e9e6dae13605e0a9a50ec80cc8905c9c826d06394dc4f02bd" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_check2": { + "enabled": true, + "fingerprint": "03eea3ba4967a34e66d84e32942c137d2548855973811cd4c301744098613880" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_checkpoint_floor_check": { + "enabled": true, + "fingerprint": "4a5954bcfbdb5854908b976ed27044b8c64b4ad502f436f3333ef82973a44f8f" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_legacy_capture_through_check": { + "enabled": true, + "fingerprint": "c5e638b4dc3130b99175082d33af80a2960e7c864129283767dd049e595aa4e8" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_legacy_generation_floor_check": { + "enabled": true, + "fingerprint": "eeadbe3edcffa3355d8dc736af0580bf1c97a9b7a404213c277af268f1e10e75" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_legacy_through_check": { + "enabled": true, + "fingerprint": "af3dcbbc378afb98c3ed16dd682316f9dfb4d66ac9d12b9118edecdc554cd19b" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_max_sequence_check": { + "enabled": true, + "fingerprint": "0fc3b5ce464cce8adb79b8a8e8379c84713d2c37902b8b969cbaa0ac5322b0ae" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_pkey": { + "enabled": true, + "fingerprint": "e65d70e61c89a93d66c4c4f4c59ef755d0ff526b0fcdd1d9ae6d830d2abea913" + }, + "constraint:reporting_projection_accounts.reporting_projection_accounts_version_check": { + "enabled": true, + "fingerprint": "905e3e7e12f64037c03d855fe84bf4319694539049e337e0ee0fcffb46d392a4" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshot_representation_version_check": { + "enabled": true, + "fingerprint": "b4bb98e6c439f502c4ae47a541ac87e553b808e2b41efeb354d1b6e97c029caa" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check": { + "enabled": true, + "fingerprint": "00c281a801bf6afa7206453af0c2ec9839dae0df4420b9b7f6a1990eeb7c358f" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check1": { + "enabled": true, + "fingerprint": "375f2992cfd0b07e7dfefe6ae5c2aaa8fd16007d5964e91e8372a28c09aa684f" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check2": { + "enabled": true, + "fingerprint": "84972326afd68c2ff32f599ad0f129c22b91072d0eb4e1380613a91ded47e6da" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check3": { + "enabled": true, + "fingerprint": "18aad02260bda2aba9f6e51349520ac53ba3901ac25e26877d4b06c9d92832e3" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check4": { + "enabled": true, + "fingerprint": "ba2bd76a123a1c07c02d86609a9275fccedb28c2a9049ac87e5441c6c4e8aa64" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check5": { + "enabled": true, + "fingerprint": "fbb1a305ea8b9cd37c90a3612dccb065ca57d04405711dcba64a98bff46e5761" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check6": { + "enabled": true, + "fingerprint": "d6b940c0765d3ae2e773a1cba51ae8b478820bbe9bc593e005115421d1e9a415" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_check7": { + "enabled": true, + "fingerprint": "87a2ec39393813ff33449cd7a02cc4738d1e91d5b6406e212b78372def2c1778" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check": { + "enabled": true, + "fingerprint": "84bdfff00c1f410606570fb522ab542616825c122b05a4e5d7b35d9937dd70e4" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check1": { + "enabled": true, + "fingerprint": "763668adc07dbcc7c5275dab0bbd3b9f0b6833893acd9725f4e9913d09c3c1e9" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check2": { + "enabled": true, + "fingerprint": "b393d759417c024e203b8521b8ba1e34d76e85dda5a8dfcdb1882ed2a56568f2" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check3": { + "enabled": true, + "fingerprint": "2053806dff13a9802996d32867a3858703e7e9a03b11735d216582cbde48ab4b" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_document_check4": { + "enabled": true, + "fingerprint": "bf8ed87ea3b56d6eb3d7c0efa3575695b2b9e56f32bc0b3d172cc2395bcb1c9d" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_ownership_mode_check": { + "enabled": true, + "fingerprint": "b642ee29e8813655ec63bb50bd53a601ac4da60cd3d4d31028340f55729aa362" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_pkey": { + "enabled": true, + "fingerprint": "69c464c154977fe93cbcf866d711d9dc62357603a4a2cc104b280b44984d3533" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_signing_key_check": { + "enabled": true, + "fingerprint": "4beb9e7f9146e3095acdb7d1dd3867766108e3ab93313deb9b6b0953e992c024" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_snapshot_id_check": { + "enabled": true, + "fingerprint": "da3012902db7fe22f476ac4cca0d315f27456cace1dbb9f2ba4b3c9eae2d0a9e" + }, + "constraint:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_snapshot_id_key": { + "enabled": true, + "fingerprint": "c97a22e931dedffcfbd8ab22824035126745be64aa5d616a90e33563643ca749" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_account_id_transaction_id_key": { + "enabled": true, + "fingerprint": "32924d7285616a887a0008aa1f0b8c418caae69c7429278d01064b64cba91dae" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_check1": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_check2": { + "enabled": true, + "fingerprint": "b330c171bc0bb87ac12aa97aca4d4ae7167e0c07d7b61b3b688c63df9588d927" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_input_check": { + "enabled": true, + "fingerprint": "cac041ba097e7ddad3ed595027b946cfb0c4a8b75bf187f5d2d9608fdb7174ac" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_pkey": { + "enabled": true, + "fingerprint": "146d890eadd6744b5c6a7bfdbe840580d36e46d49ece8c02eff7c076f4edaec3" + }, + "constraint:reporting_projection_inputs.reporting_projection_inputs_sequence_check": { + "enabled": true, + "fingerprint": "554d491362648e795a6567528a6244977d5df276216297f35e26e4750f869dcf" + }, + "constraint:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_check": { + "enabled": true, + "fingerprint": "c05b78095cbb771e19c3758c560aa94e1b86c593ea90c3242012c8164d995ad5" + }, + "constraint:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_pkey": { + "enabled": true, + "fingerprint": "b7ae4c1159881c6b7e8cdd2a56418a730fc904db9deb2d993d4cf8fb81ad9cd8" + }, + "constraint:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_check": { + "enabled": true, + "fingerprint": "c05b78095cbb771e19c3758c560aa94e1b86c593ea90c3242012c8164d995ad5" + }, + "constraint:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_pkey": { + "enabled": true, + "fingerprint": "b7ae4c1159881c6b7e8cdd2a56418a730fc904db9deb2d993d4cf8fb81ad9cd8" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_account_sequence_check": { + "enabled": true, + "fingerprint": "1a64697a81e3b39433d851f9916df1964cbbc260aa54dd6fee4076c7e805c70e" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check1": { + "enabled": true, + "fingerprint": "6db9e761fce16f6deaabe04581b3813f3495382b5edf905c2c01a772151eed89" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check2": { + "enabled": true, + "fingerprint": "6c5865ad0c28d00ec74194418def9d764ddf85b230f1dfb6d14c6e505eda3737" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_check3": { + "enabled": true, + "fingerprint": "0d806a2b956a1e8947dcc43c634352ed52c1c3f158235145b2fe5a96db5fb379" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_kind_check": { + "enabled": true, + "fingerprint": "b97ee75a4c9b39ff41cb90a8dc2faa6e4c985bbd6e2c96e4c77d6fb50f10e6cf" + }, + "constraint:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_pkey": { + "enabled": true, + "fingerprint": "5584c208839db0f8f7bc64ae38ae5a6f97c23493e9c4c19648e1e46b7fdb3f61" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_st_account_id_account_sequence_fkey": { + "enabled": true, + "fingerprint": "c9d5184cf4f53772f44f1fa97dbd3225d856113e8243d49e05177194a6d3573e" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_steps_check": { + "enabled": true, + "fingerprint": "88c2bcdd52446d3f8a7c5e21b16f277d231d44a17842e8c625d02e31cc16dd22" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_steps_input_check": { + "enabled": true, + "fingerprint": "f0f3bd2238cf6933df16e0f0cb3a2eb9049bd3e20150af23067d04760ae6189f" + }, + "constraint:reporting_projection_legacy_steps.reporting_projection_legacy_steps_pkey": { + "enabled": true, + "fingerprint": "54a1db9ba984eb27a50dc481d6e21cd2d187af90355713309f7b2f6c81927f40" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notific_account_id_consumer_namespac_fkey2": { + "enabled": true, + "fingerprint": "1e6be1e6f690856d3bc97d14b0e5837a60cc50d6f28432d40ec3f106398ed35d" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notific_account_id_consumer_namespac_fkey3": { + "enabled": true, + "fingerprint": "c33f0f1c9aaadd0bce3ad32c95b885e7d75af1fe37517c27e47f381d57f696b3" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "91982910079d78e1b6162e69e1e40d0f98ab055b811726cd8a9357c4aa9cfd22" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key4": { + "enabled": true, + "fingerprint": "bece3b7de1c22f3a600c89c175f942f83ecff078142418072b1bc1b42a670c18" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_del_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliv_notification_type_check": { + "enabled": true, + "fingerprint": "ee42647bcbefde76d58a6b7231c68bc1c4ef047e2523f779b4c28bcd9792eb47" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_delive_cause_generation_check": { + "enabled": true, + "fingerprint": "d93961696665129cf254c93dae4377741c5cef5fe9d373c6a5cb36226e5a69e2" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_cause_kind_check": { + "enabled": true, + "fingerprint": "10f7027028f807d2ac122ae119f6de51d5c3b54217dcf78d0371b7d92ac4e942" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_check": { + "enabled": true, + "fingerprint": "a61e7698dd266b3a4abee2c2716b00919f047e85bcd888a90c1a5bc5350c4d6e" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "126da0131155e9b09e48d24466c5f61b7b6dc1540a68c465c52b59769637820d" + }, + "constraint:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "ee26a73f446c794ba69fc056f5fb1b8e094b25ae48743fe5e23045749e84f1c2" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "7156cd6623f45f181906d2886b53dfba82bfdfc6cdcbdd0017c1f1cd7ceae407" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "1d1f2cee412feec063fd1340b0d7773adfc55cad1f740e21bb45f75acf185b8c" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "a21acc6c4e405fcbf7711fe5a82c55206c994c6c969b498f9afe35af859ed206" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_event_notification_type_check": { + "enabled": true, + "fingerprint": "ee42647bcbefde76d58a6b7231c68bc1c4ef047e2523f779b4c28bcd9792eb47" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_admission_epoch_check": { + "enabled": true, + "fingerprint": "6cfd9db9dd38e8c29346710c69de6ff07379d1bef2e9e5b1aba297a3ffa833f2" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_cause_generation_check": { + "enabled": true, + "fingerprint": "d93961696665129cf254c93dae4377741c5cef5fe9d373c6a5cb36226e5a69e2" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_cause_id_check": { + "enabled": true, + "fingerprint": "e86db06c50f414ddc137693ad14cc7faaf8c3805591871c6d001cc5438d31a71" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_cause_kind_check": { + "enabled": true, + "fingerprint": "10f7027028f807d2ac122ae119f6de51d5c3b54217dcf78d0371b7d92ac4e942" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check": { + "enabled": true, + "fingerprint": "efada5fd861f242458a8c57b2bf1c97280cd3c266d285050679afb2bbff363cb" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check1": { + "enabled": true, + "fingerprint": "36694052bb1b576de628eee965d5821ab01923e563295220ca21252603c2aa7a" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check10": { + "enabled": true, + "fingerprint": "627584412b64b780790cab6c0ca3a8f6e11a2b954eda505f58b709264a301956" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check11": { + "enabled": true, + "fingerprint": "1c4273a87391e21a4156104e7c9a556a70ddafaaca8810533b75335b14fa93bf" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check12": { + "enabled": true, + "fingerprint": "5cf1645ef1880c0fd61275e271b8ebfe33e12b4aa79646d82a1aef609c333b64" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check2": { + "enabled": true, + "fingerprint": "dc119dbfe2444526e08b41cedd271ea31deab1082f55816cc6d4257aa088f9d8" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check3": { + "enabled": true, + "fingerprint": "ffe54c1b8d166fbbfe66ebbf70dbd5ee8c8a61b09fdc57bbf2d5c9ccea8ad806" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check4": { + "enabled": true, + "fingerprint": "a2d30638b070f0f9f83bfaf782cc1a36cc89b2de31471a589f9568c174b8f7a1" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check5": { + "enabled": true, + "fingerprint": "7e8ae7f4e2f6d5b320600b088f0f7c9f31520fb499cfc8504a46b5fda4aa1180" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check6": { + "enabled": true, + "fingerprint": "49a10f7157a0b18ad541d7bf95ba43d28849ab45892d592d87bd720816325780" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check7": { + "enabled": true, + "fingerprint": "b7666e96be91fe368c981a8e7e0c3017823e34f97354428dd3d29c5a08a9c6a8" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check8": { + "enabled": true, + "fingerprint": "25fa4df322afde24dad3652593dffe4e696ceb9ab60b364f13d848839fea91f5" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_check9": { + "enabled": true, + "fingerprint": "dafe7fb15d13195b1d2a6a1b4f62287d70b42543bc9ac156e32b1bce4f9c3c15" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_fingerprint_check": { + "enabled": true, + "fingerprint": "e6f7fe9bd1925eb05eeab481da0229b0180558dd2448eda9e51e98faf31d6497" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_pkey": { + "enabled": true, + "fingerprint": "e8d63f44b25700915bcb3cb69e2e24b98cf21fd304f36a2e9c2992236309309d" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_scope_kind_check": { + "enabled": true, + "fingerprint": "990e8c59c76274efb82c87d70135717b5b47f7edbf86b52cdde0377a3f85194f" + }, + "constraint:reporting_projection_notification_events.reporting_projection_notification_events_version_check": { + "enabled": true, + "fingerprint": "e7b1e199b6f69994a833033a7770c2b6fcbf1b3ba77251b5df6bea466b0ee4b9" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notific_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "3325be861e9f98edd263b2b2e7952b31fde4d9463458f9044c426388b14c175a" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_exp_emission_generation_check": { + "enabled": true, + "fingerprint": "5b7b2793bd710c7eef8c02003fd8adb1df91d7c10659364fe37d7f6351d4bf5e" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_expansions_error_code_check": { + "enabled": true, + "fingerprint": "502085808cda7e7ec1e45e3d88257a660bdc65b59a71727b473303a362b9cf9d" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "8ac920e8e1ed5a59417d5d1ffb9bea608bbeb64dbcd1a431240370fe555fe6f0" + }, + "constraint:reporting_projection_notification_expansions.reporting_projection_notification_expansions_state_check": { + "enabled": true, + "fingerprint": "12b7a569e49f4bf374d5dade9ec0a0d812e4dea251366467c95bcd58bee90b9d" + }, + "constraint:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_last_attempt_check": { + "enabled": true, + "fingerprint": "5bf9c354d680faec2bf3155f8c9b27323d7893564b30688a039ba19fc8b8d5d3" + }, + "constraint:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "490860c80b1129b15ab44f458a1a967382fc8b073f694ac959bf938282a74af5" + }, + "constraint:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_principal_id_check": { + "enabled": true, + "fingerprint": "bfc70b4b01cc74b9c93c630928d7cde9b0134c3d6614e2cbb326604c865c2cac" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook__account_id_consumer_namespac_fkey": { + "enabled": true, + "fingerprint": "160e4a76316d7669011f46848f4053b65a149f749fcec376a0318c21593a9931" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "b8546a68def520db0e6139c898e3e2a76318e05548a3de1b6e568fbf3a75ee63" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_account_id_consumer_namespac_fkey1": { + "enabled": true, + "fingerprint": "ed1785d5d7786390edfd842f1a7184f234f35e136ff3b23fe5993a8451f34200" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_attempt_check": { + "enabled": true, + "fingerprint": "e05925f8a9fe41263902b6b2a72bca02958627e079787e99987820a52cb47d71" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_payload_size_bytes_check": { + "enabled": true, + "fingerprint": "95548ff5519cec8b0ff110ad66f7b77d77e6015ef8c54a52e09cd4d7d80a7118" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "5580d4f19f5ba257093ef71127d72dbba9669ac2db9b2f116f246d7f760522b4" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_response_time_ms_check": { + "enabled": true, + "fingerprint": "6698890e9bba25670f73e3a5a7a7a7f9b664ab2571637eba582f22863c92db4d" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_status_check": { + "enabled": true, + "fingerprint": "1c821d6bf41e5137a6263eae308d0a8648b7f0c95154724e5430e8c630aec30e" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_completion": { + "enabled": true, + "fingerprint": "adc02762a63dcbf72330ca318884e6236fb2418cb7e1111f861ee74f98b53316" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_identity": { + "enabled": true, + "fingerprint": "1e3f9725baa47d118c5b2efeeafbd5a2429400c1f667188fd14c4f2795114bf8" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_outcome": { + "enabled": true, + "fingerprint": "cc0cc61f029bb3b28629e12a42c6f6ddbfd943156e1b90e5eeb2346aed678e00" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_timestamps": { + "enabled": true, + "fingerprint": "0175d921479ed64020d88874f3b6ac822a9979f3ec8c3780d89bb9e0e3e3556c" + }, + "constraint:reporting_projection_webhook_attempts.reporting_projection_webhook_url_safe": { + "enabled": true, + "fingerprint": "3dd08bbc446317552434fb91bebd96e2a534395454b436910450f909ce2ab88e" + }, + "constraint:reporting_projection_writes.reporting_projection_commit": { + "enabled": true, + "fingerprint": "20ed755d29980bfc007ba63a8f0b50277c734a0703525b0c3ebeb1ebdc466162" + }, + "constraint:reporting_projection_writes.reporting_projection_writes_account_id_fkey": { + "enabled": true, + "fingerprint": "53a472fec66c9e990740ba025807acc57c5795bd0a84ea1ad29f8447fcfd0c85" + }, + "constraint:reporting_projection_writes.reporting_projection_writes_pkey": { + "enabled": true, + "fingerprint": "4d672603e38a33518c9c2166994348a7ec73c90f7b0c1f2afc73155bb6afdbc6" + }, + "constraint:reporting_status_scope_checkpoints.reporting_projection_writer_floor": { + "enabled": true, + "fingerprint": "2938b1784f2a320559cccf185d99ce7a8bb663f166eee8cbe883585e54b7814c" + }, + "function:reporting_projection_capture(owner text)": { + "enabled": true, + "fingerprint": "aa5ec57f464d517011526cdd6671e247e49ad622744a81a8d992f2db6b0b8007" + }, + "function:reporting_projection_checkpoint_guard()": { + "enabled": true, + "fingerprint": "326574129aa5b54b23eb18705b4898b600458a8b746b9a04a04a44e2575bec7b" + }, + "function:reporting_projection_commit()": { + "enabled": true, + "fingerprint": "da0a2d33359f39e067c9402d8b5474df4cb2eef1c1a49f48aa5e266c75aa7666" + }, + "function:reporting_projection_document(owner text, at_time timestamp with time zone)": { + "enabled": true, + "fingerprint": "33911a2307e5b4564bdbf6766e70ab46a1f81c5c8e5b37b88905a69f65a1c466" + }, + "function:reporting_projection_event_guard()": { + "enabled": true, + "fingerprint": "64f503d73db42c174921cbb9cbaf822383e39d35607c2ce0d9ff315c2c5f3074" + }, + "function:reporting_projection_feed_immutable()": { + "enabled": true, + "fingerprint": "b17f620de435a86c8815c3efbace4b602166bad7dc95f923434d402cddcd63f5" + }, + "function:reporting_projection_mark()": { + "enabled": true, + "fingerprint": "fa1c7a33b889525a065b07bff321794530ba8fbff996e901f6666bc75684536d" + }, + "function:reporting_projection_webhook_attempt_guard()": { + "enabled": true, + "fingerprint": "622d83f4fd58d38d06f2eef2223f011fe16eb9758438ae3155dcb0a2e5ddfe0b" + }, + "function:reporting_projection_webhook_head_guard()": { + "enabled": true, + "fingerprint": "4e448e0df3d56a5b04bc01ca41a4c5afbb8e601a4699d2e6dd0e3922ed2f0bf8" + }, + "index:reporting_projection_accounts.reporting_projection_accounts_pkey": { + "enabled": true, + "fingerprint": "4b805ae5bc3f3d509fb5f6d1551af8a6fa2065a9d6151a7f218f08c2be7a5292" + }, + "index:reporting_projection_accounts.reporting_projection_activation_pending": { + "enabled": true, + "fingerprint": "0dd7ddc05637321a246cf9236a202c9c4f96836993fdc2de75ff5fc7d6a625a8" + }, + "index:reporting_projection_accounts.reporting_projection_pending": { + "enabled": true, + "fingerprint": "97566fb1a05d93b85a923bc721b7e551bb6256184835c2caf1c089c870ed4585" + }, + "index:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_pkey": { + "enabled": true, + "fingerprint": "fbbf876488b7acbc5964c465cc63e579ec21c6036708a4baa5f8a4f36d582cd6" + }, + "index:reporting_projection_feed_snapshots.reporting_projection_feed_snapshots_snapshot_id_key": { + "enabled": true, + "fingerprint": "74d4af408b949a2021008e0e4cb1fa46fec81746804548c9333da4f4ca275f36" + }, + "index:reporting_projection_inputs.reporting_projection_inputs_account_id_transaction_id_key": { + "enabled": true, + "fingerprint": "e79866e916616016aad04b889700e57adb174019d81190ac1d9db4e8c3c475dc" + }, + "index:reporting_projection_inputs.reporting_projection_inputs_pkey": { + "enabled": true, + "fingerprint": "b74110aa7db352e4c0d8dd971620b983d43d9fca140e3da3e4379f12c3dbcf93" + }, + "index:reporting_projection_legacy_baselines.reporting_projection_legacy_baselines_pkey": { + "enabled": true, + "fingerprint": "33808c42f95cdd713a7067c81a772edd709fce85c5aa1a1f83329cf58bb7b6a2" + }, + "index:reporting_projection_legacy_checkpoints.reporting_projection_legacy_checkpoints_pkey": { + "enabled": true, + "fingerprint": "17288d53dcfa13cbec3c058b98485cf8063da6a752ff20fbe1d6493ef50bb2d2" + }, + "index:reporting_projection_legacy_checkpoints.reporting_projection_legacy_consumer": { + "enabled": true, + "fingerprint": "43909bb4e3297a0e79795b308e6157724c50ac63559e9d5bf50d565de1db9e62" + }, + "index:reporting_projection_legacy_inputs.reporting_projection_legacy_inputs_pkey": { + "enabled": true, + "fingerprint": "c3717266852eee124f070047263fc5d0c76b6680416b7c72f74834d1b25beaa9" + }, + "index:reporting_projection_legacy_steps.reporting_projection_legacy_steps_pkey": { + "enabled": true, + "fingerprint": "572e483043d844b80f0bcb05162d210c233eafe1776235f197475b6fc23631ea" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key3": { + "enabled": true, + "fingerprint": "db9f6646990410974ecfe24a7dd3b1d4eefc35678756d91789f37a8e15640344" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notifica_account_id_consumer_namespac_key4": { + "enabled": true, + "fingerprint": "41c096310d5ca152e99b979a7ba591b3276ee4c1dbb858a243201f2c93720cc9" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_due": { + "enabled": true, + "fingerprint": "dd243bfbf67a1d56f4773583a92efd477ebc262e772706d4c759bbbe0ccb5236" + }, + "index:reporting_projection_notification_deliveries.reporting_projection_notification_deliveries_pkey": { + "enabled": true, + "fingerprint": "39973dc1c3de7bdd4bb1ae7ab98477ebbda282ff9e93398b8499e4bc7169e8fc" + }, + "index:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key1": { + "enabled": true, + "fingerprint": "ffcbea4686e819d18554bf3c7357122e90aaf14a4227ab321e1bcf15248b84d8" + }, + "index:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespac_key2": { + "enabled": true, + "fingerprint": "cc35d20e8e088c65dfc86e25fd6f4f67e8644307d25b32bab6c545e144bd0e87" + }, + "index:reporting_projection_notification_events.reporting_projection_notifica_account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "ca07092e6e38681fee63f6717cf10f1446bf3b8bc117a73cde19b5dcc987be5c" + }, + "index:reporting_projection_notification_events.reporting_projection_notification_events_pkey": { + "enabled": true, + "fingerprint": "0e42c2e06c2b9faccd04e0ac781ccc0a7ff684ba827e84523a70abd427ade222" + }, + "index:reporting_projection_notification_expansions.reporting_projection_notification_expansions_due": { + "enabled": true, + "fingerprint": "a654a298089afe3acfff950faacaf4bcddc198db1680756ce3a6cc5cceba2905" + }, + "index:reporting_projection_notification_expansions.reporting_projection_notification_expansions_pkey": { + "enabled": true, + "fingerprint": "37720f22ee293ab422ad013f058a90c1d02fcc12ff547c424a724eb3b32c2501" + }, + "index:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_attempt_heads_pkey": { + "enabled": true, + "fingerprint": "34ead5983456c814e390fcd1951ece7d38545783c6b9edc52a568ea623e05475" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook__account_id_consumer_namespace_key": { + "enabled": true, + "fingerprint": "c7e1c8f8be8ff6fe8798a05bbaa74b478de73e887433c948989d1e750bc85593" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook_activity_newest": { + "enabled": true, + "fingerprint": "199287b543822d45c89d735206c1636e8d0978bea77a8240b7793640fade97d3" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook_activity_retention": { + "enabled": true, + "fingerprint": "f6f99131566c8279a4b3d4006a2122954acedeb287bd9f59eff8dfafb67c9963" + }, + "index:reporting_projection_webhook_attempts.reporting_projection_webhook_attempts_pkey": { + "enabled": true, + "fingerprint": "b2c98959ee7fa3d182d6fea57f2ddaa5beb726da8f97a7081e1b9b59da78cffe" + }, + "index:reporting_projection_writes.reporting_projection_writes_pkey": { + "enabled": true, + "fingerprint": "23e19a2a5adb02bec7292061246f159726c855318b1686e01b88357cdf1a11a9" + }, + "index:reporting_status_scope_checkpoints.reporting_projection_due_clock": { + "enabled": true, + "fingerprint": "f7343b98d419d3e2d87d25f88469d6c1e92a9838a6f359452e58712c4f406a5e" + }, + "table:reporting_projection_accounts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_feed_snapshots": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_inputs": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_baselines": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_checkpoints": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_inputs": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_legacy_steps": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_notification_deliveries": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_notification_events": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_notification_expansions": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_webhook_attempt_heads": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_webhook_attempts": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "table:reporting_projection_writes": { + "enabled": true, + "fingerprint": "1f824779ff80f110344420b019786663d8c9beaad230da90e0439795e734ccda" + }, + "trigger:reporting_adjustments.reporting_projection_mark": { + "enabled": true, + "fingerprint": "192ccdbcab9b7193a85990dcef614ef0be68fd79582525856b3c00e793d361b4" + }, + "trigger:reporting_configurations.reporting_projection_mark": { + "enabled": true, + "fingerprint": "09cb0a6ab2c13da199badf822de029b0290387a098bb0365844e72d4f74a08bb" + }, + "trigger:reporting_consumer_statuses.reporting_projection_mark": { + "enabled": true, + "fingerprint": "91d3dff97f063184909c23620c5bcd6a0cc1c07bae8c888b58a80c3a2342c5bd" + }, + "trigger:reporting_issue_lifecycle.reporting_projection_mark": { + "enabled": true, + "fingerprint": "2793d5d63ec5e17d63695f5264ca8a78f30f4353d967b2acf12d3dda1f420634" + }, + "trigger:reporting_issue_status_scopes.reporting_projection_mark": { + "enabled": true, + "fingerprint": "b6ec60e605cd956fda57ffec137fb4530ba2c41d071351d626aa65113ae4afba" + }, + "trigger:reporting_obligations.reporting_projection_mark": { + "enabled": true, + "fingerprint": "83f851a94ccf29c50b0d9ca2d8e5d5ee4edf9d3801ae806077dc47f7113a8230" + }, + "trigger:reporting_projection_feed_snapshots.reporting_projection_feed_immutable": { + "enabled": true, + "fingerprint": "4a3f5552f5c940234144e1300ff526fc54f7376c076b457a47e48d38848d06b4" + }, + "trigger:reporting_projection_inputs.reporting_projection_input_immutable": { + "enabled": true, + "fingerprint": "714e3c04ac1ffdf34f8435915e3829fcef55bd3df8ebe4b719f5d14414b6fcb1" + }, + "trigger:reporting_projection_legacy_baselines.reporting_projection_legacy_immutable": { + "enabled": true, + "fingerprint": "68c9d1f4db22039ae8ebfa2b315b0325c512d138906833c9658341279aaf1d2d" + }, + "trigger:reporting_projection_legacy_inputs.reporting_projection_legacy_immutable": { + "enabled": true, + "fingerprint": "8005ae52bd5e328ff3c7774768ab875b3eea36993d9fbdedb6db2d3766aafd8f" + }, + "trigger:reporting_projection_legacy_steps.reporting_projection_legacy_immutable": { + "enabled": true, + "fingerprint": "0979972b1537ecd106497c92511ad512a255e185646b4416c25355aad8aeaab5" + }, + "trigger:reporting_projection_notification_events.reporting_projection_event_guard": { + "enabled": true, + "fingerprint": "6ad8a2eb5e1c4b6ddf040412d468f2805b3695b055b8258dbc4bf07fed36fc4b" + }, + "trigger:reporting_projection_notification_events.reporting_projection_event_immutable": { + "enabled": true, + "fingerprint": "69fe101856c6f93e27f3366027a54e6c005953d8b1fd09fc089c8642b723a925" + }, + "trigger:reporting_projection_webhook_attempt_heads.reporting_projection_webhook_head_guard": { + "enabled": true, + "fingerprint": "f3ad96da6db3a669fc802e18373b8a8e158f0b0ca2aea04ca1b9683107dd714c" + }, + "trigger:reporting_projection_webhook_attempts.reporting_projection_webhook_attempt_guard": { + "enabled": true, + "fingerprint": "d4e53e26100053f79451da61382f1a587e382b15f37cc2b31411288278c8dc3a" + }, + "trigger:reporting_projection_writes.reporting_projection_commit": { + "enabled": true, + "fingerprint": "23308ca5359667c11fa7aef1a70801d6f250b3a03cc986e8c8482fe4c5344d0c" + }, + "trigger:reporting_reconciliation_changes.reporting_projection_mark": { + "enabled": true, + "fingerprint": "c94a3a33a4596e81710c3c803834638949e83d563c784ca4cc28efbf8cf8f918" + }, + "trigger:reporting_revisions.reporting_projection_mark": { + "enabled": true, + "fingerprint": "10aba6894d1730ce630923537d17a97bfb2eb984889d43cdf28c396820a0aa01" + }, + "trigger:reporting_status_scope_checkpoints.reporting_projection_checkpoint_guard": { + "enabled": true, + "fingerprint": "0403ed1d78051471207cda0c361b95ead9995a0a200574e65ad3bc98214f7a40" + } +} diff --git a/src/adcp/reporting/projection/schema.py b/src/adcp/reporting/projection/schema.py new file mode 100644 index 000000000..9b46a0592 --- /dev/null +++ b/src/adcp/reporting/projection/schema.py @@ -0,0 +1,26 @@ +"""B2.4's additive manifest never widens an earlier feature's required objects.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any + +from adcp.reporting.feed.schema import validate_feed_schema +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.outbox.status_schema import validate_status_schema + + +async def validate_projection_schema(connection: Any, *, notifications: bool = False) -> None: + try: + required = json.loads( + files("adcp.reporting.projection").joinpath("required_schema.json").read_text() + ) + actual = await schema_objects(connection) + if not required or any(actual.get(k) != v for k, v in required.items()): + raise ValueError + await validate_feed_schema(connection, notifications=notifications) + await validate_status_schema(connection) + except Exception: + raise ReportingNotificationError("status_projection_schema_unready") from None diff --git a/src/adcp/reporting/projection/wire.py b/src/adcp/reporting/projection/wire.py new file mode 100644 index 000000000..64346120a --- /dev/null +++ b/src/adcp/reporting/projection/wire.py @@ -0,0 +1,161 @@ +"""Tier-correct exact views over one captured, caller-private financial history.""" + +from __future__ import annotations + +from typing import Any, Protocol, runtime_checkable + +from adcp.reporting.ledger.delivery import ( + ReportingMaterializationView, + adjustment_to_wire, + materialization_to_wire, + receipt_to_wire, + revision_to_wire, +) +from adcp.reporting.ledger.delivery_models import ( + ReportingAdjustmentReceiptRecord, + ReportingDeliveryPrincipal, + ReportingDeliveryRecord, + ReportingDestinationBinding, + ReportingMaterializationAttempt, + ReportingMaterializationCheck, + ReportingMaterializationRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.models import ReportingObligationRecord, ReportingRevisionRecord +from adcp.reporting.ledger.reconciliation_projection import project_reconciliation +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.reporting.ledger.status_projection import ReportingStatusSnapshot +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.capture import private_snapshot +from adcp.reporting.outbox.status import settled_replay +from adcp.reporting.projection.capture import ReportingProjectionInput + + +@runtime_checkable +class ReportingTierStatusStore(Protocol): + async def read_tier_status( + self, + request: dict[str, Any], + *, + caller: ReportingDeliveryPrincipal, + consumer_status_enabled: bool = False, + ) -> dict[str, Any]: ... + + +def render_tier_status( + store: Any, + request: dict[str, Any], + value: ReportingProjectionInput, + caller: ReportingDeliveryPrincipal, + policy: dict[str, Any], + consumer_status_enabled: bool, +) -> dict[str, Any]: + if ( + getattr(store, "_projection_read_policy", None) != policy + or policy["consumer_status_enabled"] != consumer_status_enabled + ): + raise LedgerConflictError( + "STATUS_PROJECTION_UNAVAILABLE", "reporting projection is unavailable" + ) + return ReportingStatusHandler( + store, + consumer_status_enabled=consumer_status_enabled, + escalation=getattr(store, "_projection_read_escalation", None), + ).render_snapshot( + request, + caller=ReportingStatusCaller(caller.account_id, caller.consumer_id), + snapshot=settled_replay(private_snapshot(value.core, caller)), + reconciliation=value.reconciliation, + revision_ownership=policy["ownership_enabled"], + ) + + +def exact_revision_evidence( + core: ReportingStatusSnapshot, + revision: ReportingRevisionRecord, + owner: ReportingObligationRecord | None, + records: tuple[ReportingDeliveryRecord, ...], + caller: ReportingStatusCaller, +) -> dict[str, Any]: + if owner is None or owner.account_id != caller.account_id: + raise LedgerConflictError( + "LOOKUP_UNAVAILABLE", "no such revision is available to this caller" + ) + history = tuple( + r for r in core.revisions if r.reporting_obligation_id == owner.reporting_obligation_id + ) + project_reconciliation( + owner, history, core.adjustments, records, consumer_id=caller.consumer_id, as_of=core.as_of + ) + binding = next( + ( + r + for r in records + if isinstance(r, ReportingDestinationBinding) + and r.generation_key == owner.generation_key + ), + None, + ) + if binding is None: + return {} + attempts = { + r.reporting_materialization_id: r + for r in records + if isinstance(r, ReportingMaterializationAttempt) + and r.reporting_revision_id == revision.reporting_revision_id + } + artifacts = [ + r + for r in records + if isinstance(r, ReportingMaterializationRecord) + and r.reporting_revision_id == revision.reporting_revision_id + ] + adjustments = tuple( + a + for a in core.adjustments + if a.adjusts_reporting_revision_id == revision.reporting_revision_id + ) + adjustment_ids = {a.reporting_adjustment_id for a in adjustments} + receipts = [ + r + for r in records + if isinstance(r, ReportingRevisionReceiptRecord) + and r.reporting_revision_id == revision.reporting_revision_id + ] + adjustment_receipts = [ + r + for r in records + if isinstance(r, ReportingAdjustmentReceiptRecord) + and r.reporting_adjustment_id in adjustment_ids + ] + return { + "revision": revision_to_wire(revision, obligation=owner), + "adjustments": [adjustment_to_wire(a) for a in adjustments], + "materializations": [ + materialization_to_wire( + ReportingMaterializationView( + attempts[r.reporting_materialization_id], + binding, + r, + tuple( + c + for c in records + if isinstance(c, ReportingMaterializationCheck) + and c.reporting_materialization_id == r.reporting_materialization_id + ), + ), + obligation=owner, + ) + for r in artifacts + ], + "receipts": [receipt_to_wire(r) for r in receipts], + "adjustment_receipts": [receipt_to_wire(r) for r in adjustment_receipts], + "pagination": { + "total_count": 1 + + len(adjustments) + + len(artifacts) + + len(receipts) + + len(adjustment_receipts), + "has_more": False, + }, + } diff --git a/src/adcp/reporting/receipts/handler.py b/src/adcp/reporting/receipts/handler.py index 106cb9fbb..19086dec2 100644 --- a/src/adcp/reporting/receipts/handler.py +++ b/src/adcp/reporting/receipts/handler.py @@ -166,14 +166,14 @@ async def authorize() -> ReportingDeliveryPrincipal: raise ReportingFeedError("UNAUTHORIZED") from None caller = await authorize() - if request.get("view") == "periods": - async def reauthorize() -> None: - # A still-authorized alias must not change which account or - # canonical consumer owns the already captured boundary. - if await authorize() != caller: - raise ReportingFeedError("UNAUTHORIZED") + async def reauthorize() -> None: + # A still-authorized alias must not change which account or + # canonical consumer owns the already captured boundary. + if await authorize() != caller: + raise ReportingFeedError("UNAUTHORIZED") + if request.get("view") == "periods": response = await self.reporting_feed_store.read_reporting_feed( request, caller=caller, @@ -193,13 +193,24 @@ async def reauthorize() -> None: raise ReportingFeedError("INVALID_CHECKPOINT") if isinstance(self.receipt_store, ReportingLedgerStore): try: - return await ReportingStatusHandler( - self.receipt_store, - consumer_status_enabled=self._feed_consumer_status_enabled, - ).handle( - request, - caller=ReportingStatusCaller(caller.account_id, caller.consumer_id), - ) + from adcp.reporting.projection.wire import ReportingTierStatusStore + + if isinstance(self.receipt_store, ReportingTierStatusStore): + response = await self.receipt_store.read_tier_status( + request, + caller=caller, + consumer_status_enabled=self._feed_consumer_status_enabled, + ) + else: + response = await ReportingStatusHandler( + self.receipt_store, + consumer_status_enabled=self._feed_consumer_status_enabled, + ).handle( + request, + caller=ReportingStatusCaller(caller.account_id, caller.consumer_id), + ) + await reauthorize() + return response except LedgerConflictError as error: # Only the legacy Core projector exposes its established # domain errors. ACL/provider/feed failures stay redacted. diff --git a/src/adcp/server/a2a_server.py b/src/adcp/server/a2a_server.py index 4404e43e8..91d944c0c 100644 --- a/src/adcp/server/a2a_server.py +++ b/src/adcp/server/a2a_server.py @@ -1540,6 +1540,11 @@ def agent_card_url(request: Request) -> str: registered=list(executor.supported_skills), ) + if hasattr(handler, "production"): + from adcp.reporting.production.service import register_production_mount + + register_production_mount(handler, app, transport="a2a", dispatcher=executor) + return app diff --git a/src/adcp/server/serve.py b/src/adcp/server/serve.py index 4e2afb603..7e130bc4d 100644 --- a/src/adcp/server/serve.py +++ b/src/adcp/server/serve.py @@ -2571,6 +2571,10 @@ def create_mcp_server( ) _install_adcp_mcp_transport_methods(mcp) mcp._session_manager = _create_adcp_mcp_session_manager(mcp) + if hasattr(handler, "production"): + from adcp.reporting.production.service import register_production_mount + + register_production_mount(handler, mcp, transport="mcp", dispatcher=mcp) return mcp diff --git a/src/adcp/types/__init__.py b/src/adcp/types/__init__.py index 0ebc6ba98..92adf6f6a 100644 --- a/src/adcp/types/__init__.py +++ b/src/adcp/types/__init__.py @@ -138,6 +138,8 @@ "PaymentTerms", "ReportUsageRequest", "ReportUsageResponse", + "ReportingAdjustment", + "ReportingAdjustmentReceipt", "ReportingBucket", "ReportingCanonicalContentDigest", "ReportingCanonicalizationContract", @@ -1819,6 +1821,8 @@ def __dir__() -> list[str]: RegistryAcceptancePolicyProfileReference, Renders, RepeatableAssetGroup, + ReportingAdjustment, + ReportingAdjustmentReceipt, ReportingAuthoritativeParty, ReportingBucket, ReportingCanonicalContentDigest, diff --git a/src/adcp/types/_eager.py b/src/adcp/types/_eager.py index ea68a4903..5cb5df8d6 100644 --- a/src/adcp/types/_eager.py +++ b/src/adcp/types/_eager.py @@ -345,6 +345,8 @@ RefineProposalsResponse, RegistryAcceptancePolicyProfileReference, Renders, + ReportingAdjustment, + ReportingAdjustmentReceipt, ReportingBucket, ReportingCanonicalContentDigest, ReportingCanonicalizationContract, @@ -1755,6 +1757,8 @@ def __init__(self, *args: object, **kwargs: object) -> None: "ReportPlanAdjustmentResponse", "ReportUsageRequest", "ReportUsageResponse", + "ReportingAdjustment", + "ReportingAdjustmentReceipt", "ReportingBucket", "ReportingCanonicalContentDigest", "ReportingCanonicalizationContract", diff --git a/src/adcp/types/base.py b/src/adcp/types/base.py index d06fbd6b0..f415e5331 100644 --- a/src/adcp/types/base.py +++ b/src/adcp/types/base.py @@ -275,12 +275,11 @@ class AdCPBaseModel(BaseModel): model_config = ConfigDict(extra=_EXTRA_POLICY, defer_build=True) @model_serializer(mode="wrap") - def _explicit_reporting_wire_defaults(self, handler: SerializerFunctionWrapHandler) -> Any: - """Do not synthesize conditional reporting promises from generated defaults. + def _notification_config_wire_defaults(self, handler: SerializerFunctionWrapHandler) -> Any: + """Retain the unrelated product default only for product subscriptions. - The generated classes remain untouched. This wrapper also runs for - nested account notification configs and model_dump_json, without - mutating the adopter's model or its fields-set information. + Reporting capability defaults and tier validation live in their own + generated models; this wrapper does not mask reporting attributes. """ value = handler(self) if not isinstance(value, dict): @@ -295,27 +294,6 @@ def _explicit_reporting_wire_defaults(self, handler: SerializerFunctionWrapHandl str(getattr(e, "value", e)).startswith("product.") for e in events ): value.pop("product_payload_view", None) - elif any( - name == "ReportingDeliveryCapabilities" - and module.endswith(".core.reporting_delivery_capabilities") - for module, name in wire_bases - ): - for field in ( - "reliable_reporting_version", - "managed_delivery", - "reconciled_billing", - "configuration_task", - "status_task", - "consumer_status_task", - "revision_content_task", - "receipt_task", - "readiness_notification", - "status_notification", - "ledger_notification", - "supports_webhook_activity", - ): - if field not in self.model_fields_set: - value.pop(field, None) return value def model_dump(self, **kwargs: Any) -> dict[str, Any]: diff --git a/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py b/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py index cdf5428d9..4f183eb75 100644 --- a/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py +++ b/src/adcp/types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py @@ -1,9 +1,12 @@ # generated by datamodel-codegen: # filename: bundled/protocol/get_adcp_capabilities_response.json -# timestamp: 2026-09-14T14:16:02+00:00 +# timestamp: 2026-09-18T13:06:39+00:00 from __future__ import annotations +from typing import Any +from pydantic import SerializerFunctionWrapHandler, model_serializer, model_validator + from datetime import date from adcp.types._str_enum import StrEnum from typing import Annotated, Any, Dict, Literal @@ -4631,61 +4634,61 @@ class ReportingDelivery(AdCPBaseModel): ) supported: Literal[True] reliable_reporting_version: Annotated[ - Literal['1.0'], + Literal['1.0'] | None, Field( description='Explicit adoption declaration for the proper-name AdCP 3.2 Reliable Reporting contract. Presence, together with supported: true and the media_buy.reporting_delivery experimental feature gate, is the affirmative machine-readable answer. Absence denotes the earlier experimental managed-reporting shape.' ), - ] = '1.0' + ] = None managed_delivery: Annotated[ bool | None, Field( description='Tier flag: this seller supports managed file, dataset-share, or warehouse delivery. Offerings whose method names a delivery pattern require this tier. When false or absent, every offering is API-delivered and Core-only.' ), - ] = False + ] = None reconciled_billing: Annotated[ bool | None, Field( description='Tier flag: this seller supports canonical-digest verification and authenticated consumer receipts for both report materializations and post-official adjustments through receipt_task. Offerings with reconciliation_mode consumer_receipt and billing-grade canonicalization require this tier.' ), - ] = False - configuration_task: Literal['sync_accounts'] = 'sync_accounts' - status_task: Literal['get_reporting_status'] = 'get_reporting_status' + ] = None + configuration_task: Literal['sync_accounts'] | None = None + status_task: Literal['get_reporting_status'] | None = None consumer_status_task: Annotated[ - Literal['sync_reporting_status'], + Literal['sync_reporting_status'] | None, Field( description='Opt-in consumer-status loop during the published migration window, becoming required Core in the next eligible minor after that window. Buyers call this seller-hosted task to record whether each expected reporting period was received, missing, or unreadable. Buyers expose no reverse endpoint, and the status is not a billing receipt.' ), - ] = 'sync_reporting_status' + ] = None revision_content_task: Annotated[ - Literal['get_media_buy_delivery'], + Literal['get_media_buy_delivery'] | None, Field( description='Reliable Reporting exact-content read: callers select reporting_revision_id and receive immutable revision metadata plus authoritative canonical reporting_rows.' ), - ] = 'get_media_buy_delivery' + ] = None receipt_task: Annotated[ - Literal['sync_reporting_receipts'], + Literal['sync_reporting_receipts'] | None, Field( description='Required when reconciled_billing is true: the task consumers call to submit and read back authenticated revision and adjustment receipts.' ), - ] = 'sync_reporting_receipts' + ] = None readiness_notification: Annotated[ - Literal['reporting.delivery_ready'], + Literal['reporting.delivery_ready'] | None, Field( description='Optional managed-delivery-only positive-readiness doorbell. It names a materialization at a destination, so Core sellers MUST omit it.' ), - ] = 'reporting.delivery_ready' + ] = None status_notification: Annotated[ - Literal['reporting.status_changed'], + Literal['reporting.status_changed'] | None, Field( description='Optional tier-independent invalidation doorbell for health transitions in either direction, including clock-driven waiting-to-delayed and delayed-to-action_required. Valid for Core: it names no destination. Polling status_task remains the authoritative recovery path whether or not this is offered.' ), - ] = 'reporting.status_changed' + ] = None ledger_notification: Annotated[ - Literal['reporting.ledger_changed'], + Literal['reporting.ledger_changed'] | None, Field( description='Optional tier-independent invalidation for every newly committed revision or post-official adjustment, even when health does not change. Receivers repair through get_reporting_status changes_after; polling remains authoritative.' ), - ] = 'reporting.ledger_changed' + ] = None offerings: Annotated[ list[Offering], Field( @@ -4733,7 +4736,7 @@ class ReportingDelivery(AdCPBaseModel): ge=1, ), ] = None - supports_webhook_activity: bool | None = False + supports_webhook_activity: bool | None = None authorization_revocation_seconds: Annotated[ int | None, Field( @@ -4742,6 +4745,22 @@ class ReportingDelivery(AdCPBaseModel): ), ] = None + @model_validator(mode='after') + def _validate_reporting_tiers(self) -> ReportingDelivery: + if self.reconciled_billing is True and self.managed_delivery is not True: + raise ValueError('reconciled_billing requires managed_delivery') + if self.readiness_notification is not None and self.managed_delivery is not True: + raise ValueError('readiness_notification requires managed_delivery') + if self.receipt_task is not None and self.reconciled_billing is not True: + raise ValueError('receipt_task requires reconciled_billing') + return self + + @model_serializer(mode='wrap') + def _omit_absent_reporting_promises( + self, handler: SerializerFunctionWrapHandler + ) -> dict[str, Any]: + return {key: value for key, value in handler(self).items() if value is not None} + class TranslationTarget(AdCPBaseModel): model_config = ConfigDict( diff --git a/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py b/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py index e3dd37398..0cf914ecb 100644 --- a/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py +++ b/src/adcp/types/generated_poc/core/reporting_delivery_capabilities.py @@ -1,9 +1,12 @@ # generated by datamodel-codegen: # filename: core/reporting_delivery_capabilities.json -# timestamp: 2026-09-14T14:16:02+00:00 +# timestamp: 2026-09-18T13:06:39+00:00 from __future__ import annotations +from typing import Any +from pydantic import SerializerFunctionWrapHandler, model_serializer, model_validator + from typing import Annotated, Literal from adcp.types.base import AdCPBaseModel @@ -38,61 +41,61 @@ class ReportingDeliveryCapabilities(AdCPBaseModel): ) supported: Literal[True] reliable_reporting_version: Annotated[ - Literal['1.0'], + Literal['1.0'] | None, Field( description='Explicit adoption declaration for the proper-name AdCP 3.2 Reliable Reporting contract. Presence, together with supported: true and the media_buy.reporting_delivery experimental feature gate, is the affirmative machine-readable answer. Absence denotes the earlier experimental managed-reporting shape.' ), - ] = '1.0' + ] = None managed_delivery: Annotated[ bool | None, Field( description='Tier flag: this seller supports managed file, dataset-share, or warehouse delivery. Offerings whose method names a delivery pattern require this tier. When false or absent, every offering is API-delivered and Core-only.' ), - ] = False + ] = None reconciled_billing: Annotated[ bool | None, Field( description='Tier flag: this seller supports canonical-digest verification and authenticated consumer receipts for both report materializations and post-official adjustments through receipt_task. Offerings with reconciliation_mode consumer_receipt and billing-grade canonicalization require this tier.' ), - ] = False - configuration_task: Literal['sync_accounts'] = 'sync_accounts' - status_task: Literal['get_reporting_status'] = 'get_reporting_status' + ] = None + configuration_task: Literal['sync_accounts'] | None = None + status_task: Literal['get_reporting_status'] | None = None consumer_status_task: Annotated[ - Literal['sync_reporting_status'], + Literal['sync_reporting_status'] | None, Field( description='Opt-in consumer-status loop during the published migration window, becoming required Core in the next eligible minor after that window. Buyers call this seller-hosted task to record whether each expected reporting period was received, missing, or unreadable. Buyers expose no reverse endpoint, and the status is not a billing receipt.' ), - ] = 'sync_reporting_status' + ] = None revision_content_task: Annotated[ - Literal['get_media_buy_delivery'], + Literal['get_media_buy_delivery'] | None, Field( description='Reliable Reporting exact-content read: callers select reporting_revision_id and receive immutable revision metadata plus authoritative canonical reporting_rows.' ), - ] = 'get_media_buy_delivery' + ] = None receipt_task: Annotated[ - Literal['sync_reporting_receipts'], + Literal['sync_reporting_receipts'] | None, Field( description='Required when reconciled_billing is true: the task consumers call to submit and read back authenticated revision and adjustment receipts.' ), - ] = 'sync_reporting_receipts' + ] = None readiness_notification: Annotated[ - Literal['reporting.delivery_ready'], + Literal['reporting.delivery_ready'] | None, Field( description='Optional managed-delivery-only positive-readiness doorbell. It names a materialization at a destination, so Core sellers MUST omit it.' ), - ] = 'reporting.delivery_ready' + ] = None status_notification: Annotated[ - Literal['reporting.status_changed'], + Literal['reporting.status_changed'] | None, Field( description='Optional tier-independent invalidation doorbell for health transitions in either direction, including clock-driven waiting-to-delayed and delayed-to-action_required. Valid for Core: it names no destination. Polling status_task remains the authoritative recovery path whether or not this is offered.' ), - ] = 'reporting.status_changed' + ] = None ledger_notification: Annotated[ - Literal['reporting.ledger_changed'], + Literal['reporting.ledger_changed'] | None, Field( description='Optional tier-independent invalidation for every newly committed revision or post-official adjustment, even when health does not change. Receivers repair through get_reporting_status changes_after; polling remains authoritative.' ), - ] = 'reporting.ledger_changed' + ] = None offerings: Annotated[ list[reporting_delivery_offering.ReportingDeliveryOffering], Field( @@ -140,7 +143,7 @@ class ReportingDeliveryCapabilities(AdCPBaseModel): ge=1, ), ] = None - supports_webhook_activity: bool | None = False + supports_webhook_activity: bool | None = None authorization_revocation_seconds: Annotated[ int | None, Field( @@ -148,3 +151,19 @@ class ReportingDeliveryCapabilities(AdCPBaseModel): ge=0, ), ] = None + + @model_validator(mode='after') + def _validate_reporting_tiers(self) -> ReportingDeliveryCapabilities: + if self.reconciled_billing is True and self.managed_delivery is not True: + raise ValueError('reconciled_billing requires managed_delivery') + if self.readiness_notification is not None and self.managed_delivery is not True: + raise ValueError('readiness_notification requires managed_delivery') + if self.receipt_task is not None and self.reconciled_billing is not True: + raise ValueError('receipt_task requires reconciled_billing') + return self + + @model_serializer(mode='wrap') + def _omit_absent_reporting_promises( + self, handler: SerializerFunctionWrapHandler + ) -> dict[str, Any]: + return {key: value for key, value in handler(self).items() if value is not None} diff --git a/src/adcp/validation/schema_loader.py b/src/adcp/validation/schema_loader.py index 2b289e292..760ed96ce 100644 --- a/src/adcp/validation/schema_loader.py +++ b/src/adcp/validation/schema_loader.py @@ -31,7 +31,7 @@ import threading import warnings from copy import deepcopy -from datetime import datetime +from datetime import date from importlib.resources import as_file, files from pathlib import Path from typing import Any, Literal @@ -56,7 +56,8 @@ r"^\d{4}-\d{2}-\d{2}[Tt]" r"(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d" r"(?:\.\d+)?" - r"(?:[Zz]|[+-](?:[01]\d|2[0-3]):[0-5]\d)$" + r"(?:[Zz]|[+-](?:[01]\d|2[0-3]):[0-5]\d)$", + re.ASCII, ) @@ -73,9 +74,12 @@ def _is_rfc3339_date_time(instance: Any) -> bool: return True if _RFC3339_DATE_TIME.fullmatch(instance) is None: return False - normalized = instance[:-1] + "+00:00" if instance.endswith(("Z", "z")) else instance try: - datetime.fromisoformat(normalized) + # The grammar already checks time and offset ranges. Validate only + # the calendar here: Python 3.10's datetime parser accepts only three + # or six fractional digits, whereas RFC 3339 permits any positive + # number. Validation must neither coerce nor truncate the wire value. + date.fromisoformat(instance[:10]) except ValueError: return False return True @@ -284,7 +288,7 @@ def _ensure_state(version: str | None = None) -> _LoaderState | None: def _load_schema_registry(state: _LoaderState) -> None: - """Register every modular schema by canonical ``$id``. + """Register every modular schema by its bundle path and canonical ``$id``. Older bundles mostly referenced files under ``core/``, so loading only that directory was sufficient. AdCP 3.2.0-beta.4 introduced canonical @@ -305,6 +309,13 @@ def _load_schema_registry(state: _LoaderState) -> None: except (OSError, json.JSONDecodeError) as exc: logger.warning("Failed to load core schema %s: %s", file, exc) continue + # New modular documents may omit $id while their callers still use + # canonical bundle URLs. Resolve those documents locally too: fetching + # a different remote representation can break their fragment refs. + canonical_path = f"/schemas/{state.bundle_key}/{relative.as_posix()}" + state.registry[f"https://adcontextprotocol.org{canonical_path}"] = schema + state.registry[f"file://{canonical_path}"] = schema + state.registry[file.resolve().as_uri()] = schema schema_id = schema.get("$id") if isinstance(schema_id, str): state.registry[schema_id] = schema @@ -350,7 +361,7 @@ def _make_ref_resolver(state: _LoaderState, base_file: Path, schema: dict[str, A ) from exc _load_schema_registry(state) - base_uri = base_file.resolve().parent.as_uri() + "/" + base_uri = base_file.resolve().as_uri() return RefResolver(base_uri=base_uri, referrer=schema, store=dict(state.registry)) @@ -454,6 +465,45 @@ def visit(value: Any, *, root: bool = False) -> None: return normalized +def _effective_task_schema( + schema: dict[str, Any], tool_name: str, direction: Direction, *, bundle_key: str +) -> dict[str, Any]: + """Apply the SDK's captured-schedule contract without rewriting signed bundles. + + Reporting health describes existing evidence; it does not cancel a frozen + generation's future commitment (#1179). The 3.2.0-rc.3 status schema couples + the two at /allOf/2/then/not. Correct only that exact known rule and version. + Its if, scope closure and coverage requirements remain unchanged. A changed + or different-version rule is left intact for explicit compatibility review. + """ + if (tool_name, direction, bundle_key) != ("get_reporting_status", "sync", "3.2.0-rc.3"): + return schema + known_rule = { + "if": { + "properties": {"health": {"const": "complete"}}, + "required": ["health"], + }, + "then": { + "properties": { + "scope": { + "properties": { + "scope_closed": {"const": True}, + "coverage_complete": {"const": True}, + }, + "required": ["scope_closed", "coverage_complete"], + } + }, + "not": {"required": ["next_expected_at"]}, + }, + } + conditions = schema.get("allOf") + if not isinstance(conditions, list) or len(conditions) < 3 or conditions[2] != known_rule: + return schema + result = deepcopy(schema) + del result["allOf"][2]["then"]["not"] + return result + + def get_validator( tool_name: str, direction: Direction, @@ -490,6 +540,7 @@ def get_validator( return None if file.is_relative_to(state.root.bundled): schema = _normalize_bundled_schema_for_validation(schema) + schema = _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key) try: from jsonschema import Draft7Validator, FormatChecker @@ -579,7 +630,7 @@ def get_named_validator( from jsonschema import RefResolver resolver = RefResolver( - base_uri=file.resolve().parent.as_uri() + "/", + base_uri=file.resolve().as_uri(), referrer=schema, store=_reachable_schema_store(state, file, schema), ) @@ -628,7 +679,9 @@ def get_schema( if not isinstance(schema, dict): logger.warning("Schema %s is not a JSON object", file) return None - return deepcopy(schema) + return deepcopy( + _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key) + ) def get_named_schema_document( @@ -720,7 +773,11 @@ def get_portable_schema( schema = json.loads(file.read_text()) if not isinstance(schema, dict): raise ValueError("schema root is not an object") - portable = _self_contained_schema(state, file, schema) + portable = _self_contained_schema( + state, + file, + _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key), + ) except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: logger.warning("Failed to make schema %s portable for %s: %s", file, key, exc) return None @@ -873,7 +930,11 @@ def get_mcp_schema( logger.warning("MCP schema %s is not a JSON object", file) return None try: - portable = _self_contained_schema(state, file, schema) + portable = _self_contained_schema( + state, + file, + _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key), + ) except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: logger.warning("Failed to make MCP schema %s portable: %s", file, exc) return None diff --git a/tests/conformance/reporting/_production_delivery_process.py b/tests/conformance/reporting/_production_delivery_process.py new file mode 100644 index 000000000..4e92e2b7b --- /dev/null +++ b/tests/conformance/reporting/_production_delivery_process.py @@ -0,0 +1,133 @@ +"""Real SIGKILL after receiver acceptance; cold retry uses the durable deadline.""" + +import asyncio +import hashlib +import json +import sys +from datetime import datetime +from pathlib import Path +from types import SimpleNamespace + + +async def main(settings): + import pytest + from psycopg_pool import AsyncConnectionPool + + import adcp.reporting.production.delivery_window as window_module + from adcp.reporting.outbox.routing import ReportingEnvelopeCipher + from adcp.reporting.production.notifications import ( + ReportingProductionSigning, + production_notification_workers, + ) + from adcp.reporting.production.pg import PgReportingProductionStore + from adcp.reporting.projection.pg import PgReportingStatusProjection + + from ._reliable_support import ( + Barrier, + DeterministicReceiverStore, + FailurePlan, + ManualClock, + ScriptedNotificationReceiver, + ScriptedSigning, + ScriptedSubscriptions, + _BytesStore, + notification_subscription, + ) + from .test_reporting_production_notifications import EVENTS, retained_windows + + origin = Path(window_module.__file__).resolve() + assert hashlib.sha256(origin.read_bytes()).hexdigest() == settings["module_sha256"] + if settings["installed"]: + assert origin.is_relative_to(Path(sys.prefix)) and "site-packages" in str(origin) + clock = ManualClock(datetime.fromisoformat(settings["at"])) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + store = PgReportingProductionStore(pool=pool, clock=clock, notifications=True) + await store.create_schema() + projection = PgReportingStatusProjection(store, revision_ownership=True) + failures = FailurePlan() + subscriptions = ScriptedSubscriptions(failures) + subscriptions.put( + notification_subscription( + subscriber="buyer", + principal=settings["consumer"], + events=EVENTS, + url="https://receiver.example.test/reporting", + ) + ) + resolver = ScriptedSigning(failures) + resolver.generation = 1 if settings["pause"] else 2 + signing = ReportingProductionSigning( + resolver, ("ed25519",), brand_json_url="https://seller.example.test/brand.json" + ) + worker = production_notification_workers( + store, + projection, + subscriptions=subscriptions, + cipher=ReportingEnvelopeCipher(b"b" * 32), + signing=signing, + )[settings["queue"]] + blobs = _BytesStore(pool) + await blobs.create_schema() + received = DeterministicReceiverStore(blobs, failures) + receiver = ScriptedNotificationReceiver( + SimpleNamespace(clock=clock, failures=failures, receiver=received) + ) + with pytest.MonkeyPatch.context() as patch: + receiver.install(patch) + if settings["pause"]: + accepted = Barrier() + failures.at("http.accepted", accepted) + task = asyncio.create_task(worker.deliver_one(account_id="acct_a")) + await accepted.wait() + windows = await retained_windows(SimpleNamespace(pool=pool)) + assert len(windows) == 1 + row = windows[0] + print( + json.dumps( + { + "point": "accepted_before_ack", + "started_at": row[4].isoformat(), + "expires_at": row[5].isoformat(), + "key_sha256": hashlib.sha256(row[1].encode()).hexdigest(), + "body_sha256": row[3], + "origin": str(origin), + } + ), + flush=True, + ) + await asyncio.to_thread(sys.stdin.readline) + accepted.release() + await task + raise AssertionError("paused child must be killed") + before = await retained_windows(SimpleNamespace(pool=pool)) + assert await worker.deliver_one(account_id="acct_a") + assert await retained_windows(SimpleNamespace(pool=pool)) == before + row = before[0] + states = await worker.outbox.list_deliveries(account_id="acct_a") + target = next(s for s in states if s.delivery.binding.idempotency_key == row[1]) + body = await received.read("acct_a", row[1]) + assert body is not None and hashlib.sha256(body).hexdigest() == row[3] + attempts = await worker.outbox.list_activity( + account_id="acct_a", consumer_id=settings["consumer"] + ) + result = { + "point": "done", + "http_calls": len(receiver.received), + "state": target.state, + "error_code": target.error_code, + "activity_count": len(attempts), + "key_sha256": hashlib.sha256(row[1].encode()).hexdigest(), + "body_sha256": row[3], + "started_at": row[4].isoformat(), + "expires_at": row[5].isoformat(), + "origin": str(origin), + } + if receiver.received: + assert "key-2" in receiver.received[0].headers["signature-input"] + return result + + +if __name__ == "__main__": + print(json.dumps(asyncio.run(main(json.loads(sys.stdin.readline())))), flush=True) diff --git a/tests/conformance/reporting/_production_installed.py b/tests/conformance/reporting/_production_installed.py new file mode 100644 index 000000000..128b489ae --- /dev/null +++ b/tests/conformance/reporting/_production_installed.py @@ -0,0 +1,142 @@ +"""Floor-runtime conformance with complete installed module/schema provenance.""" + +import contextlib +import hashlib +import importlib +import importlib.metadata +import importlib.util +import json +import os +import subprocess +import sys +import time +from importlib.resources import files +from pathlib import Path + +import pytest + + +def main(settings): + root = Path(settings["fixtures"]) + workspace = Path(settings["workspace"]) + assert sys.version_info[:2] == tuple(settings["python"]) + assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) + assert not (root / "adcp").exists() and not (root / "src").exists() + sys.path.insert(0, str(root)) + from tests.conformance.reporting._hardening_installed import Results + + origins = {} + for name, expected in settings["modules"].items(): + path = Path(importlib.import_module(name).__file__).resolve() + assert path.is_relative_to(Path(sys.prefix)) and not path.is_relative_to(workspace) + assert hashlib.sha256(path.read_bytes()).hexdigest() == expected + origins[name] = str(path) + for name, expected in settings["assets"].items(): + assert ( + hashlib.sha256(files("adcp.reporting").joinpath(name).read_bytes()).hexdigest() + == expected + ) + from adcp.validation import schema_loader + + schema_root = schema_loader._resolve_schema_root("3.2.0-rc.3").root + assert schema_root.is_relative_to(Path(sys.prefix)) + for name, expected in settings["schemas"].items(): + assert hashlib.sha256((schema_root / name).read_bytes()).hexdigest() == expected + if settings["driver_absent"]: + assert importlib.util.find_spec("psycopg") is None + assert importlib.util.find_spec("psycopg_pool") is None + os.environ.pop("ADCP_PG_TEST_URL", None) + else: + assert os.environ.get("ADCP_PG_TEST_URL") + evidence = Path(settings["evidence"]) + evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + log = evidence / (settings["label"] + ".log") + recorder = Results() + command = [ + *(str(root / path) for path in settings["tests"]), + "-v", + "-s", + "-ra", + "-o", + "asyncio_mode=auto", + "-p", + "no:cacheprovider", + "--basetemp", + str(root / "temp"), + "--deselect=tests/test_reporting_capability_models.py::test_post_generation_repair_is_idempotent_for_both_actual_model_layouts", + ] + started = time.monotonic() + with ( + log.open("x") as stream, + contextlib.redirect_stdout(stream), + contextlib.redirect_stderr(stream), + ): + code = int(pytest.main(command, plugins=[recorder])) + result = { + "command": command, + "pytest_exit": code, + "passed": recorder.passed, + "failed": recorder.failed, + "errors": recorder.errors, + "skipped": recorder.skipped, + "deselected": recorder.deselected, + "seconds": round(time.monotonic() - started, 3), + "log": str(log), + "bytes": log.stat().st_size, + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + "valid": code == 0 and recorder.failed == recorder.errors == 0 and recorder.passed > 0, + } + if settings["driver_absent"]: + assert recorder.skipped > 0 + else: + result["valid"] &= recorder.skipped == 0 + result["valid"] &= recorder.deselected == 1 + config = root / "mypy.ini" + config.write_text( + "[mypy]\npython_version=3.10\nstrict=True\nplugins=adcp.types.mypy_plugin\nfollow_imports=silent\n" + ) + typing_command = [ + sys.executable, + "-I", + "-m", + "mypy", + "--config-file", + str(config), + "--strict", + "--no-incremental", + str(root / "adopter.py"), + ] + typed = subprocess.run(typing_command, cwd=root, capture_output=True, timeout=120) + typing_log = evidence / (settings["label"] + "-adopter.log") + typing_log.write_bytes(typed.stdout + typed.stderr) + result["valid"] &= typed.returncode == 0 + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): + assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix)) + record = { + "python": sys.version, + "source_basis": settings["source_basis"], + "direct_url": json.loads( + importlib.metadata.distribution("adcp").read_text("direct_url.json") + ), + "origins": origins, + "distribution_version": importlib.metadata.version("adcp"), + "wheel_sha256": settings["wheel_sha256"], + "assets": settings["assets"], + "schemas": settings["schemas"], + "driver_absent": settings["driver_absent"], + "result": result, + "adopter": { + "command": typing_command, + "exit": typed.returncode, + "log": str(typing_log), + "bytes": typing_log.stat().st_size, + "sha256": hashlib.sha256(typing_log.read_bytes()).hexdigest(), + }, + } + (evidence / (settings["label"] + ".json")).write_text(json.dumps(record, indent=2) + "\n") + print(json.dumps(record), flush=True) + + +if __name__ == "__main__": + main(json.load(sys.stdin)) diff --git a/tests/conformance/reporting/_production_installed_process.py b/tests/conformance/reporting/_production_installed_process.py new file mode 100644 index 000000000..8db3c347e --- /dev/null +++ b/tests/conformance/reporting/_production_installed_process.py @@ -0,0 +1,283 @@ +"""Installed activation, SIGKILL, and immutable legacy/new representation walks.""" + +import asyncio +import hashlib +import importlib +import json +import sys +from dataclasses import asdict +from importlib.resources import files +from pathlib import Path +from types import SimpleNamespace + + +async def main(settings): + root = Path(settings["fixtures"]) + assert not (root / "src").exists() and not (root / "adcp").exists() + sys.path.insert(0, str(root)) + assert sys.version_info[:2] == (3, 10) + origins = {} + for name, digest in settings["modules"].items(): + path = Path(importlib.import_module(name).__file__).resolve() + assert path.is_relative_to(Path(sys.prefix)) + assert hashlib.sha256(path.read_bytes()).hexdigest() == digest + origins[name] = str(path) + for name, digest in settings["assets"].items(): + assert ( + hashlib.sha256(files("adcp.reporting").joinpath(name).read_bytes()).hexdigest() + == digest + ) + + from psycopg_pool import AsyncConnectionPool + from pydantic import TypeAdapter + + from adcp.reporting.ledger import ReportingMaterializationAttempt + from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal + from adcp.reporting.ownership import page_revision_ownership + from tests.conformance.reporting._production_support import production_harness + from tests.conformance.reporting._production_transport import MountedProduction + from tests.conformance.reporting._projection_support import drain + from tests.conformance.reporting.test_reporting_production_lock_order import source_turn + + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + async with production_harness( + "postgres", + Path(settings["destination"]), + notifications=settings["notifications"], + count=0, + reconciled=True, + identity_prefix="b24-", + existing_pool=pool, + ) as h: + mount = MountedProduction(h) + subject = SimpleNamespace( + obligation=SimpleNamespace(account_id=settings["caller"]["account_id"]), + binding=SimpleNamespace(consumer_id=settings["caller"]["consumer_id"]), + ) + mount.authorize(subject) + historical = settings["historical_pending"] + original_attempt = TypeAdapter(ReportingMaterializationAttempt).validate_python( + historical["attempt"] + ) + original_id = original_attempt.reporting_materialization_id + + async def pending_identity(): + async with pool.connection() as c: + row = await ( + await c.execute( + "SELECT external_id,generation,admission_epoch,state" + " FROM reporting_materializer_work" + " WHERE account_id=%s AND consumer_id=%s" + " AND reporting_materialization_id=%s", + ( + original_attempt.scope.principal.account_id, + original_attempt.scope.consumer_id, + original_id, + ), + ) + ).fetchone() + assert row[:3] == ( + historical["external_id"], + historical["generation"], + historical["epoch"], + ) + records = await h.store.read_reconciliation_snapshot( + caller=original_attempt.scope.principal + ) + assert original_attempt in records.records + return row[3] + + if settings["pause"]: + assert await pending_identity() == "pending" + assert await h.production.activate(account_id="acct_a") + # Run the real producer's lease acquisition/release after + # activation while the actual parent's attempt is pending. + # The already-killed parent's lease deliberately spans setup. + # Persist its expiry as a bounded crash-recovery fault, without + # changing any original attempt, generation or external identity. + assert (await source_turn(h.production)).leased is not None + async with pool.connection() as c, c.transaction(): + account = original_attempt.scope.principal.account_id + await h.store._lock_account(c, account) + await c.execute( + "UPDATE reporting_materializer_work SET lease_until=clock_timestamp()," + " due_at=clock_timestamp() WHERE account_id=%s AND consumer_id=%s" + " AND reporting_materialization_id=%s AND state='pending'", + (account, original_attempt.scope.consumer_id, original_id), + ) + await c.execute( + "UPDATE reporting_materializer_accounts SET due_at=clock_timestamp()" + " WHERE account_id=%s", + (account,), + ) + for _ in range(40): + turn = await h.production.materializer.run_once() + if turn.state == "verified": + assert turn.reporting_materialization_id == original_id + break + assert turn.state in {"idle", "discovered", "parked", "pending"} + await asyncio.sleep(0.05) + else: + raise AssertionError("historical pending work did not resume to verified") + assert h.item.writer.writes == 1 + assert await pending_identity() == "acked" + async with pool.connection() as c: + rows = await ( + await c.execute( + "SELECT (SELECT count(*)" + " FROM reporting_materializer_notification_events" + " WHERE reporting_materialization_id=%s AND admission_epoch=0)," + " (SELECT count(*)" + " FROM reporting_materializer_notification_expansions x" + " JOIN reporting_materializer_notification_events e" + " USING(account_id,consumer_namespace,notification_id)" + " WHERE e.reporting_materialization_id=%s AND x.state='quarantined')," + " (SELECT count(*) FROM reporting_production_notification_events" + " WHERE reporting_materialization_id=%s)", + (original_id,) * 3, + ) + ).fetchone() + assert rows == (int(settings["notifications"]), int(settings["notifications"]), 0) + await drain(h.projection, "acct_a") + else: + assert not await h.production.activate(account_id="acct_a") + assert await pending_identity() == "acked" + # A new eligible period was committed after both first pages. + # Finish it through bounded real turns if startup first handled + # another candidate. Neither snapshot may gain that membership. + if "new_revision_after_snapshot" in settings: + for _ in range(32): + outcomes = [ + r + for r in await h.item.outcomes() + if r.scope.generation_key == h.item.scope.generation_key + ] + if any( + r.reporting_revision_id == settings["new_revision_after_snapshot"] + and r.status == "delivered" + for r in outcomes + ): + break + await h.production.materializer.run_once() + else: + raise AssertionError("new eligible period did not complete after restart") + assert {r.reporting_revision_id for r in outcomes} == { + h.item.revision.reporting_revision_id, + settings["new_revision_after_snapshot"], + } + assert h.item.writer.writes == int("new_revision_after_snapshot" in settings) + caller = ReportingDeliveryPrincipal(**settings["caller"]) + query = { + "account": {"account_id": caller.account_id}, + "view": "periods", + "pagination": {"max_results": 1}, + } + + async def walk(client, request, transport): + pages = [] + request = json.loads(json.dumps(request)) + for _ in range(1000): + _, page = await mount.call( + client, "get_reporting_status", request, transport=transport + ) + assert "pagination" in page + pages.append(page) + if not page["pagination"]["has_more"]: + break + request["pagination"]["cursor"] = page["pagination"]["cursor"] + else: + raise AssertionError("installed cursor walk exceeded its bound") + assert len({p["changes_checkpoint"] for p in pages}) == 1 + snapshot = await h.store.read_reporting_feed_snapshot( + pages[0]["ledger_snapshot_id"], caller=caller + ) + return { + "pages": pages, + "binding": snapshot.binding, + "version": snapshot.representation_version, + "ownership_mode": snapshot.ownership_mode, + } + + async with mount.client() as client: + if settings["pause"]: + _, new_first = await mount.call(client, "get_reporting_status", query) + assert page_revision_ownership(new_first) is not None + expected_new = await walk( + client, + { + **query, + "pagination": { + "max_results": 1, + "cursor": new_first["pagination"]["cursor"], + }, + }, + "mcp", + ) + print( + json.dumps( + { + "point": "activated", + "origins": origins, + "first": new_first, + "new_remaining": expected_new, + "verification_key": asdict(h.item.verifier.key), + "pending_continuation": { + "state": "verified", + "epoch": 0, + "external_id": historical["external_id"], + "generation": historical["generation"], + "materialization_id": original_id, + "original_attempt_unchanged": True, + "original_quarantine_preserved": True, + "expiry_control": "persisted expiry after parent SIGKILL", + }, + } + ), + flush=True, + ) + await asyncio.to_thread(sys.stdin.readline) + raise AssertionError("activated process must be killed") + old_walks = [] + new_walks = [] + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps["media_buy"]["reporting_delivery"]["reconciled_billing"] is True + old_walks.append(await walk(client, settings["continuation"], transport)) + new_walks.append(await walk(client, settings["new_continuation"], transport)) + _, replay = await mount.call( + client, + "sync_reporting_receipts", + settings["receipt_request"], + transport=transport, + ) + assert replay == settings["receipt_response"] + assert old_walks[0] == old_walks[1] == old_walks[2] + assert new_walks[0] == new_walks[1] == new_walks[2] + assert all(page_revision_ownership(p) is None for p in old_walks[0]["pages"]) + assert all(page_revision_ownership(p) is not None for p in new_walks[0]["pages"]) + mount.grants.clear() + h.authorized_bindings.clear() + _, refused = await mount.call( + client, "get_reporting_status", settings["continuation"] + ) + assert "UNAUTHORIZED" in json.dumps(refused) + for name, module in tuple(sys.modules.items()): + if (name == "adcp" or name.startswith("adcp.")) and getattr( + module, "__file__", None + ): + assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix)) + return { + "point": "done", + "legacy": old_walks[0], + "new": new_walks[0], + "origins": origins, + "fresh_external_writes": h.item.writer.writes, + } + + +if __name__ == "__main__": + print(json.dumps(asyncio.run(main(json.loads(sys.stdin.readline())))), flush=True) diff --git a/tests/conformance/reporting/_production_legacy_process.py b/tests/conformance/reporting/_production_legacy_process.py new file mode 100644 index 000000000..2328e2ebb --- /dev/null +++ b/tests/conformance/reporting/_production_legacy_process.py @@ -0,0 +1,137 @@ +"""Executed by the real historical binary: incompatible C writers fail closed.""" + +import asyncio +import json +import sys +from dataclasses import asdict +from datetime import datetime, timezone +from pathlib import Path + + +async def main(settings): + import hashlib + + from psycopg_pool import AsyncConnectionPool + from pydantic import TypeAdapter + + import adcp.reporting.materializer.pg as materializer + import adcp.reporting.outbox.status_pg as implementation + from adcp.reporting.feed import PgReportingFeedStore + from adcp.reporting.materializer import ReportingVerificationKey + from adcp.reporting.materializer.work import ReportingMaterializerLease + from adcp.reporting.outbox.status_pg import PgStatusNotificationStore + + path = Path(implementation.__file__).resolve() + assert path.is_relative_to(Path(sys.prefix)) + assert hashlib.sha256(path.read_bytes()).hexdigest() == settings["module_sha256"] + materializer_path = Path(materializer.__file__).resolve() + assert materializer_path.is_relative_to(Path(sys.prefix)) + assert ( + hashlib.sha256(materializer_path.read_bytes()).hexdigest() + == settings["materializer_module_sha256"] + ) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=1, max_size=1, open=False + ) as pool: + store = PgReportingFeedStore( + pool=pool, + clock=lambda: datetime(2099, 1, 1, tzinfo=timezone.utc), + # This is an intentionally incompatible historical notification + # writer, even when the new deployment uses polling only. + notifications=settings["notifications"] if settings.get("pending") else True, + ) + key = TypeAdapter(ReportingVerificationKey).validate_python(settings["verification_key"]) + await store.materializer_ready() + if settings.get("pending"): + # The actual parent binary reserves on its original schema before + # any B2.4 migration. Its durable identity, not a reconstruction, + # must survive the process being killed and the child's activation. + for _ in range(16): + lease = await store.claim_materialization(keys=(key,), lease_seconds=300) + if isinstance(lease, ReportingMaterializerLease): + assert lease.attempt.reporting_revision_id == settings["revision_id"] + # The historical public lease has no epoch attribute. + # Read its real durable work row instead of assuming a + # child's newer dataclass shape or inventing a default. + async with pool.connection() as c: + identity = await ( + await c.execute( + "SELECT admission_epoch,generation,external_id" + " FROM reporting_materializer_work WHERE account_id=%s" + " AND consumer_id=%s AND reporting_materialization_id=%s", + ( + lease.scope.principal.account_id, + lease.scope.consumer_id, + lease.attempt.reporting_materialization_id, + ), + ) + ).fetchone() + assert identity == (0, lease.generation, lease.request.external_id) + return { + "point": "pending", + "attempt": TypeAdapter(type(lease.attempt)).dump_python( + lease.attempt, mode="json" + ), + "external_id": lease.request.external_id, + "generation": lease.generation, + "epoch": identity[0], + "lease_expires_at": lease.expires_at.isoformat(), + "scope": TypeAdapter(type(lease.scope)).dump_python( + lease.scope, mode="json" + ), + "verification_key": asdict(key), + "materializer_origin": str(materializer_path), + "origin": str(path), + } + raise AssertionError("actual parent did not reserve eligible pending work") + old = PgStatusNotificationStore(store) + checkpoints = await old.checkpoints(account_id="acct_a") + assert checkpoints + try: + async with old._transaction("acct_a") as connection: + await old._write_on(connection, checkpoints[0]) + except Exception as error: + assert getattr(error, "sqlstate", None) == "23514" + assert "status_projection_writer_fenced" in str(error) + else: + raise AssertionError("historical projector changed a v2 boundary") + try: + lease = await old.claim_due(account_id="acct_a") + except Exception as error: + # A legacy policy refusal or the actual row trigger is closed. + assert ( + getattr(error, "sqlstate", None) == "23514" + or getattr(error, "code", None) == "status_policy_conflict" + ) + sweeper = "fenced" + else: + assert lease is None + sweeper = "no_mutation" + try: + async with pool.connection() as connection, connection.transaction(): + await store._lock_account(connection, "acct_a") + # Execute the historical reserve algorithm, including its real + # discovery and selection. Keep the bounded turns in one + # transaction so the trigger's refusal proves full rollback. + for _ in range(16): + turn = await store._claim_account_on(connection, "acct_a", (key,), 30) + assert not hasattr(turn, "token"), "historical worker acquired new work" + raise AssertionError("historical reservation did not reach the production fence") + except Exception as error: + assert getattr(error, "sqlstate", None) == "23514" + assert "reporting_production_old_worker_fenced" in str(error) + return { + "historical_projection": "trigger_fenced", + "historical_sweeper": sweeper, + "historical_materializer": "reservation_trigger_fenced", + "materializer_origin": str(materializer_path), + "origin": str(path), + } + + +if __name__ == "__main__": + settings = json.loads(sys.stdin.readline()) + print(json.dumps(asyncio.run(main(settings))), flush=True) + if settings.get("pending"): + sys.stdin.readline() + raise AssertionError("historical pending worker must be killed") diff --git a/tests/conformance/reporting/_production_packaging.py b/tests/conformance/reporting/_production_packaging.py new file mode 100644 index 000000000..b6ace99de --- /dev/null +++ b/tests/conformance/reporting/_production_packaging.py @@ -0,0 +1,270 @@ +"""Build and inspect actual production distributions; fixtures never alias SDK source.""" + +import hashlib +import json +import os +import shutil +import subprocess +import tarfile +import zipfile +from pathlib import Path + +from .test_reporting_notification_packaging import ROOT, run_step + +ASSETS = ( + "ledger/reporting_materializer.sql", + "materializer/required_schema.json", + "ledger/reporting_receipt_ingestion.sql", + "receipts/required_schema.json", + "ledger/reporting_feed.sql", + "feed/required_schema.json", + "ledger/reporting_status_notifications.sql", + "outbox/required_status_schema.json", + "ledger/reporting_status_selector_version.sql", + "outbox/required_status_selector_schema.json", + "ledger/reporting_projection.sql", + "ledger/reporting_projection_notifications.sql", + "ledger/reporting_projection_feed.sql", + "projection/required_schema.json", + "ledger/reporting_production.sql", + "production/required_schema.json", +) +SCHEMAS = ( + "core/reporting-delivery-config-state.json", + "media-buy/get-reporting-status-response.json", + "bundled/media-buy/get-reporting-status-response.json", + "mcp/2026-07-28/profiles/production/media-buy/get-reporting-status-response.json", + "protocol/get-adcp-capabilities-response.json", + "bundled/protocol/get-adcp-capabilities-response.json", +) + + +def production_modules(): + paths = [ + p + for part in ("reporting/production", "reporting/projection") + for p in (ROOT / "src/adcp" / part).glob("*.py") + ] + paths += [ + ROOT / "src/adcp" / name + for name in ( + "reporting/ledger/status.py", + "reporting/ledger/status_snapshot.py", + "reporting/ledger/status_projection.py", + "reporting/ledger/reconciliation_projection.py", + "reporting/ledger/schedule.py", + "reporting/ledger/producer.py", + "reporting/ledger/producer_progress.py", + "reporting/materializer/memory.py", + "reporting/materializer/pg.py", + "reporting/materializer/publication.py", + "reporting/materializer/service.py", + "reporting/materializer/verification.py", + "reporting/feed/snapshot.py", + "reporting/feed/projection.py", + "reporting/feed/memory.py", + "reporting/feed/pg.py", + "reporting/ownership.py", + "reporting/_timestamp.py", + "reporting/_reconcile.py", + "reporting/receipts/handler.py", + "reporting/outbox/memory.py", + "reporting/outbox/_activity_pg.py", + "reporting/outbox/worker.py", + "validation/schema_loader.py", + "types/base.py", + "types/generated_poc/core/reporting_delivery_capabilities.py", + "types/generated_poc/bundled/protocol/get_adcp_capabilities_response.py", + ) + ] + return { + "adcp." + + str(p.relative_to(ROOT / "src/adcp")) + .removesuffix(".py") + .replace("/", ".") + .removesuffix(".__init__"): hashlib.sha256(p.read_bytes()) + .hexdigest() + for p in paths + } + + +def inspect_distribution(wheel, source): + modules = production_modules() + assets = {name: (ROOT / "src/adcp/reporting" / name).read_bytes() for name in ASSETS} + with zipfile.ZipFile(wheel) as archive, tarfile.open(source) as sdist: + prefix = sdist.getnames()[0].split("/")[0] + for name, digest in modules.items(): + member = name.replace(".", "/") + ".py" + if member not in archive.namelist(): + member = name.replace(".", "/") + "/__init__.py" + raw = archive.read(member) + assert hashlib.sha256(raw).hexdigest() == digest + assert sdist.extractfile(f"{prefix}/src/{member}").read() == raw + for name, raw in assets.items(): + assert archive.read("adcp/reporting/" + name) == raw + assert sdist.extractfile(f"{prefix}/src/adcp/reporting/{name}").read() == raw + return modules, {name: hashlib.sha256(raw).hexdigest() for name, raw in assets.items()} + + +def copied_fixtures(root, label): + destination = root / ("production-" + label) + destination.mkdir(mode=0o700) + shutil.copytree( + ROOT / "tests", destination / "tests", ignore=shutil.ignore_patterns("__pycache__") + ) + shutil.copy2(ROOT / "examples/reporting_production.py", destination / "adopter.py") + assert not (destination / "adcp").exists() and not (destination / "src").exists() + return destination + + +def source_basis(wheel, source, *, evidence, label): + """Record actual build inputs; a dirty checkout's HEAD is lineage only.""" + + def git(*args): + return subprocess.check_output(["git", *args], cwd=ROOT) + + head, tree = (git("rev-parse", value).decode().strip() for value in ("HEAD", "HEAD^{tree}")) + dirty = bool(git("status", "--porcelain=v1", "-uall")) + members = [] + with tarfile.open(source) as archive: + prefix = archive.getnames()[0].split("/")[0] + "/" + for member in archive.getmembers(): + if not member.isfile(): + continue + relative = Path(member.name.removeprefix(prefix)) + if any(p.endswith(".egg-info") for p in relative.parts): + continue + path = ROOT / relative + if not path.is_file(): + continue # Generated sdist metadata is identified by its archive digest. + raw = archive.extractfile(member).read() + assert path.read_bytes() == raw, relative + members.append( + { + "path": str(relative), + "bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ) + raw_manifest = (json.dumps(sorted(members, key=lambda r: r["path"]), indent=2) + "\n").encode() + evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + manifest = evidence / (label + "-build-inputs.json") + with manifest.open("xb") as stream: + stream.write(raw_manifest) + basis = { + "kind": "development-export" if dirty else "git-commit", + "head": head, + "tree": tree, + "clean": not dirty, + "head_role": "parent lineage only" if dirty else "actual built commit", + "build_input_manifest": str(manifest), + "build_input_count": len(members), + "build_input_manifest_sha256": hashlib.sha256(raw_manifest).hexdigest(), + "sdist_sha256": hashlib.sha256(source.read_bytes()).hexdigest(), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + } + if dirty: + patch = git("diff", "--binary", "HEAD") + patch_file = evidence / (label + "-development.patch") + with patch_file.open("xb") as stream: + stream.write(patch) + basis["patch_sha256"] = hashlib.sha256(patch).hexdigest() + basis["patch"] = str(patch_file) + new_files = { + p.decode(): hashlib.sha256((ROOT / p.decode()).read_bytes()).hexdigest() + for p in git("ls-files", "--others", "--exclude-standard", "-z").split(b"\0") + if p and (ROOT / p.decode()).is_file() + } + new_manifest = evidence / (label + "-development-new-files.json") + raw_new = (json.dumps(new_files, indent=2, sort_keys=True) + "\n").encode() + with new_manifest.open("xb") as stream: + stream.write(raw_new) + basis["new_files_manifest"] = str(new_manifest) + basis["new_files_manifest_sha256"] = hashlib.sha256(raw_new).hexdigest() + if os.environ.get("ADCP_PRODUCTION_EVIDENCE"): + retained = evidence / (label + "-artifacts") + retained.mkdir(mode=0o700) + for path in (wheel, source): + shutil.copy2(path, retained / path.name) + basis["retained_artifacts"] = str(retained) + return basis + + +def installed_production(root, python, wheel, source, *, label, driver_absent): + fixture_root = copied_fixtures(root, label) + script = fixture_root / "run_installed.py" + shutil.copy2(Path(__file__).with_name("_production_installed.py"), script) + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [ + *installer, + "pytest==9.1.1", + "pytest-asyncio==1.4.0", + "respx==0.23.1", + "asgi-lifespan==2.1.0", + "mypy==1.20.2", + ], + label=label + "-production-conformance-dependencies", + cwd=root, + timeout=180, + ) + modules, assets = inspect_distribution(wheel, source) + tests = sorted( + { + *ROOT.glob("tests/conformance/reporting/test_reporting_production*.py"), + *ROOT.glob("tests/conformance/reporting/test_reporting_projection*.py"), + } + ) + tests = [ + p + for p in tests + if p.name + not in {"test_reporting_production_packaging.py", "test_reporting_production_rolling.py"} + ] + tests += [ + ROOT / name + for name in ( + "tests/conformance/reporting/test_reporting_tier_projection.py", + "tests/conformance/reporting/test_reporting_schedule_schema.py", + "tests/test_reporting_revision_ownership.py", + "tests/test_reporting_capability_models.py", + "tests/test_reporting_production_public.py", + "tests/test_schema_datetime_formats.py", + ) + ] + evidence = Path(os.environ.get("ADCP_PRODUCTION_EVIDENCE", str(root / "production-evidence"))) + settings = { + "workspace": str(ROOT), + "fixtures": str(fixture_root), + "label": label, + "modules": modules, + "assets": assets, + "schemas": { + name: hashlib.sha256( + (ROOT / "schemas/cache/3.2.0-rc.3" / name).read_bytes() + ).hexdigest() + for name in SCHEMAS + }, + "tests": [str(p.relative_to(ROOT)) for p in tests], + "driver_absent": driver_absent, + "python": [3, 10], + "source_basis": source_basis(wheel, source, evidence=evidence, label=label), + "wheel_sha256": hashlib.sha256(wheel.read_bytes()).hexdigest(), + "evidence": str(evidence), + } + result = json.loads( + run_step( + [str(python), "-I", str(script)], + label=label + "-installed-production", + cwd=fixture_root, + value=settings, + timeout=1800, + ) + ) + print(json.dumps({"installed_production": label, **result}), flush=True) + assert result["result"]["valid"], result["result"] + return result diff --git a/tests/conformance/reporting/_production_progress_process.py b/tests/conformance/reporting/_production_progress_process.py new file mode 100644 index 000000000..df5c93eca --- /dev/null +++ b/tests/conformance/reporting/_production_progress_process.py @@ -0,0 +1,57 @@ +"""Cold producer restart: committed bounded window, retained lease, then SIGKILL.""" + +import asyncio +import json +import sys +from datetime import timedelta +from pathlib import Path + + +async def main(): + from psycopg_pool import AsyncConnectionPool + + from ._production_support import production_harness + from .test_reporting_production_lock_order import source_turn + from .test_reporting_production_progress import turn_document + + settings = json.loads(await asyncio.to_thread(sys.stdin.readline)) + async with AsyncConnectionPool( + settings["conninfo"], kwargs=settings["kwargs"], min_size=2, max_size=4, open=False + ) as pool: + async with production_harness( + "postgres", Path(settings["path"]), count=0, periods=131, existing_pool=pool + ) as h: + support = h.production + producer = support.offerings[0].producer + source = producer._source + h.source_clock.advance(timedelta(hours=131, seconds=0 if settings["pause"] else 61)) + if settings["pause"]: + original = producer._acquire_pending + + async def pause(configuration, turn, *, now): + await original(configuration, turn, now=now) + print( + json.dumps( + {"point": "committed_before_release", **turn_document(turn, source)} + ), + flush=True, + ) + # The parent observes the real committed cursor and live + # configuration lease, then kills this process at this edge. + await asyncio.to_thread(sys.stdin.readline) + raise AssertionError("paused child must be killed") + + producer._acquire_pending = pause + turn = await source_turn(support) + result = turn_document(turn, source) + repeated = await source_turn(support) + assert not repeated.obligations_committed and not repeated.revisions_committed + assert result["executions"] == [ + r.identity.source_execution_key for r in source.requests + ] + await support.aclose() + print(json.dumps({"point": "done", **result}), flush=True) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/tests/conformance/reporting/_production_support.py b/tests/conformance/reporting/_production_support.py new file mode 100644 index 000000000..161d4231b --- /dev/null +++ b/tests/conformance/reporting/_production_support.py @@ -0,0 +1,637 @@ +"""Real independent SQLite destination, mounted SDK composition, shared store vectors. + +The destination is a test provider, not the development writer with a changed +eligibility flag. Its immutable rows and grants survive a new provider process. +""" + +import hashlib +import json +import sqlite3 +from contextlib import asynccontextmanager +from dataclasses import asdict, replace +from datetime import datetime, timedelta, timezone + +from adcp.decisioning.capabilities import Account as AccountCapabilities +from adcp.reporting.fixtures import redacted_capabilities +from adcp.reporting.inline_source import ( + FileSystemStagingStore, + InlineFetchResult, + InlineReportingSource, + SealedSlice, +) +from adcp.reporting.ledger import ReportingRevisionRecord, revision_content_sha256 +from adcp.reporting.ledger.delivery_models import ( + ReportingDestinationBinding, + ReportingResourceRecord, +) +from adcp.reporting.ledger.models import ReportingDeliveryEscalation +from adcp.reporting.ledger.producer import ProducerOfferings, ReportingProducer +from adcp.reporting.materializer import ( + ReportingDestinationIO, + ReportingRevisionVerifierRegistry, + reference_digest, + reference_verifier, +) +from adcp.reporting.materializer.contracts import ( + ReportingDestinationLocator, + ReportingDestinationPage, + ReportingDestinationSession, + ReportingWriterCapability, + binding_fingerprint, + failure, +) +from adcp.reporting.materializer.service import ReportingMaterializerService +from adcp.reporting.production.configuration import ReportingProductionConfigurationTask +from adcp.reporting.production.contracts import ( + ReportingProductionDestinationBinding, + ReportingProductionMethod, + ReportingProductionSourceBinding, +) +from adcp.reporting.production.memory import InMemoryReportingProductionStore +from adcp.reporting.production.offerings import ReportingProductionOffering +from adcp.reporting.production.service import ReportingProductionSupport +from adcp.reporting.projection.memory import InMemoryReportingStatusProjection +from adcp.reporting.receipts.errors import ReportingReceiptError +from adcp.reporting.source import ( + ReportingSourceCapabilitiesV1, + SourceBatchManifestReferenceV1, + reporting_source_capabilities_sha256_v1, +) +from adcp.server.serve import create_mcp_server +from adcp.types import ReportingDeliveryOffering + +from ._durable_materializer_support import DurableCase, DurableHarness +from ._generation_support import END, START, configuration, isolated_reporting_pool, obligation_for +from ._materializer_support import reference_rows +from ._reliable_support import ManualClock + + +class SQLiteDestination: + production_eligible = True + resource_retention_days = 400 + authorization_revocation_seconds = 0 + + def __init__(self, path, key): + self.path, self.key = path, key + self.capabilities = (key.capability,) + self.delivery_methods = ( + ReportingProductionMethod( + key.capability, + { + "pattern": key.capability.method, + "transport": key.capability.transport, + "format": key.capability.format, + "provider": {"domain": "fixture.example.test"}, + "orchestration": "producer_managed", + "destination_modes": ["provision"], + "reader_compatibility": ["fixture-sql-v1"], + }, + ), + ) + self.opens = self.closes = self.writes = 0 + with sqlite3.connect(path) as c: + c.execute("CREATE TABLE IF NOT EXISTS grants (binding TEXT PRIMARY KEY)") + c.execute( + "CREATE TABLE IF NOT EXISTS methods" + " (binding TEXT PRIMARY KEY, method TEXT NOT NULL)" + ) + c.execute( + "CREATE TABLE IF NOT EXISTS artifacts (id TEXT PRIMARY KEY, content TEXT NOT NULL)" + ) + + def grant(self, binding): + with sqlite3.connect(self.path) as c: + c.execute("INSERT OR IGNORE INTO grants VALUES (?)", (binding_fingerprint(binding),)) + c.execute( + "INSERT OR IGNORE INTO methods VALUES (?,?)", + ( + binding_fingerprint(binding), + json.dumps( + { + "pattern": binding.method, + "transport": binding.transport, + "orchestration": "producer_managed", + "destination": { + "mode": "provision", + "provider": {"domain": "fixture.example.test"}, + "location": "reporting/" + binding.destination_ref, + }, + } + ), + ), + ) + + def configuration_binding(self, binding): + with sqlite3.connect(self.path) as c: + row = c.execute( + "SELECT method FROM methods JOIN grants USING(binding) WHERE binding=?", + (binding_fingerprint(binding),), + ).fetchone() + if row is None: + return None + return ReportingProductionDestinationBinding( + binding, self.delivery_methods[0], json.loads(row[0]) + ) + + def revoke(self, binding): + with sqlite3.connect(self.path) as c: + c.execute("DELETE FROM grants WHERE binding=?", (binding_fingerprint(binding),)) + + def resolve(self, request, *, phase, context): + return _SQLiteSession(self, request, phase, context) + + +class _SQLiteSession(ReportingDestinationSession): + def __init__(self, provider, request, phase, context): + super().__init__(request, phase, context) + self.provider, self.connection = provider, None + + async def _open(self): + p = self.provider + self.connection = sqlite3.connect(p.path) + p.opens += 1 + if ( + self.request.verification_key != p.key + or self.connection.execute( + "SELECT 1 FROM grants WHERE binding=?", (self.request.binding_fingerprint,) + ).fetchone() + is None + ): + raise failure("AUTHORIZATION_DENIED") + + async def _close(self): + if self.connection is not None: + self.connection.close() + self.connection = None + self.provider.closes += 1 + + async def write(self, content): + assert self.phase == "write" and self.connection is not None + r = self.request + path = "fixture/" + hashlib.sha256(r.external_id.encode()).hexdigest() + row = self.connection.execute( + "SELECT content FROM artifacts WHERE id=?", (r.external_id,) + ).fetchone() + if row: + data = json.loads(row[0]) + if data["rows"] != [v.decode() for v in content.rows]: + raise failure("WRITE_FAILED") + else: + resource = ReportingResourceRecord( + "fixture_" + hashlib.sha256(r.external_id.encode()).hexdigest(), + "warehouse_relation", + path + "/table", + "immutable_location", + datetime.now(timezone.utc) + + timedelta(days=self.provider.resource_retention_days + 1), + reader_compatibility=content.binding.reader_compatibility, + ) + data = { + "rows": [v.decode() for v in content.rows], + "resource": asdict(resource), + "binding": r.binding_fingerprint, + "revision": r.reporting_revision_id, + } + data["resource"]["expires_at"] = resource.expires_at.isoformat() + self.connection.execute( + "INSERT INTO artifacts VALUES (?,?)", (r.external_id, json.dumps(data)) + ) + self.connection.commit() + self.provider.writes += 1 + resource = dict(data["resource"]) + resource["expires_at"] = datetime.fromisoformat(resource["expires_at"]) + resource["object_refs"] = tuple(resource["object_refs"]) + resource["reader_compatibility"] = tuple(resource["reader_compatibility"]) + return ReportingDestinationLocator( + r.external_id, r.binding_fingerprint, ReportingResourceRecord(**resource) + ) + + async def read_rows(self, locator, *, cursor, limit): + assert self.phase == "readback" and self.connection is not None + raw = self.connection.execute( + "SELECT content FROM artifacts WHERE id=?", (locator.external_id,) + ).fetchone() + if raw is None or locator.external_id != self.request.external_id: + raise failure("RESOURCE_UNAVAILABLE") + data = json.loads(raw[0]) + if data["binding"] != self.request.binding_fingerprint: + raise failure("AUTHORIZATION_DENIED") + offset = 0 if cursor is None else int(cursor) + rows = tuple(v.encode() for v in data["rows"][offset : offset + limit]) + following = offset + len(rows) + more = following < len(data["rows"]) + return ReportingDestinationPage( + data["revision"], + rows, + len(data["rows"]), + more, + str(following) if more else None, + self.request.verification_key.capability.format, + "destination", + ) + + +class SQLiteSeals: + def __init__(self, path): + self.path = path + with sqlite3.connect(path) as c: + c.execute( + "CREATE TABLE IF NOT EXISTS seals (account TEXT, execution TEXT," + " reference TEXT NOT NULL, manifest BLOB NOT NULL, PRIMARY KEY(account,execution))" + ) + + async def get(self, *, account_id, source_execution_key): + with sqlite3.connect(self.path) as c: + row = c.execute( + "SELECT reference,manifest FROM seals WHERE account=? AND execution=?", + (account_id, source_execution_key), + ).fetchone() + return ( + None + if row is None + else SealedSlice(SourceBatchManifestReferenceV1.model_validate_json(row[0]), row[1]) + ) + + async def put(self, *, account_id, source_execution_key, sealed): + with sqlite3.connect(self.path) as c: + c.execute( + "INSERT OR IGNORE INTO seals VALUES (?,?,?,?)", + ( + account_id, + source_execution_key, + sealed.reference.model_dump_json(), + sealed.manifest_bytes, + ), + ) + return await self.get(account_id=account_id, source_execution_key=source_execution_key) + + +class Source: + def __init__( + self, + key, + path, + rows=None, + *, + clock=None, + product_ids=("catalog-7391", "catalog-5820"), + official=False, + ): + raw = redacted_capabilities().model_dump(mode="json") + raw["offerings"] = [raw["offerings"][int(official)]] + self.source_id = raw["offerings"][0]["offering_id"] + if official: + raw["offerings"][0].update( + grain="fixed_window", + source_timezone="UTC", + source_local_ready_time="00:00", + days_after_period_end=0, + expected_availability_lag="PT0S", + windowing={ + "kind": "fixed_closed_window", + "minimum_window": "PT1H", + "maximum_window": "P1D", + "overlapping_windows_supported": False, + }, + ) + raw["offerings"][0]["product_ids"] = list(product_ids) + raw["offerings"][0]["contract"] = { + "report_definition_id": key.report_definition_id, + "reporting_profile": key.reporting_profile, + **{ + k: getattr(key.definition, k) + for k in ( + "report_definition_uri", + "report_definition_sha256", + "schema_version", + "schema_uri", + "schema_sha256", + "schema_dialect", + "schema_ref_policy", + ) + }, + } + raw["offerings"][0]["worst_case_availability_lag"] = "PT1H" + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + self.capabilities = ReportingSourceCapabilitiesV1.model_validate(raw) + self.bindings = {} + self.rows, self.requests = rows, [] + self.inline = InlineReportingSource( + capabilities=self.capabilities, + fetch=self.fetch, + staging=FileSystemStagingStore(path.with_suffix(".staging")), + seals=SQLiteSeals(path.with_suffix(".seals")), + constituent_of=lambda row, req: req.coverage.constituents[0].constituent_id, + clock=clock or (lambda: END), + ) + self.reader = self.inline.staging + + def bind_generation(self, configuration, *, product_id="catalog-7391"): + binding = ReportingProductionSourceBinding.for_configuration( + configuration, + capabilities_sha256=self.capabilities.capabilities_sha256, + media_buy_products=tuple( + (media_buy_id, product_id) for media_buy_id in configuration.media_buy_ids + ), + ) + self.bindings[configuration.generation_key] = binding + return binding + + def configuration_binding(self, configuration): + return self.bindings.get(configuration.generation_key) + + async def fetch(self, request): + assert self.rows is not None, "this seeded publication needs no new acquisition" + self.requests.append(request) + return InlineFetchResult(self.rows, data_through=request.period.end, currency="USD") + + async def execute(self, request, *, cancel, heartbeat=None): + return await self.inline.execute(request, cancel=cancel, heartbeat=heartbeat) + + +async def account_task(request, context, admit): + return {"accounts": []} + + +@asynccontextmanager +async def production_harness( + backend, + path, + *, + notifications=False, + count=503, + second_source=False, + source_publication=False, + notification_delivery=False, + periods=None, + existing_store=None, + existing_pool=None, + source_bindings=(), + account_handler=None, + reconciled=False, + feedback=False, + identity_prefix="", +): + from contextlib import AsyncExitStack + + async with AsyncExitStack() as stack: + clock = ManualClock(datetime.now(timezone.utc)) + pool = existing_pool + if existing_store is not None: + store = existing_store + clock = store._clock + elif backend == "memory": + store = InMemoryReportingProductionStore(clock=clock, notifications=notifications) + else: + from adcp.reporting.production.pg import PgReportingProductionStore + + if pool is None: + pool = await stack.enter_async_context(isolated_reporting_pool(autocommit=True)) + store = PgReportingProductionStore(pool=pool, notifications=notifications) + await store.create_schema() + h = DurableHarness(store, clock, pool) + cap = ReportingWriterCapability( + "warehouse_materialization", + "fixture-sql", + "jsonl", + "canonical_digest", + "destination", + "immutable_location", + "sha256", + "conditional_create", + ) + verifier = reference_verifier(cap) + key = verifier.key + config = replace( + configuration(), + delivery_config_id=identity_prefix + configuration().delivery_config_id, + deactivated_at=None if periods is None else START + timedelta(hours=periods), + definition=key.definition, + report_definition_id=key.report_definition_id, + feed_purpose="billing" if reconciled else "analytics", + required_finality="official" if reconciled else "snapshot", + ) + await store.put_configuration(config) + original_obligation = obligation_for(config) + obligation = await store.commit_obligation( + replace( + original_obligation, + reporting_obligation_id=identity_prefix + + original_obligation.reporting_obligation_id, + ) + ) + binding = ReportingDestinationBinding( + config.generation_key, + "https://buyer.example.test/agent", + "destination", + "trusted-provider-binding", + cap.method, + cap.transport, + cap.verification_profile, + "consumer_receipt" if reconciled else "delivery_only", + config.feed_purpose, + 400, + START, + cap.format, + ("fixture-sql-v1",), + "delivered", + ) + await store.put_destination_binding(binding) + rows = reference_rows(count) + _, totals = verifier.canonicalize(rows) + pairs = tuple((t.name, t.value) for t in totals) + revision = ReportingRevisionRecord( + identity_prefix + "production-revision", + config.account_id, + obligation.reporting_obligation_id, + config.required_finality, + revision_content_sha256( + reporting_revision_id=identity_prefix + "production-revision", + row_count=count, + control_totals=pairs, + reporting_rows=rows, + control_total_evidence=totals, + ), + count, + pairs, + END, + END, + END, + finality_basis="source_final" if reconciled else None, + finality_policy_id="fixture-official-v1" if reconciled else None, + finalized_at=END if reconciled else None, + canonical_content_digest=reference_digest(verifier, rows), + managed_control_totals=totals, + ) + if not source_publication: + await store.commit_revision(revision, rows) + writer = SQLiteDestination(path, key) + writer.grant(binding) + registry = ReportingRevisionVerifierRegistry((verifier,)) + io = ReportingDestinationIO(registry, writer) + item = DurableCase( + store, + config, + obligation, + binding, + revision, + rows, + verifier, + registry, + writer, + writer, + io, + ) + source_clock = ManualClock(END) + source = Source( + key, + path.with_name("source"), + rows if source_publication or periods is not None else None, + clock=source_clock, + official=reconciled, + ) + if not second_source: + source.bind_generation(config) + escalation = ReportingDeliveryEscalation() + producer = ReportingProducer( + source=source, + offerings=ProducerOfferings( + snapshot_offering_id=None if reconciled else source.source_id, + official_offering_id=source.source_id if reconciled else None, + publication_namespace=source.capabilities.offerings[0].publication_namespace, + source_scope=source.capabilities.source_scope, + ), + store=store, + escalation=escalation, + clock=source_clock, + revision_verifier=verifier, + object_reader=source.reader, + ) + if pool is None: + projection = InMemoryReportingStatusProjection( + store, + revision_ownership=True, + escalation=escalation, + consumer_status_enabled=feedback, + ) + else: + from adcp.reporting.projection.pg import PgReportingStatusProjection + + projection = PgReportingStatusProjection( + store, + revision_ownership=True, + escalation=escalation, + consumer_status_enabled=feedback, + ) + profile = { + "id": key.reporting_profile, + "version": key.definition.schema_version, + "schema_uri": key.definition.schema_uri, + "schema_sha256": key.definition.schema_sha256, + "schema_dialect": key.definition.schema_dialect, + "schema_ref_policy": key.definition.schema_ref_policy, + "grain": "row", + "primary_keys": ["row_id"], + "canonicalization_id": key.canonicalization.canonicalization_id, + "canonicalization_uri": key.canonicalization.canonicalization_uri, + "canonicalization_sha256": key.canonicalization.canonicalization_sha256, + } + offering = ReportingDeliveryOffering.model_validate( + { + "offering_id": "reconciled-fixture" if reconciled else "managed-fixture", + "feed_purpose": config.feed_purpose, + "report_definition_id": key.report_definition_id, + "report_definition_uri": key.definition.report_definition_uri, + "report_definition_sha256": key.definition.report_definition_sha256, + "reporting_profile": profile, + "schedule": { + "period_duration": "PT1H", + "alignment": "utc", + "delivery_sla": "PT1H", + }, + "supported_finality": [config.required_finality], + "reconciliation_mode": binding.reconciliation_mode, + "method": writer.delivery_methods[0].wire(), + } + ) + admitted = ReportingProductionOffering(offering, producer, key, source.source_id) + offerings = (admitted,) + if second_source: + other = Source(key, path.with_name("other-source")) + other_producer = ReportingProducer( + source=other, + offerings=producer._offerings, + store=store, + escalation=escalation, + clock=lambda: END, + revision_verifier=verifier, + object_reader=other.reader, + ) + offerings += ( + ReportingProductionOffering( + offering.model_copy(update={"offering_id": "managed-other"}), + other_producer, + key, + other.source_id, + ), + ) + + for configured, offering_id, product_id in source_bindings: + selected = next(o for o in offerings if o.offering_id == offering_id) + selected.producer._source.bind_generation(configured, product_id=product_id) + + h.authorized_bindings = {(config.account_id, binding.consumer_id)} + + async def authorize(account, context, consumer): + account_id = account.get("account_id") + if ( + account != {"account_id": account_id} + or (account_id, consumer) not in h.authorized_bindings + ): + raise ReportingReceiptError("UNAUTHORIZED") + return account_id + + workers = () + if notification_delivery: + from adcp.reporting.outbox.routing import ReportingEnvelopeCipher + from adcp.reporting.production.notifications import ( + ReportingProductionSigning, + production_notification_workers, + ) + + from ._reliable_support import FailurePlan, ScriptedSigning, ScriptedSubscriptions + + h.notification_failures = FailurePlan() + h.subscriptions = ScriptedSubscriptions(h.notification_failures) + h.signing = ScriptedSigning(h.notification_failures) + workers = production_notification_workers( + store, + projection, + subscriptions=h.subscriptions, + signing=ReportingProductionSigning( + h.signing, + ("ed25519",), + brand_json_url="https://seller.example.test/brand.json", + ), + cipher=ReportingEnvelopeCipher(b"b" * 32), + ) + support = ReportingProductionSupport( + ReportingMaterializerService(store, io, writer), + projection, + offerings=offerings, + configuration_task=ReportingProductionConfigurationTask( + account_handler or account_task, + AccountCapabilities(supported_billing=["operator"], require_operator_auth=True), + ), + resolve_account=authorize, + notification_workers=workers, + poll_seconds=60, + ) + h.production, h.projection, h.item = support, projection, item + h.source_clock = source_clock + h.mount = create_mcp_server(support.handler) + try: + await support.start() + yield h + finally: + await support.aclose() diff --git a/tests/conformance/reporting/_production_transport.py b/tests/conformance/reporting/_production_transport.py new file mode 100644 index 000000000..b232ad999 --- /dev/null +++ b/tests/conformance/reporting/_production_transport.py @@ -0,0 +1,35 @@ +"""The actual production handler on authenticated MCP and both A2A mounts.""" + +import json + +from ._receipt_transport import MountedReceipts + + +class MountedProduction(MountedReceipts): + def __init__(self, h): + super().__init__(h) + self.handler = h.production.handler + + def authorize(self, s, *, token="token-one"): + super().authorize(s, token=token) + self.h.authorized_bindings.add((s.obligation.account_id, s.binding.consumer_id)) + + async def middleware(self, name, params, context, call_next): + return await call_next() + + async def call(self, client, task, request, *, transport="mcp", token="token-one"): + request = {"adcp_version": "3.2-rc.3", **request} + + def route(wire): + value = json.loads(wire) + if transport == "mcp": + value["params"]["name"] = task + else: + value["params"]["message"]["parts"][0]["data"]["skill"] = task + return json.dumps(value) + + if transport == "mcp": + return await self.mcp(client, request, token=token, mutate_wire=route) + return await self.a2a( + client, request, token=token, mutate_wire=route, v1=transport == "a2a-1.0" + ) diff --git a/tests/conformance/reporting/_projection_support.py b/tests/conformance/reporting/_projection_support.py new file mode 100644 index 000000000..5feeb53a5 --- /dev/null +++ b/tests/conformance/reporting/_projection_support.py @@ -0,0 +1,75 @@ +"""B2.4 shared execution: actual stores, no forged readiness certificate.""" + +from contextlib import asynccontextmanager + +import pytest + +from adcp.reporting.projection.memory import ( + InMemoryReportingProjectionStore, + InMemoryReportingStatusProjection, +) + +from ._durable_materializer_support import DurableHarness +from ._generation_support import isolated_reporting_pool +from ._reconciliation_support import Clock + + +@asynccontextmanager +async def projection_harness(backend, *, notifications=False, feedback=False, ownership=True): + clock = Clock() + if backend == "memory": + store = InMemoryReportingProjectionStore(clock=clock, notifications=notifications) + h = DurableHarness(store, clock) + h.projection = InMemoryReportingStatusProjection( + store, consumer_status_enabled=feedback, revision_ownership=ownership + ) + yield h + else: + from adcp.reporting.projection.pg import ( + PgReportingProjectionStore, + PgReportingStatusProjection, + ) + + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingProjectionStore(pool=pool, notifications=notifications) + await store.create_schema() + h = DurableHarness(store, clock, pool) + h.projection = PgReportingStatusProjection( + store, consumer_status_enabled=feedback, revision_ownership=ownership + ) + yield h + + +@pytest.fixture( + params=[("memory", False), ("memory", True), ("postgres", False), ("postgres", True)] +) +async def projections(request): + backend, notifications = request.param + async with projection_harness(backend, notifications=notifications) as h: + yield h + + +async def drain(projection, account_id): + turns = [] + for _ in range(100): + result = await projection.project_one(account_id=account_id) + if not result.did_work: + return turns + turns.append(result) + raise AssertionError("projection failed to converge") + + +async def inputs(h, account_id): + if h.pool is None: + return tuple(h.store._projection_accounts[account_id].inputs) + from adcp.reporting.projection.capture import decode_projection_input + + async with h.pool.connection() as c: + rows = await ( + await c.execute( + "SELECT input FROM reporting_projection_inputs" + " WHERE account_id=%s ORDER BY sequence", + (account_id,), + ) + ).fetchall() + return tuple(decode_projection_input(r[0]) for r in rows) diff --git a/tests/conformance/reporting/test_reporting_feed_rolling.py b/tests/conformance/reporting/test_reporting_feed_rolling.py index 3bf508f75..277c1e6ea 100644 --- a/tests/conformance/reporting/test_reporting_feed_rolling.py +++ b/tests/conformance/reporting/test_reporting_feed_rolling.py @@ -19,6 +19,8 @@ from adcp.reporting.ledger.delivery import receipt_to_wire from adcp.reporting.materializer import PgReportingMaterializerStore from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.production.pg import PgReportingProductionStore +from adcp.reporting.production.schema import validate_production_schema from adcp.reporting.receipts import PgReportingReceiptStore from ._durable_materializer_support import DurableHarness, durable_case @@ -249,6 +251,17 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou frozen = await feed.read_reporting_feed_snapshot( first["ledger_snapshot_id"], caller=case.scope.principal ) + # Final B2.4 integration: the actual old binaries below exercise + # their ordinary read/write contract on all new objects, before + # incompatible autonomous projectors are drained and activated. + production = PgReportingProductionStore(pool=pool, notifications=notifications) + await production.create_schema() + await production.create_schema() + async with pool.connection() as c: + production_objects = await schema_objects(c) + await validate_production_schema(c, notifications=notifications) + assert {k: production_objects[k] for k in new_objects} == new_objects + production_added = production_objects.keys() - new_objects.keys() after = await frozen_call( installed_feed_history, pool, "exercise", phase="after", **kwargs ) @@ -313,6 +326,8 @@ async def test_nine_actual_artifacts_preserve_ordinary_writes_and_frozen_b22_mou "historical_wheel_sha256": installed_feed_history[3]["wheel_sha256"], "b22_wheel_sha256": approved_feed_b22[3]["wheel_sha256"], "feed_objects": len(added), + "b24_additive_objects": len(production_added), + "b24_activation": False, "page_count": len(expected[0]), "quarantine_preserved": True, } diff --git a/tests/conformance/reporting/test_reporting_production.py b/tests/conformance/reporting/test_reporting_production.py new file mode 100644 index 000000000..06fcf0cbd --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production.py @@ -0,0 +1,124 @@ +"""Admitted production work preserves exact transactions and epoch-zero history.""" + +import pytest + +from adcp.reporting.materializer import reference_digest +from adcp.reporting.materializer.work import ReportingMaterializerLease +from adcp.server.base import ToolContext +from adcp.validation.schema_loader import get_named_validator + +from ._production_support import production_harness +from ._projection_support import drain +from .test_reporting_production_lock_order import source_turn + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("count", [0, 503]) +async def test_actual_source_publication_builds_canonical_evidence(backend, count, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", count=count, source_publication=True + ) as h: + support, item = h.production, h.item + source = support.offerings[0].producer._source + assert source.requests == [] + await support.activate(account_id=item.config.account_id) + turn = await source_turn(support) + assert not turn.slices_failed and len(turn.revisions_committed) == 1 + revisions = await h.store.list_revisions( + account_id=item.config.account_id, + reporting_obligation_id=item.obligation.reporting_obligation_id, + ) + assert len(revisions) == 1 + item.revision = revisions[0] + assert item.revision.canonical_content_digest == reference_digest(item.verifier, item.rows) + assert item.revision.managed_control_totals == item.verifier.canonicalize(item.rows)[1] + assert len(source.requests) == 1 + repeat = await source_turn(support) + assert not repeat.revisions_committed and len(source.requests) == 1 + result = await support.materializer.run_once() + assert result.state == "verified" + assert item.writer.writes == 1 + await drain(h.projection, item.config.account_id) + page = await support.handler.get_reporting_status( + {"account": {"account_id": item.config.account_id}, "view": "summary"}, + ToolContext(caller_identity=item.binding.consumer_id), + ) + assert page["health"] == "complete" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_actual_admission_verified_finish_and_private_polling( + backend, notifications, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=notifications + ) as h: + support, item = h.production, h.item + before = await support.reporting_delivery() + assert bool(before) == (backend == "postgres") + if before: + assert before["managed_delivery"] is True + assert "reconciled_billing" not in before + assert "receipt_task" not in before + assert "readiness_notification" not in before + validator = get_named_validator("core/reporting-delivery-capabilities.json") + assert validator is not None + assert not list(validator.iter_errors(before)) + assert "sync_reporting_receipts" not in support.handler.advertised_tools_for_instance() + assert await support.activate(account_id=item.config.account_id) + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) + assert lease.admission_epoch == 2 + prepared, verified = await item.verified(lease) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + assert result.state == "verified" + assert item.writer.writes == 1 + assert item.writer.opens == item.writer.closes == 2 + assert not (await h.queue())[0] + assert await h.store.read_materializer_boundaries(caller=item.binding.principal) == () + assert len(await h.store.read_production_boundaries(caller=item.binding.principal)) == 1 + if h.pool is None: + queue = h.store._production_outbox + count = len(queue.events) if queue is not None else 0 + else: + async with h.pool.connection() as c: + count = ( + await ( + await c.execute( + "SELECT count(*) FROM reporting_production_notification_events" + ) + ).fetchone() + )[0] + assert count == int(notifications) + await drain(h.projection, item.config.account_id) + context = ToolContext(caller_identity=item.binding.consumer_id) + response = await support.handler.get_reporting_status( + {"account": {"account_id": item.config.account_id}, "view": "summary"}, context + ) + assert response["health"] == "complete" + rows = [] + request = { + "account": {"account_id": item.config.account_id}, + "reporting_revision_id": item.revision.reporting_revision_id, + "pagination": {"max_results": 100}, + } + for _ in range(10): + page = await support.handler.get_media_buy_delivery(request, context) + assert isinstance(page, dict) + validator = get_named_validator("media-buy/get-media-buy-delivery-response.json") + assert validator is not None + assert not list(validator.iter_errors(page)) + rows.extend(page["reporting_rows"]) + if not page["pagination"]["has_more"]: + break + assert len(page["pagination"]["cursor"]) <= 2048 + request["pagination"]["cursor"] = page["pagination"]["cursor"] + else: + pytest.fail("exact revision walk did not terminate") + assert rows == item.rows + repeat = await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + assert repeat.state == "verified" + assert item.writer.writes == 1 + await support.aclose() + assert await support.reporting_delivery() == {} diff --git a/tests/conformance/reporting/test_reporting_production_bindings.py b/tests/conformance/reporting/test_reporting_production_bindings.py new file mode 100644 index 000000000..ceae59381 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_bindings.py @@ -0,0 +1,228 @@ +"""Full provider methods and account-qualified frozen source mappings.""" + +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.store import LedgerConflictError +from adcp.reporting.materializer.contracts import ReportingWriterError +from adcp.reporting.production.contracts import ReportingProductionMethod + +from ._generation_support import END, obligation_for +from ._production_support import production_harness +from ._production_transport import MountedProduction +from .test_reporting_production_lock_order import source_turn + + +async def source_documents(h): + if h.pool is None: + return sorted( + (key.account_id, key.delivery_config_id, key.delivery_config_version, value.document()) + for key, value in h.store._production_source_bindings.items() + ) + async with h.pool.connection() as connection: + return await ( + await connection.execute( + "SELECT account_id,delivery_config_id,delivery_config_version,source_binding" + " FROM reporting_production_generations" + " ORDER BY account_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_full_method_controls_admission_acquisition_and_raw_discovery( + backend, notifications, tmp_path +): + async with production_harness( + backend, + tmp_path / "destination.sqlite", + count=0, + source_publication=True, + notifications=notifications, + ) as h: + support, item = h.production, h.item + offering = support.offerings[0] + producer, provider = offering.producer, item.writer + await support.activate(account_id=item.config.account_id) + mounted = MountedProduction(h) + mounted.authorize(item) + original = provider.delivery_methods + new = replace(item.config, delivery_config_version=2) + new_binding = replace(item.binding, generation_key=new.generation_key) + provider.grant(new_binding) + producer._source.bind_generation(new) + before = await source_documents(h) + async with mounted.client() as client: + for field, different in ( + ("provider", {"domain": "different-provider.example.test"}), + ("destination_modes", ["existing"]), + ("access_mode", "read_only"), + ("reader_compatibility", ["different-reader-v2"]), + ): + raw = original[0].wire() + raw[field] = different + provider.delivery_methods = ( + ReportingProductionMethod(item.verifier.key.capability, raw), + ) + assert provider.capabilities == (item.verifier.key.capability,) + try: + with pytest.raises(ReportingWriterError) as denied: + await h.store.admit_production_configuration( + new, new_binding, offering_id=offering.offering_id + ) + assert denied.value.failure.code == "BINDING_MISMATCH" + token = support._producer_turn.set(producer) + try: + with pytest.raises((LedgerConflictError, ReportingWriterError)): + await producer.acquire_obligation(item.config, item.obligation, now=END) + finally: + support._producer_turn.reset(token) + assert producer._source.requests == [] + assert (await source_turn(support)).leased is None + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps.get("status") == "completed", caps + assert "reporting_delivery" not in caps.get("media_buy", {}), caps + assert "different-provider" not in json.dumps(caps) + _, status = await mounted.call( + client, + "get_reporting_status", + {"account": {"account_id": item.config.account_id}, "view": "summary"}, + transport=transport, + ) + assert status.get("status") == "completed", status + assert status["health"] != "complete" + finally: + provider.delivery_methods = original + assert await source_documents(h) == before + _, restored = await mounted.call(client, "get_adcp_capabilities", {}) + assert restored.get("status") == "completed", restored + claims = restored.get("media_buy", {}).get("reporting_delivery", {}) + assert bool(claims.get("managed_delivery")) == (backend == "postgres") + assert "reconciled_billing" not in claims and "receipt_task" not in claims + result = await source_turn(support) + assert len(result.revisions_committed) == 1 and not result.slices_failed + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_products_are_explicit_for_shared_definition_and_colliding_account_keys( + backend, tmp_path +): + path = tmp_path / "destination.sqlite" + async with production_harness( + backend, path, count=0, source_publication=True, second_source=True + ) as h: + support, item = h.production, h.item + first, second = support.offerings + outside = replace(item.config, account_id="acct_b") + peer = replace(item.config, delivery_config_id="other-source") + entries = ( + (item.config, item.obligation, item.binding, first, "catalog-7391"), + ( + outside, + obligation_for(outside), + replace(item.binding, generation_key=outside.generation_key), + first, + "catalog-5820", + ), + ( + peer, + replace(obligation_for(peer), reporting_obligation_id="peer-obligation"), + replace(item.binding, generation_key=peer.generation_key), + second, + "catalog-5820", + ), + ) + assert first.verification_key.definition == second.verification_key.definition + assert outside.media_buy_ids == item.config.media_buy_ids + assert outside.delivery_config_id == item.config.delivery_config_id + # Supported discovery does not need a first trusted account binding. + assert not first.producer._source.bindings and not second.producer._source.bindings + assert bool(await support.reporting_delivery()) == (backend == "postgres") + for config, obligation, binding, offering, product_id in entries: + source = offering.producer._source + source.rows = item.rows + source.bind_generation(config, product_id=product_id) + item.writer.grant(binding) + assert product_id != config.report_definition_id + await h.store.admit_production_configuration( + config, binding, offering_id=offering.offering_id + ) + await h.store.commit_obligation(obligation) + frozen = await source_documents(h) + assert len(frozen) == 3 + # Admission fixes bindings even before activation can claim work. + assert (await source_turn(support)).leased is None + for account in ("acct_a", "acct_b"): + await support.activate(account_id=account) + for config, obligation, binding, offering, product_id in entries: + producer = offering.producer + token = support._producer_turn.set(producer) + try: + revision = await producer.acquire_obligation(config, obligation, now=END) + finally: + support._producer_turn.reset(token) + assert revision is not None + request = producer._source.requests[-1] + assert request.identity.account_id == config.account_id + assert request.identity.delivery_config_id == config.delivery_config_id + assert {c.product_id for c in request.coverage.constituents} == {product_id} + assert {c.media_buy_id for c in request.coverage.constituents} == set( + config.media_buy_ids + ) + assert await source_documents(h) == frozen + await support.aclose() + async with production_harness( + backend, + path, + count=0, + source_publication=True, + second_source=True, + existing_store=h.store if h.pool is None else None, + existing_pool=h.pool, + source_bindings=tuple( + (config, offering.offering_id, product) + for config, _, _, offering, product in entries + ), + ) as restarted: + fresh = restarted.production + assert await source_documents(restarted) == frozen + for config, obligation, binding, old_offering, product_id in entries: + offering = next( + o for o in fresh.offerings if o.offering_id == old_offering.offering_id + ) + offering.producer._source.bind_generation(config, product_id=product_id) + await restarted.store.admit_production_configuration( + config, binding, offering_id=offering.offering_id + ) + first_fresh = fresh.offerings[0] + source = first_fresh.producer._source + old = source.bindings[item.config.generation_key] + source.bind_generation(item.config, product_id="catalog-5820") + try: + with pytest.raises(ReportingNotificationError, match="source_conflict"): + await restarted.store.admit_production_configuration( + item.config, item.binding, offering_id=first_fresh.offering_id + ) + token = fresh._producer_turn.set(first_fresh.producer) + try: + with pytest.raises((LedgerConflictError, ReportingWriterError)): + await restarted.store.producer_constituents(item.config, item.obligation) + finally: + fresh._producer_turn.reset(token) + assert await source_documents(restarted) == frozen + finally: + source.bindings[item.config.generation_key] = old + token = fresh._producer_turn.set(first_fresh.producer) + try: + restored = await restarted.store.producer_constituents(item.config, item.obligation) + finally: + fresh._producer_turn.reset(token) + assert {c.product_id for c in restored} == {"catalog-7391"} + assert await source_documents(restarted) == frozen diff --git a/tests/conformance/reporting/test_reporting_production_configuration.py b/tests/conformance/reporting/test_reporting_production_configuration.py new file mode 100644 index 000000000..dcfafb029 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_configuration.py @@ -0,0 +1,277 @@ +"""Frozen provider configuration, mounted admission and semantic replay checks.""" + +import copy +import json +import sqlite3 +from dataclasses import replace +from datetime import timedelta, timezone + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.materializer.contracts import ReportingWriterError, binding_fingerprint +from adcp.reporting.materializer.work import ReportingMaterializerLease +from adcp.reporting.production.configuration import ReportingConfigurationAdmission +from adcp.validation.schema_loader import get_named_validator + +from ._feed_support import feed_request, walk +from ._generation_support import END +from ._production_support import production_harness +from ._production_transport import MountedProduction +from ._receipt_transport import error_code +from .test_reporting_production_bindings import source_documents + + +def wire_configuration(h): + config, binding = h.item.config, h.item.binding + offering = h.production.offerings[0] + return { + "delivery_config_id": config.delivery_config_id, + "delivery_config_version": config.delivery_config_version, + "offering_id": offering.offering_id, + "active": config.deactivated_at is None, + "feed_purpose": config.feed_purpose, + "report_definition_id": config.report_definition_id, + "reporting_profile": config.reporting_profile, + "scope": {"media_buy_ids": list(config.media_buy_ids)}, + "coverage_requirement": "full", + "required_finality": config.required_finality, + "reconciliation_mode": binding.reconciliation_mode, + "schedule": offering.configuration_schedule(config), + "method": h.item.writer.configuration_binding(binding).wire(), + } + + +def state_for(h, configuration): + ids = list(h.item.config.media_buy_ids) + coverage = { + "status": "full", + "evaluated_at": END.isoformat(), + "media_buy_ids": ids, + "fully_covered_media_buy_ids": ids, + "partially_covered_media_buy_ids": [], + "unsupported_media_buy_ids": [], + "unknown_media_buy_ids": [], + "package_ids": [], + "covered_package_ids": [], + "unsupported_package_ids": [], + "unknown_package_ids": [], + "limitations": [], + } + return { + "configuration": configuration, + "state": "ready", + "destination_ref": h.item.binding.destination_ref, + "validated_at": END.isoformat(), + "activated_at": h.item.config.activated_at.isoformat(), + "current_coverage": coverage, + } + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("fractional_offset", [False, True]) +async def test_mounted_configuration_is_the_admitted_method_scope_and_schedule( + backend, notifications, fractional_offset, tmp_path +): + selected = {} + + async def handle(request, context, admit): + h = selected["h"] + wire = request["accounts"][0]["reporting_delivery_configs"][0] + await admit( + ReportingConfigurationAdmission( + h.production.offerings[0].offering_id, + h.item.config, + h.item.binding, + configuration_wire=wire, + ) + ) + state = state_for(h, copy.deepcopy(wire)) + if fractional_offset: + at = h.item.config.activated_at + assert at.microsecond == 0 + state["activated_at"] = ( + at.astimezone(timezone(timedelta(hours=5, minutes=45))).strftime( + "%Y-%m-%dT%H:%M:%S" + ) + + ".00000+05:45" + ) + if selected.get("response_mutation"): + state["configuration"]["method"]["destination"]["location"] = "other/location" + return { + "accounts": [ + { + "account_id": h.item.config.account_id, + "brand": {"domain": "advertiser.example.test"}, + "operator": "buyer.example.test", + "action": "unchanged", + "status": "active", + "billing": "operator", + "timezone": "UTC", + "reporting_delivery_configs": [state], + } + ] + } + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=notifications, + count=0, + account_handler=handle, + ) as h: + selected["h"] = h + mounted = MountedProduction(h) + mounted.authorize(h.item) + desired = wire_configuration(h) + request = { + "idempotency_key": "configuration-exact-replay-0001", + "accounts": [ + { + "account": {"account_id": h.item.config.account_id}, + "reporting_delivery_configs": [desired], + } + ], + } + async with mounted.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, result = await mounted.call( + client, "sync_accounts", request, transport=transport + ) + assert result.get("status") == "completed", result + validator = get_named_validator("account/sync-accounts-response.json") + assert not list(validator.iter_errors(result)), result + assert ( + result["accounts"][0]["reporting_delivery_configs"][0]["configuration"] + == desired + ) + expected_time = h.item.config.activated_at.isoformat() + if fractional_offset: + expected_time = ( + h.item.config.activated_at.astimezone( + timezone(timedelta(hours=5, minutes=45)) + ).strftime("%Y-%m-%dT%H:%M:%S") + + ".00000+05:45" + ) + assert ( + result["accounts"][0]["reporting_delivery_configs"][0]["activated_at"] + == expected_time + ) + before = await h.image() + frozen = await source_documents(h) + for where, name, value in ( + ("method.destination", "location", "different/provider/location"), + ("method.destination", "provider", {"domain": "different.example.test"}), + ("schedule", "delivery_sla", "PT2H"), + ("scope", "media_buy_ids", ["unrelated-media-buy"]), + ("", "report_definition_id", "different-definition"), + ): + bad = copy.deepcopy(request) + target = bad["accounts"][0]["reporting_delivery_configs"][0] + for segment in where.split(".") if where else (): + target = target[segment] + target[name] = value + _, rejected = await mounted.call(client, "sync_accounts", bad) + assert error_code(rejected) == "REPORTING_CONFIGURATION_UNAVAILABLE", rejected + assert "different/provider" not in json.dumps(rejected) + assert await h.image() == before + assert await source_documents(h) == frozen + selected["response_mutation"] = True + _, rejected = await mounted.call(client, "sync_accounts", request) + assert error_code(rejected) == "REPORTING_CONFIGURATION_UNADMITTED", rejected + assert "other/location" not in json.dumps(rejected) + selected.pop("response_mutation") + # Replays reauthorize the same consumer, even after an earlier + # accepted configuration and with the same transport key. + h.authorized_bindings.clear() + _, rejected = await mounted.call(client, "sync_accounts", request, transport="a2a-1.0") + assert error_code(rejected) == "UNAUTHORIZED", rejected + + +async def destination_documents(h): + if h.pool is None: + return copy.deepcopy(h.store._production_destination_bindings) + async with h.pool.connection() as connection: + return await ( + await connection.execute( + "SELECT * FROM reporting_production_destination_bindings" + " ORDER BY account_id,consumer_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_provider_binding_is_frozen_across_restart_and_later_mutation( + backend, notifications, tmp_path +): + path = tmp_path / "destination.sqlite" + async with production_harness(backend, path, notifications=notifications, count=0) as h: + support, item = h.production, h.item + await support.activate(account_id=item.config.account_id) + frozen = await destination_documents(h) + source = await source_documents(h) + request = feed_request(item) + first = await h.store.read_reporting_feed(request, caller=item.binding.principal) + walk_before = await walk(h.store, request, item.binding.principal, first=first) + with sqlite3.connect(path) as connection: + original = connection.execute( + "SELECT method FROM methods WHERE binding=?", (binding_fingerprint(item.binding),) + ).fetchone()[0] + changed = json.loads(original) + changed["destination"]["location"] = "new-location-same-label" + connection.execute( + "UPDATE methods SET method=? WHERE binding=?", + (json.dumps(changed), binding_fingerprint(item.binding)), + ) + assert item.writer.configuration_binding(item.binding).wire() == changed + before = await h.image() + with pytest.raises(ReportingNotificationError, match="destination_conflict"): + await h.store.admit_production_configuration( + item.config, item.binding, offering_id=support.offerings[0].offering_id + ) + assert await h.image() == before + denied = await item.claim() + assert not isinstance(denied, ReportingMaterializerLease), denied + assert item.writer.writes == 0 + assert await destination_documents(h) == frozen + assert await source_documents(h) == source + assert await walk(h.store, request, item.binding.principal, first=first) == walk_before + await support.aclose() + async with production_harness( + backend, + path, + notifications=notifications, + count=0, + existing_store=h.store if h.pool is None else None, + existing_pool=h.pool, + ) as restarted: + assert await destination_documents(restarted) == frozen + assert await source_documents(restarted) == source + assert ( + await walk(restarted.store, request, item.binding.principal, first=first) + == walk_before + ) + with sqlite3.connect(path) as connection: + connection.execute( + "UPDATE methods SET method=? WHERE binding=?", + (original, binding_fingerprint(item.binding)), + ) + await restarted.store.admit_production_configuration( + item.config, item.binding, offering_id=restarted.production.offerings[0].offering_id + ) + assert await destination_documents(restarted) == frozen + # The source mapping is also a semantic-generation contract, not + # just a lookup by business keys. Same key, changed definition fails. + old = restarted.production.offerings[0].producer._source.bindings[ + item.config.generation_key + ] + changed_config = replace(item.config, account_timezone="Etc/UTC") + with pytest.raises(ReportingWriterError): + old.check( + changed_config, + restarted.production.offerings[0].producer._source.capabilities, + restarted.production.offerings[0].source_offering_id, + ) + assert await source_documents(restarted) == source diff --git a/tests/conformance/reporting/test_reporting_production_lifecycle.py b/tests/conformance/reporting/test_reporting_production_lifecycle.py new file mode 100644 index 000000000..a7acd8af2 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_lifecycle.py @@ -0,0 +1,297 @@ +"""Actual source/I/O, mounted financial receipts, private frozen reads and health cycles.""" + +import copy +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingRevisionReceiptRecord, + revision_content_sha256, +) +from adcp.reporting.ledger.delivery import adjustment_to_wire, receipt_to_wire +from adcp.reporting.materializer.work import ReportingMaterializerLease +from adcp.reporting.ownership import page_revision_ownership +from adcp.validation.schema_loader import get_validator + +from ._generation_support import END +from ._production_support import production_harness +from ._production_transport import MountedProduction +from ._projection_support import drain +from ._receipt_transport import error_code +from .test_reporting_production_lock_order import source_turn +from .test_reporting_schedule_schema import assert_original_rejection + + +async def public_walk(mounted, client, request, *, first=None, transport="mcp"): + request = copy.deepcopy(request) + pages = [] + page = first + for _ in range(40): + if page is None: + _, page = await mounted.call( + client, "get_reporting_status", request, transport=transport + ) + assert page.get("status") == "completed", page + page_revision_ownership(page) + pages.append(page) + assert page["changes_checkpoint"] == pages[0]["changes_checkpoint"] + if not page["pagination"]["has_more"]: + return pages + request["pagination"]["cursor"] = page["pagination"]["cursor"] + page = None + pytest.fail("production cursor walk exceeded its bound") + + +async def generation(h): + account = h.item.config.account_id + await drain(h.projection, account) + values = [ + value.generation + for value in await h.projection.checkpoints(account_id=account) + if value.scope.consumer_id == h.item.binding.consumer_id + and value.scope.reporting_obligation_id == h.item.obligation.reporting_obligation_id + ] + assert values + return max(values) + + +async def observed_now(h): + # PostgreSQL owns materialization/receipt commit time. The app fixture's + # frozen clock predates I/O and is not a valid consumer observation time. + if h.pool is None: + return h.clock() + async with h.pool.connection() as connection: + return (await (await connection.execute("SELECT clock_timestamp()")).fetchone())[0] + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("feedback", [False, True]) +async def test_production_official_receipts_adjustment_and_frozen_wire_cycle( + backend, notifications, feedback, tmp_path +): + async with production_harness( + backend, + tmp_path / "provider.sqlite", + count=3, + source_publication=True, + reconciled=True, + notifications=notifications, + feedback=feedback, + ) as h: + support, item = h.production, h.item + snapshot_id = "retained-unlinked-snapshot" + old = replace( + item.revision, + reporting_revision_id=snapshot_id, + finality="snapshot", + finality_basis=None, + finality_policy_id=None, + finalized_at=None, + revision_content_sha256=revision_content_sha256( + reporting_revision_id=snapshot_id, + row_count=len(item.rows), + control_totals=item.revision.control_totals, + reporting_rows=item.rows, + control_total_evidence=item.revision.managed_control_totals, + ), + ) + await h.store.commit_revision(old, item.rows) + mounted = MountedProduction(h) + mounted.authorize(item) + await support.activate(account_id=item.config.account_id) + async with mounted.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps.get("status") == "completed", caps + claims = caps.get("media_buy", {}).get("reporting_delivery", {}) + if backend == "postgres": + assert claims["managed_delivery"] and claims["reconciled_billing"] + assert claims["receipt_task"] == "sync_reporting_receipts" + # Complete polling requires no optional HTTP delivery worker. + assert "readiness_notification" not in claims + else: + assert not claims + selected = await item.claim() + assert not isinstance(selected, ReportingMaterializerLease) + assert item.writer.writes == 0 # never falls back to the retained snapshot + source_result = await source_turn(support) + assert len(source_result.revisions_committed) == 1 and not source_result.slices_failed + revisions = await h.store.list_revisions( + account_id=item.config.account_id, + reporting_obligation_id=item.obligation.reporting_obligation_id, + ) + official = next(r for r in revisions if r.finality == "official") + assert {r.reporting_revision_id for r in revisions} == { + snapshot_id, + official.reporting_revision_id, + } + item.revision = official + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) + assert lease.attempt.reporting_revision_id == official.reporting_revision_id + prepared, verified = await item.verified(lease) + assert ( + await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + ).state == "verified" + before_receipts = await generation(h) + receipt = ReportingRevisionReceiptRecord( + item.scope, + "production-rejected", + official.reporting_revision_id, + lease.attempt.reporting_materialization_id, + "rejected", + item.binding.verification_profile, + official.row_count, + official.managed_control_totals, + await observed_now(h), + observed_canonical_content_digest=official.canonical_content_digest, + rejection_codes=("LOAD_FAILED",), + ) + request = { + "account": {"account_id": item.config.account_id}, + "idempotency_key": "production-rejected-0001", + "receipts": [receipt_to_wire(receipt)], + } + _, rejected = await mounted.call(client, "sync_reporting_receipts", request) + assert rejected["results"][0]["result"] == "recorded", rejected + after_rejection = await generation(h) + assert after_rejection > before_receipts + assert not isinstance(await item.claim(), ReportingMaterializerLease) + assert item.writer.writes == 1 # consumer rejection cannot schedule a retry + feed_request = { + "account": request["account"], + "view": "periods", + "pagination": {"max_results": 1}, + } + frozen = await public_walk(mounted, client, feed_request) + accepted = replace( + receipt, + reporting_receipt_id="production-accepted", + status="accepted", + supersedes_reporting_receipt_id=receipt.reporting_receipt_id, + rejection_codes=(), + observed_at=await observed_now(h), + ) + request = { + **request, + "idempotency_key": "production-accepted-0001", + "receipts": [receipt_to_wire(accepted)], + } + _, response = await mounted.call( + client, "sync_reporting_receipts", request, transport="a2a-1.0" + ) + assert response["results"][0]["result"] == "recorded", response + accepted_generation = await generation(h) + assert accepted_generation > after_rejection + + async def summary(): + _, value = await mounted.call( + client, + "get_reporting_status", + {"account": request["account"], "view": "summary"}, + ) + assert value.get("status") == "completed", value + return value + + assert (await summary())["health"] == "complete" + observed = await observed_now(h) + adjustment = ReportingAdjustmentRecord( + "production-adjustment", + item.config.account_id, + official.reporting_revision_id, + "source_correction", + END, + END + timedelta(days=30), + (("spend", "-0.50"),), + observed, + observed, + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-0.50", "decimal", "USD"), + ), + ) + await h.store.commit_adjustment(adjustment) + pending_generation = await generation(h) + assert pending_generation > accepted_generation + assert (await summary())["health"] != "complete" + replacement = replace( + accepted, + reporting_receipt_id="forbidden-accepted-replacement", + supersedes_reporting_receipt_id=accepted.reporting_receipt_id, + ) + mixed = { + "account": request["account"], + "idempotency_key": "production-mixed-final-0001", + "receipts": [receipt_to_wire(replacement)], + "adjustment_receipts": [ + { + "reporting_receipt_id": "production-adjustment-accepted", + "reporting_adjustment_id": adjustment.reporting_adjustment_id, + "adjusts_reporting_revision_id": official.reporting_revision_id, + "status": "accepted", + "observed_adjustment_sha256": adjustment_to_wire(adjustment)[ + "canonical_adjustment_sha256" + ], + "observed_at": (await observed_now(h)).isoformat(), + } + ], + } + _, mixed_response = await mounted.call( + client, "sync_reporting_receipts", mixed, transport="a2a-0.3" + ) + assert mixed_response["results"][0]["reporting_receipt_id"] == ( + replacement.reporting_receipt_id + ) + assert mixed_response["results"][0]["errors"][0]["code"] == ( + "ACCEPTED_RECEIPT_TERMINAL" + ) + assert mixed_response["results"][1]["result"] == "recorded", mixed_response + complete_generation = await generation(h) + assert complete_generation > pending_generation + assert (await summary())["health"] == "complete" + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, complete = await mounted.call( + client, + "get_reporting_status", + {"account": request["account"], "view": "summary"}, + transport=transport, + ) + assert complete["health"] == "complete" + assert "next_expected_at" in complete + assert_original_rejection(complete) + get_validator("get_reporting_status", "sync").validate(complete) + _, replay = await mounted.call( + client, "sync_reporting_receipts", mixed, transport=transport + ) + assert replay == mixed_response + assert await generation(h) == complete_generation + assert ( + await public_walk( + mounted, client, feed_request, first=frozen[0], transport=transport + ) + == frozen + ) + _, exact = await mounted.call( + client, + "get_media_buy_delivery", + { + "account": request["account"], + "reporting_revision_id": official.reporting_revision_id, + }, + transport=transport, + ) + assert exact["reporting_rows"] == item.rows, exact + assert page_revision_ownership(exact) == { + official.reporting_revision_id: item.obligation.reporting_obligation_id + } + assert item.writer.writes == 1 + assert not (await h.queue())[0] # epoch-zero readiness stays quarantined + h.authorized_bindings.clear() + _, denied = await mounted.call(client, "sync_reporting_receipts", mixed) + assert error_code(denied) == "UNAUTHORIZED" diff --git a/tests/conformance/reporting/test_reporting_production_lock_order.py b/tests/conformance/reporting/test_reporting_production_lock_order.py new file mode 100644 index 000000000..276ef6f59 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_lock_order.py @@ -0,0 +1,265 @@ +"""Coordinated real lock order, with the executable wrong-order control restored.""" + +import asyncio +import json +from dataclasses import replace +from datetime import timedelta +from types import MethodType + +import pytest + +from adcp.reporting.ledger.pg import PgReportingLedgerStore + +from ._generation_support import END +from ._production_support import production_harness + + +async def source_turn(support): + producer = support.offerings[0].producer + token = support._producer_turn.set(producer) + try: + return await producer.run_worker() + finally: + support._producer_turn.reset(token) + + +@pytest.mark.parametrize("wrong_order", [False, True]) +async def test_activation_and_producer_actual_trigger_lock_order( + wrong_order, tmp_path, monkeypatch +): + psycopg = pytest.importorskip("psycopg") + async with production_harness("postgres", tmp_path / "destination.sqlite") as h: + support, store, item = h.production, h.store, h.item + locked, release, executing = asyncio.Event(), asyncio.Event(), asyncio.Event() + pids = {} + original_step = support.projection._activation_step_on + original_execute = psycopg.AsyncConnection.execute + original_lease = store.lease_period_close.__func__ + tasks = [] + activation = None + + async def activation_step(connection, account_id): + # This phase already owns the real account lock and will acquire + # configuration FK locks while installing the live checkpoints. + # Earlier activation phases commit their own bounded transactions. + if asyncio.current_task() is activation and not locked.is_set(): + pids["activation"] = connection.info.backend_pid + locked.set() + await asyncio.wait_for(release.wait(), 5) + return await original_step(connection, account_id) + + async def observe(connection, query, *args, **kwargs): + if isinstance(query, str) and query.startswith( + "UPDATE reporting_configurations SET lease_worker_id" + ): + pids["producer"] = connection.info.backend_pid + executing.set() + return await original_execute(connection, query, *args, **kwargs) + + async def wait_for_actual_trigger_wait(): + await asyncio.wait_for(executing.wait(), 5) + async with h.pool.connection() as observer: + for _ in range(100): + row = await ( + await observer.execute( + "SELECT EXISTS(SELECT 1 FROM pg_locks WHERE pid=%s" + " AND locktype='advisory' AND NOT granted)," + " %s=ANY(pg_blocking_pids(%s))", + (pids["producer"], pids["activation"], pids["producer"]), + ) + ).fetchone() + if row == (True, True): + return + await asyncio.sleep(0.01) + pytest.fail("the producer did not block in the real account-lock trigger") + + # Both modes retain the inherited trigger and its original function. + async with h.pool.connection() as c: + trigger = await ( + await c.execute( + "SELECT t.tgenabled,pg_get_functiondef(t.tgfoid)" + " FROM pg_trigger t WHERE t.tgrelid='reporting_configurations'::regclass" + " AND t.tgname='reporting_materializer_configuration'" + ) + ).fetchone() + assert trigger[0] == "O" and "pg_advisory_xact_lock" in trigger[1] + with monkeypatch.context() as patch: + patch.setattr(support.projection, "_activation_step_on", activation_step) + patch.setattr(psycopg.AsyncConnection, "execute", observe) + if wrong_order: + # Restore the actual predecessor acquisition; do not emulate + # its result or replace the lock-taking database trigger. + patch.setattr( + store, + "lease_period_close", + MethodType(PgReportingLedgerStore.lease_period_close, store), + ) + activation = asyncio.create_task(support.activate(account_id=item.config.account_id)) + tasks.append(activation) + try: + await asyncio.wait_for(locked.wait(), 5) + producer = asyncio.create_task(source_turn(support)) + tasks.append(producer) + if wrong_order: + await wait_for_actual_trigger_wait() + else: + turn = await asyncio.wait_for(asyncio.shield(producer), 5) + assert turn.leased is None + assert not executing.is_set() + release.set() + results = await asyncio.wait_for( + asyncio.gather(activation, producer, return_exceptions=True), 8 + ) + finally: + release.set() + for task in tasks: + if not task.done(): + task.cancel() + await asyncio.gather(*tasks, return_exceptions=True) + assert store.lease_period_close.__func__ is original_lease + deadlocks = [r for r in results if isinstance(r, psycopg.errors.DeadlockDetected)] + assert len(deadlocks) == int(wrong_order) + assert all(not isinstance(r, BaseException) or r in deadlocks for r in results) + + async with h.pool.connection() as c: + rows = await ( + await c.execute( + "SELECT (SELECT count(*) FROM reporting_projection_accounts)," + " (SELECT count(*) FROM reporting_production_accounts)," + " (SELECT count(*) FROM reporting_production_work)," + " (SELECT count(*) FROM reporting_materializer_work)," + " (SELECT count(*) FROM reporting_production_notification_events)" + ) + ).fetchone() + lease = await ( + await c.execute( + "SELECT lease_worker_id,lease_expires_at FROM reporting_configurations" + " WHERE account_id=%s", + (item.config.account_id,), + ) + ).fetchone() + if isinstance(results[0], BaseException): + assert rows == (1, 0, 0, 0, 0) + assert not await support.projection.baseline_ready(account_id=item.config.account_id) + else: + assert rows == (1, 1, 0, 0, 0) + # A failed lease rolls back; a winning producer releases in its real + # finally block. Neither path can leak a lease or reserve epoch-zero I/O. + assert lease == (None, None) + assert ( + await store.get_revision( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + ) + == item.revision + ) + # The negative control restores the real methods before resuming the + # incomplete phase. Its original input and epoch-zero queues persist. + await support.activate(account_id=item.config.account_id) + assert await support.projection.baseline_ready(account_id=item.config.account_id) + if not wrong_order: + executing.clear() + with monkeypatch.context() as patch: + patch.setattr(psycopg.AsyncConnection, "execute", observe) + turn = await asyncio.wait_for(source_turn(support), 5) + assert turn.leased is not None and executing.is_set() + assert turn.revisions_committed == [] + print( + json.dumps( + { + "production_lock_order": "wrong_order_control" if wrong_order else "restored", + "actual_trigger": True, + "observed_trigger_wait": wrong_order, + "deadlocks": len(deadlocks), + "rollback_or_commit_verified": True, + "restored_acquisition": store.lease_period_close.__func__ is original_lease, + } + ) + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_selected_source_enrollment_and_expired_lease_fairness(backend, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", second_source=True + ) as h: + support, store, item = h.production, h.store, h.item + selected, other = support.offerings + peer = replace(item.config, delivery_config_id="peer") + other_config = replace(item.config, delivery_config_id="other-source") + unadmitted = replace(item.config, delivery_config_id="unadmitted") + outside = replace(item.config, account_id="acct_b") + for configuration in (peer, other_config, unadmitted, outside): + await store.put_configuration(configuration) + before = { + account: await store.list_configurations(account_id=account) + for account in ("acct_a", "acct_b") + } + + async def acquire(offering, worker, now=END): + token = support._producer_turn.set(offering.producer) + try: + return await store.lease_period_close(worker_id=worker, now=now, lease_seconds=1) + finally: + support._producer_turn.reset(token) + + assert await acquire(selected, "preactivation") is None + assert await acquire(other, "preactivation") is None + await support.activate(account_id="acct_a") + # Two actual source instances intentionally have identical report + # contracts. Only explicit, verified offering admission chooses one. + assert await acquire(selected, "unbound") is None + for configuration, offering in ( + (item.config, selected), + (peer, selected), + (other_config, other), + ): + offering.producer._source.bind_generation(configuration) + binding = replace(item.binding, generation_key=configuration.generation_key) + item.writer.grant(binding) + await store.admit_production_configuration( + configuration, binding, offering_id=offering.offering_id + ) + first = await acquire(selected, "crashed") + second = await acquire(selected, "live") + assert first.generation_key == item.config.generation_key + assert second.generation_key == peer.generation_key + await store.release_period_close(second, worker_id="live") + expired = await acquire(selected, "replacement", END + timedelta(seconds=2)) + assert expired.generation_key == first.generation_key + await store.release_period_close(expired, worker_id="replacement") + isolated = await acquire(other, "other") + assert isolated.generation_key == other_config.generation_key + await store.release_period_close(isolated, worker_id="other") + assert { + account: await store.list_configurations(account_id=account) + for account in ("acct_a", "acct_b") + } == before + if h.pool is None: + turns = store._lease_turns + assert unadmitted.generation_key not in turns + assert outside.generation_key not in turns + assert not store._leases + else: + async with h.pool.connection() as c: + turns = await ( + await c.execute( + "SELECT account_id,delivery_config_id" + " FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id" + ) + ).fetchall() + assert turns == [ + ("acct_a", "daily"), + ("acct_a", "other-source"), + ("acct_a", "peer"), + ] + assert ( + await ( + await c.execute( + "SELECT count(*) FROM reporting_configurations" + " WHERE lease_worker_id IS NOT NULL" + ) + ).fetchone() + )[0] == 0 + assert not (await h.queue())[0] diff --git a/tests/conformance/reporting/test_reporting_production_migration.py b/tests/conformance/reporting/test_reporting_production_migration.py new file mode 100644 index 000000000..4bf1673e1 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_migration.py @@ -0,0 +1,252 @@ +"""Populated, concurrent and interrupted B2.4 bootstrap with old objects intact.""" + +import asyncio +import json +from copy import deepcopy +from datetime import timedelta +from importlib.resources import files + +import pytest + +from adcp.reporting.feed import PgReportingFeedStore +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.materializer import ReportingMaterializerLease +from adcp.reporting.outbox._schema import schema_objects +from adcp.reporting.outbox.status_pg import PgStatusNotificationStore +from adcp.reporting.production.pg import PgReportingProductionStore +from adcp.reporting.production.schema import validate_production_schema +from adcp.reporting.projection.schema import validate_projection_schema + +from ._durable_materializer_support import DurableHarness, durable_case +from ._feed_support import feed_request, mixed_case, walk +from ._generation_support import isolated_reporting_pool +from ._reconciliation_support import Clock +from .test_reporting_feed_migration import fairness + + +def manifests(): + return { + package: json.loads( + files("adcp.reporting." + package).joinpath("required_schema.json").read_text() + ) + for package in ("materializer", "receipts", "feed", "projection", "production") + } + + +def original_rows(image, before): + # This one documented additive column fences newly activated checkpoints. + # Its default leaves old C checkpoints compatible until explicit activation. + result = deepcopy({key: image[key] for key in before}) + for (row,) in result.get("reporting_status_scope_checkpoints", []): + assert row.pop("projection_writer_floor", 1) == 1 + return result + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("autocommit", [False, True]) +async def test_populated_repeat_concurrent_migration_keeps_history_fairness_and_frozen_pages( + notifications, autocommit +): + async with isolated_reporting_pool(autocommit=autocommit) as pool: + parent = PgReportingFeedStore(pool=pool, notifications=notifications) + await parent.create_schema() + # Install the optional historical C objects through their supported + # notification-enabled owner. The actual receipt/feed writer retains + # this cell's requested off/on mode. + old_projection = PgStatusNotificationStore( + PgReportingFeedStore(pool=pool, notifications=True) + ) + await old_projection.create_schema() + h = DurableHarness(parent, Clock(), pool) + case, request, response = await mixed_case(h) + pending = await durable_case(parent, account="pending-account") + lease = None + for _ in range(8): + candidate = await pending.claim() + if isinstance(candidate, ReportingMaterializerLease): + lease = candidate + break + assert lease is not None and lease.admission_epoch == 0 + if notifications: + await old_projection.baseline(account_id=case.obligation.account_id) + first = await parent.read_reporting_feed(feed_request(case), caller=case.binding.principal) + original = await parent.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=case.binding.principal + ) + expected = await walk(parent, feed_request(case), case.binding.principal, first=first) + before = await h.image() + old_turns = await fairness(pool) + async with pool.connection() as c: + old_objects = await schema_objects(c) + with pytest.raises(ReportingNotificationError): + await validate_production_schema(c, notifications=notifications) + child = PgReportingProductionStore(pool=pool, notifications=notifications) + await asyncio.wait_for(asyncio.gather(*(child.create_schema() for _ in range(3))), 30) + await child.create_schema() + async with pool.connection() as c: + current = await schema_objects(c) + await validate_projection_schema(c, notifications=notifications) + await validate_production_schema(c, notifications=notifications) + assert ( + await ( + await c.execute("SELECT count(*) FROM reporting_production_accounts") + ).fetchone() + )[0] == 0 + assert {key: current[key] for key in old_objects} == old_objects + required = manifests() + assert set(required["projection"]).isdisjoint(required["production"]) + assert set(current) - set(old_objects) == set(required["projection"]) | set( + required["production"] + ) + for objects in required.values(): + assert all(current.get(key) == value for key, value in objects.items()) + assert original_rows(await h.image(), before) == before + assert await fairness(pool) == old_turns + assert await parent.ingest_receipt_batch(request, caller=case.binding.principal) == response + assert ( + await walk(child, feed_request(case), case.binding.principal, first=first) == expected + ) + assert ( + await child.read_reporting_feed_snapshot( + original.snapshot_id, caller=case.binding.principal + ) + == original + ) + assert original_rows(await h.image(), before) == before + print( + json.dumps( + { + "b24_migration": "concurrent-repeat", + "notifications": notifications, + "autocommit": autocommit, + "preserved_objects": len(old_objects), + "isolated_additions": { + key: len(required[key]) for key in ("projection", "production") + }, + "pending_epoch": lease.admission_epoch, + "old_pages": len(expected[0]), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_interrupted_complete_migration_rolls_back_every_new_object( + notifications, monkeypatch +): + async with isolated_reporting_pool(autocommit=True) as pool: + parent = PgReportingFeedStore(pool=pool, notifications=notifications) + await parent.create_schema() + await PgStatusNotificationStore( + PgReportingFeedStore(pool=pool, notifications=True) + ).create_schema() + h = DurableHarness(parent, Clock(), pool) + case, request, response = await mixed_case(h) + before = await h.image() + async with pool.connection() as c: + original = await schema_objects(c) + child = PgReportingProductionStore(pool=pool, notifications=notifications) + entered = asyncio.Event() + from psycopg import AsyncConnection + + execute = AsyncConnection.execute + + async def interrupt(connection, query, *args, **kwargs): + result = await execute(connection, query, *args, **kwargs) + if isinstance(query, str) and query.startswith("-- B2.4 production admission."): + entered.set() + await asyncio.Event().wait() + return result + + with monkeypatch.context() as patch: + patch.setattr(AsyncConnection, "execute", interrupt) + task = asyncio.create_task(child.create_schema()) + try: + await asyncio.wait_for(entered.wait(), 20) + async with pool.connection() as c: + # Catalog deparsing can acquire a relation lock behind + # the intentionally paused ALTER TABLE. Observe raw MVCC + # catalog visibility now; compare every definition after + # cancellation releases those DDL locks. + assert ( + await ( + await c.execute( + "SELECT count(*) FROM pg_class c" + " JOIN pg_namespace n ON n.oid=c.relnamespace" + " WHERE n.nspname=current_schema()" + " AND c.relname='reporting_production_delivery_windows'" + ) + ).fetchone() + )[0] == 0 + finally: + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert await h.image() == before + async with pool.connection() as c: + assert await schema_objects(c) == original + await child.create_schema() + assert original_rows(await h.image(), before) == before + assert await parent.ingest_receipt_batch(request, caller=case.binding.principal) == response + + +@pytest.mark.parametrize( + "damage", + [ + "ALTER TABLE reporting_production_delivery_windows" + " DISABLE TRIGGER reporting_production_delivery_window_immutable", + "ALTER TABLE reporting_production_delivery_windows ALTER COLUMN expires_at DROP NOT NULL", + "DROP TABLE reporting_production_delivery_windows", + "DROP INDEX reporting_production_source_pending", + "ALTER TABLE reporting_projection_inputs" + " DISABLE TRIGGER reporting_projection_input_immutable", + "ALTER TABLE reporting_status_scope_checkpoints" + " DISABLE TRIGGER reporting_projection_checkpoint_guard", + ], +) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_partial_or_mismatched_production_objects_refuse_fresh_readiness( + damage, notifications +): + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingProductionStore(pool=pool, notifications=notifications) + await store.create_schema() + async with pool.connection() as c: + await c.execute(damage) + with pytest.raises(ReportingNotificationError) as caught: + await validate_production_schema(c, notifications=notifications) + assert caught.value.code == "reporting_production_schema_unready" + + +async def test_retry_window_is_immutable_and_repeated_bootstrap_never_restarts_deadline(): + async with isolated_reporting_pool(autocommit=True) as pool: + store = PgReportingProductionStore(pool=pool) + await store.create_schema() + started = Clock()() + row = ( + "account-window", + "window-key", + "core", + "a" * 64, + started, + started + timedelta(seconds=86400), + ) + async with pool.connection() as c: + await c.execute( + "INSERT INTO reporting_production_delivery_windows VALUES(%s,%s,%s,%s,%s,%s)", row + ) + for statement in ( + "UPDATE reporting_production_delivery_windows" + " SET expires_at=expires_at+interval '1 second'", + "DELETE FROM reporting_production_delivery_windows", + ): + async with pool.connection() as c: + with pytest.raises(Exception): + async with c.transaction(): + await c.execute(statement) + await store.create_schema() + async with pool.connection() as c: + assert await ( + await c.execute("SELECT * FROM reporting_production_delivery_windows") + ).fetchall() == [row] diff --git a/tests/conformance/reporting/test_reporting_production_notifications.py b/tests/conformance/reporting/test_reporting_production_notifications.py new file mode 100644 index 000000000..769c997ba --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_notifications.py @@ -0,0 +1,688 @@ +"""All actual production queues use durable fanout and signed at-least-once delivery.""" + +import asyncio +import json +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import timedelta +from types import SimpleNamespace + +import pytest + +from adcp.reporting.outbox import validate_notification_payload +from adcp.reporting.production.notifications import production_notification_workers +from adcp.signing.jwks import StaticJwksResolver +from adcp.signing.webhook_verifier import WebhookVerifyOptions, verify_webhook_signature + +from ._generation_support import configuration, obligation_for, revision_for +from ._production_support import production_harness +from ._production_transport import MountedProduction +from ._projection_support import drain +from ._reliable_support import ( + DeterministicReceiverStore, + ScriptedNotificationReceiver, + SimulatedCrash, + _BytesStore, + notification_subscription, + notification_verification_keys, +) + +EVENTS = ( + "reporting.ledger_changed", + "reporting.status_changed", + "reporting.delivery_ready", +) + + +@asynccontextmanager +async def queued_production(backend, tmp_path, monkeypatch): + async with production_harness( + backend, + tmp_path / "destination.sqlite", + count=0, + notifications=True, + notification_delivery=True, + ) as h: + item, support = h.item, h.production + for subscriber, principal in ( + ("buyer", item.binding.consumer_id), + ("second", item.binding.consumer_id), + ("outsider", "https://buyer.example.test/other"), + ): + h.subscriptions.put( + notification_subscription( + subscriber=subscriber, + principal=principal, + events=EVENTS, + url="https://receiver.example.test/reporting", + ) + ) + h.subscriptions.put(notification_subscription(account="acct_b", events=EVENTS)) + blobs = _BytesStore(h.pool) + await blobs.create_schema() + receiver_store = DeterministicReceiverStore(blobs, h.notification_failures) + receiver = ScriptedNotificationReceiver( + SimpleNamespace( + clock=h.clock, failures=h.notification_failures, receiver=receiver_store + ) + ) + receiver.install(monkeypatch) + # The harness's original Core event was already expanded at startup, + # before registrations existed. Publish a new ordinary Core revision + # through the real ledger transaction after registering recipients. + core = replace(configuration(), delivery_config_id="ordinary-core") + await h.store.put_configuration(core) + obligation = await h.store.commit_obligation( + replace(obligation_for(core), reporting_obligation_id="ordinary-core-obligation") + ) + revision, rows = revision_for(obligation, suffix="ordinary-core") + await h.store.commit_revision(revision, rows) + await support.activate(account_id=item.config.account_id) + assert (await support.materializer.run_once()).state == "verified" + assert item.writer.writes == 1 + for readable in (False, True): + await h.store.set_revision_readable( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + readable=readable, + ) + await drain(h.projection, item.config.account_id) + if h.pool is not None: + mount = MountedProduction(h) + mount.authorize(item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, response = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + reporting = response["media_buy"]["reporting_delivery"] + assert reporting["managed_delivery"] is True + assert response["webhook_signing"] == { + "supported": True, + "profile": "adcp/webhook-signing/v1", + "algorithms": ["ed25519"], + "legacy_hmac_fallback": False, + "delivery_retry_horizon_seconds": 86400, + } + assert [ + reporting[k] + for k in ( + "ledger_notification", + "status_notification", + "readiness_notification", + ) + ] == list(EVENTS) + workers = support.notification_workers + await support.aclose() + assert all([await w.outbox.list_events(account_id="acct_a") for w in workers]) + h.receiver, h.receiver_store = receiver, receiver_store + h.workers = workers + yield h + + +def fresh_workers(h): + return production_notification_workers( + h.store, + h.projection, + subscriptions=h.subscriptions, + cipher=h.workers[0].cipher, + signing=h.workers[0].signing, + ) + + +async def expire_queue_lease(h, worker, *, expansion=False): + if h.pool is None: + h.clock.advance(timedelta(seconds=61)) + return + table = ( + "reporting_notification_expansions" if expansion else "reporting_notification_deliveries" + ) + # The fixed SDK queue adapter maps this identifier to its actual queue. + async with worker.outbox._connection() as c: + await c.execute( + f"UPDATE {table} SET lease_expires_at=clock_timestamp()-interval '1 second'" + " WHERE account_id=%s AND state='leased'", + ("acct_a",), + ) + + +async def retained_windows(h): + if h.pool is None: + return tuple( + sorted((*key, *value) for key, value in h.store._production_delivery_windows.items()) + ) + async with h.pool.connection() as connection: + return tuple( + await ( + await connection.execute( + "SELECT account_id,idempotency_key,queue,body_sha256,started_at,expires_at" + " FROM reporting_production_delivery_windows" + " ORDER BY account_id,idempotency_key" + ) + ).fetchall() + ) + + +def use_clock(h, moment): + h.clock.now = moment + h.store._clock = h.clock + # The status queue is retained by the projector; the other two are + # reconstructed by the factory. All database clock seams must agree. + for worker in h.workers: + worker.outbox._clock = h.clock + + +async def pin_current_clock(h): + moment = h.clock() + if h.pool is not None: + from adcp.reporting.outbox.pg import database_now + + async with h.pool.connection() as connection: + moment = await database_now(connection, None) + use_clock(h, moment) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("fault", [False, True], ids=["same-anchor", "reservation-fault"]) +async def test_retry_window_and_http_attempt_share_the_reservation_boundary( + backend, fault, tmp_path, monkeypatch +): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[2] + await pin_current_clock(h) + assert await worker.expand_one(account_id="acct_a") + calls = 0 + with monkeypatch.context() as patch: + if h.pool is None: + if fault: + import adcp.reporting.outbox.memory as memory + + original = memory.token_hex + + def fail(*args): + nonlocal calls + calls += 1 + if calls == 2: + raise ReportingNotificationError("injected_reservation_failure") + return original(*args) + + patch.setattr(memory, "token_hex", fail) + else: + original = worker.outbox._next_attempt_on + + async def step(*args): + result = await original(*args) + if fault: + raise ReportingNotificationError("injected_reservation_failure") + # Deterministic time passes while the transaction reserves + # its ordinal. The persisted anchor must use fired_at. + h.clock.advance(timedelta(microseconds=1)) + return result + + patch.setattr(worker.outbox, "_next_attempt_on", step) + if fault: + with pytest.raises(ReportingNotificationError, match="injected_reservation"): + await worker.deliver_one(account_id="acct_a") + else: + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + windows = await retained_windows(h) + activity = await worker.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + if fault: + if h.pool is None: + assert calls == 2 # after lease claim, at the HTTP reservation + assert not windows, "failed HTTP reservation retained a first-attempt window" + assert not activity + assert not h.receiver.received + else: + assert len(windows) == len(activity) == 1 + assert windows[0][4] == activity[0].fired_at + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_window_insert_failure_rolls_back_attempt_head_and_all_state( + backend, queue, tmp_path, monkeypatch +): + from adcp.reporting.ledger.notification_models import ReportingNotificationError + from adcp.reporting.outbox.activity import ActivityRequest + + async with queued_production(backend, tmp_path, monkeypatch) as h: + await pin_current_clock(h) + worker = h.workers[queue] + assert await worker.expand_one(account_id="acct_a") + lease = await worker.outbox.claim_delivery( + account_id="acct_a", now=h.clock(), lease_seconds=60 + ) + assert lease is not None + with monkeypatch.context() as patch: + if h.pool is None: + + class FaultWindows(dict): + def setdefault(self, *args): + super().setdefault(*args) + raise ReportingNotificationError("injected_window_failure") + + patch.setattr(h.store, "_production_delivery_windows", FaultWindows()) + else: + original = worker.outbox._connection + + class FaultConnection: + def __init__(self, connection): + self.connection = connection + + def transaction(self): + return self.connection.transaction() + + async def execute(self, query, params=None): + result = await self.connection.execute(query, params) + if query.startswith("INSERT INTO reporting_production_delivery_windows"): + raise ReportingNotificationError("injected_window_failure") + return result + + @asynccontextmanager + async def connection(): + async with original() as bound: + yield FaultConnection(bound) + + patch.setattr(worker.outbox, "_connection", connection) + before = await h.image() + with pytest.raises(ReportingNotificationError, match="injected_window_failure"): + await worker.delivery_window.reserve_attempt( + worker.outbox, + lease, + request=ActivityRequest("https://receiver.example.test/reporting", 1), + now=h.clock(), + ) + assert await h.image() == before + assert not await retained_windows(h) + assert not await worker.outbox.list_activity( + account_id="acct_a", consumer_id=lease.delivery.binding.principal_id + ) + attempt, deadline, expired = await worker.delivery_window.reserve_attempt( + worker.outbox, + lease, + request=ActivityRequest("https://receiver.example.test/reporting", 1), + now=h.clock(), + ) + assert not expired and attempt.attempt == 1 + assert deadline == attempt.fired_at + timedelta(seconds=86400) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_timeout_backoff_duplicate_workers_and_configuration_change_keep_first_deadline( + backend, queue, tmp_path, monkeypatch, caplog +): + import httpx + + async with queued_production(backend, tmp_path, monkeypatch) as h: + await pin_current_clock(h) + for key in tuple(h.subscriptions.values): + if key[0] == "acct_a" and key[1] != "buyer": + del h.subscriptions.values[key] + worker = h.workers[queue] + assert await worker.expand_one(account_id="acct_a") + h.receiver.responses["buyer"].extend([429, httpx.ReadTimeout("controlled timeout")]) + assert await worker.deliver_one(account_id="acct_a") + original = await retained_windows(h) + assert len(original) == 1 + first = (await worker.outbox.list_deliveries(account_id="acct_a"))[0] + assert first.state == "pending" + duplicate = fresh_workers(h)[queue] + assert await asyncio.gather( + worker.deliver_one(account_id="acct_a"), duplicate.deliver_one(account_id="acct_a") + ) == [False, False] + h.clock.advance(timedelta(seconds=5)) + # A reconstructed worker with a longer retry interval cannot postpone + # expiration or turn it into another day of delivery eligibility. + restarted = fresh_workers(h)[queue] + restarted.retry_seconds = 2 * 86400 + assert await restarted.deliver_one(account_id="acct_a") + assert await retained_windows(h) == original + await h.store.put_configuration(replace(h.item.config, deactivated_at=h.clock())) + h.signing.generation = 2 + use_clock(h, original[0][5] - timedelta(microseconds=1)) + assert not await fresh_workers(h)[queue].deliver_one(account_id="acct_a") + use_clock(h, original[0][5]) + current, duplicate = fresh_workers(h)[queue], fresh_workers(h)[queue] + assert sorted( + await asyncio.gather( + current.deliver_one(account_id="acct_a"), + duplicate.deliver_one(account_id="acct_a"), + ) + ) == [False, True] + final = (await current.outbox.list_deliveries(account_id="acct_a"))[0] + assert final.delivery == first.delivery + assert (final.state, final.error_code) == ("suppressed", "lease_expired") + assert await retained_windows(h) == original + activity = await current.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + assert [r.outcome.status for r in activity] == ["timeout", "failed"] + assert [r.attempt for r in activity] == [2, 1] + assert len(h.receiver.received) == 1 + assert not await current.deliver_one(account_id="acct_b") + assert not caplog.records + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_worker_timeout_after_reservation_preserves_original_window_and_pending_activity( + backend, queue, tmp_path, monkeypatch +): + from ._reliable_support import Barrier + + async with queued_production(backend, tmp_path, monkeypatch) as h: + await pin_current_clock(h) + for key in tuple(h.subscriptions.values): + if key[0] == "acct_a" and key[1] != "buyer": + del h.subscriptions.values[key] + worker = h.workers[queue] + worker.lease_seconds = 1 + assert await worker.expand_one(account_id="acct_a") + barrier = Barrier() + h.notification_failures.at("http.before", barrier) + task = asyncio.create_task(worker.deliver_one(account_id="acct_a")) + try: + await barrier.wait() + original = await retained_windows(h) + assert len(original) == 1 + use_clock(h, original[0][5]) + assert await asyncio.wait_for(task, 3) + finally: + barrier.release() + if not task.done(): + task.cancel() + await asyncio.gather(task, return_exceptions=True) + restarted = fresh_workers(h)[queue] + assert await restarted.deliver_one(account_id="acct_a") + assert await retained_windows(h) == original + activity = await restarted.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + assert len(activity) == 1 and activity[0].outcome is None + assert not h.receiver.received + assert (await restarted.outbox.list_deliveries(account_id="acct_a"))[ + 0 + ].state == "suppressed" + + +async def test_pg_deadline_crossed_while_reserving_an_ordinal_rolls_it_back(tmp_path, monkeypatch): + async with queued_production("postgres", tmp_path, monkeypatch) as h: + worker = h.workers[2] + assert await worker.expand_one(account_id="acct_a") + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + original = await retained_windows(h) + use_clock(h, original[0][5] - timedelta(microseconds=1)) + restarted = fresh_workers(h)[2] + before = await restarted.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + original_next = restarted.outbox._next_attempt_on + + async def cross_deadline(*args): + ordinal = await original_next(*args) + h.clock.advance(timedelta(microseconds=1)) + return ordinal + + monkeypatch.setattr(restarted.outbox, "_next_attempt_on", cross_deadline) + assert await restarted.deliver_one(account_id="acct_a") + assert len(h.receiver.received) == 1 + assert await retained_windows(h) == original + assert ( + await restarted.outbox.list_activity( + account_id="acct_a", consumer_id=h.item.binding.consumer_id + ) + == before + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +@pytest.mark.parametrize("offset", [-1, 0, 1], ids=["before", "exact", "after"]) +async def test_retry_horizon_is_immutable_across_crash_rotation_and_restart( + backend, queue, offset, tmp_path, monkeypatch +): + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[queue] + assert await worker.expand_one(account_id="acct_a") + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + received = h.receiver.received[0] + original = await retained_windows(h) + assert len(original) == 1 + row = original[0] + assert row[1] == received.idempotency_key + assert (row[5] - row[4]).total_seconds() == 86400 + # Only the deterministic database clock seam is advanced. The immutable + # persisted timestamps/bytes are not edited to fabricate expiration. + use_clock(h, row[5] + timedelta(microseconds=offset)) + h.signing.generation = 2 + restarted = fresh_workers(h)[queue] + assert await restarted.deliver_one(account_id="acct_a") + received_count = len(h.receiver.received) + assert received_count == (2 if offset < 0 else 1) + assert await retained_windows(h) == original + target = next( + value + for value in await restarted.outbox.list_deliveries(account_id="acct_a") + if value.delivery.binding.idempotency_key == received.idempotency_key + ) + assert (target.state, target.error_code) == ( + ("complete", None) if offset < 0 else ("suppressed", "lease_expired") + ) + assert await h.receiver_store.read("acct_a", received.idempotency_key) == received.body + # Expiry cannot erase the original ambiguous attempt. Existing public + # account-activity projection and authenticated polling still recover + # the retained history; no fabricated late HTTP outcome is inserted. + from adcp.decisioning.accounts import ResolveContext + from adcp.decisioning.context import AuthInfo + from adcp.reporting.outbox import ReportingActivityProjector + + principal = h.subscriptions.values[("acct_a", received.subscriber_id)].principal_id + activity = ReportingActivityProjector(restarted.outbox) + rows = await activity.for_account( + account_id="acct_a", + context=ResolveContext(auth_info=AuthInfo(kind="bearer", principal=principal)), + ) + assert len(rows) == (2 if offset < 0 else 1) + assert ( + await activity.for_account( + account_id="acct_b", + context=ResolveContext(auth_info=AuthInfo(kind="bearer", principal=principal)), + ) + == [] + ) + mount = MountedProduction(h) + mount.authorize(h.item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, polling = await mount.call( + client, + "get_reporting_status", + {"account": {"account_id": "acct_a"}, "view": "periods"}, + transport=transport, + ) + assert polling["status"] == "completed" + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_actual_queues_retry_exact_signed_bytes_after_acceptance_before_ack( + backend, queue, tmp_path, monkeypatch +): + async with queued_production(backend, tmp_path, monkeypatch) as h: + original_quarantine = await h.queue() + for worker in h.workers: + for _ in range(100): + if not await worker.expand_one(account_id="acct_a"): + break + else: + pytest.fail("production fanout did not reach a bounded idle state") + worker = h.workers[queue] + assert await worker.outbox.list_deliveries(account_id="acct_a") + h.notification_failures.at("http.accepted", SimulatedCrash()) + with pytest.raises(SimulatedCrash): + await worker.deliver_one(account_id="acct_a") + first = h.receiver.received[-1] + assert json.loads(first.body)["notification_type"] == EVENTS[queue] + assert await h.receiver_store.read("acct_a", first.idempotency_key) == first.body + await expire_queue_lease(h, worker) + h.signing.generation = 2 + for restarted in fresh_workers(h): + for _ in range(100): + if not await restarted.deliver_one(account_id="acct_a"): + break + else: + pytest.fail("production delivery did not reach a bounded idle state") + assert { + r.state for r in await restarted.outbox.list_deliveries(account_id="acct_a") + } == {"complete"} + assert not await restarted.outbox.list_deliveries(account_id="acct_b") + repeated = [r for r in h.receiver.received if r.idempotency_key == first.idempotency_key] + assert len(repeated) == 2 and repeated[0].body == repeated[1].body + assert "key-1" in repeated[0].headers["signature-input"] + assert "key-2" in repeated[1].headers["signature-input"] + options = WebhookVerifyOptions( + jwks_resolver=StaticJwksResolver({"keys": notification_verification_keys()}), + clock=lambda: h.clock().timestamp(), + ) + assert {json.loads(r.body)["notification_type"] for r in h.receiver.received} == set(EVENTS) + for received in h.receiver.received: + value = json.loads(received.body) + validate_notification_payload(value) + verify_webhook_signature( + method="POST", + url="https://receiver.example.test" + received.target, + headers=received.headers, + body=received.body, + options=options, + ) + if value["notification_type"] == "reporting.delivery_ready": + assert received.subscriber_id in {"buyer", "second"} + assert received.account_id == "acct_a" + assert await h.queue() == original_quarantine + assert h.item.writer.writes == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +async def test_new_queue_fanout_failure_rolls_back_all_recipients( + backend, queue, tmp_path, monkeypatch +): + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[queue] + lease = await worker.outbox.claim_expansion( + account_id="acct_a", now=h.clock(), lease_seconds=60 + ) + assert lease is not None + from adcp.reporting.ledger.notification_models import decode_event + + event = decode_event(lease.event) + subscriptions = await h.subscriptions.list_active( + account_id="acct_a", notification_type=event.notification_type + ) + deliveries = tuple( + worker.cipher.prepare(event, s, lease.emission_generation) + for s in subscriptions + if s.matches(event) + ) + assert len(deliveries) >= 2 + before = await h.image() + with monkeypatch.context() as patch: + if h.pool is None: + import adcp.reporting.outbox.memory as memory + + original = memory._finish + + def fail(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected fanout finish failure") + + patch.setattr(memory, "_finish", fail) + else: + original = worker.outbox._insert_delivery + + async def fail(*args, **kwargs): + await original(*args, **kwargs) + raise RuntimeError("injected fanout insert failure") + + patch.setattr(worker.outbox, "_insert_delivery", fail) + with pytest.raises(RuntimeError, match="injected fanout"): + await worker.outbox.complete_expansion(lease, deliveries, now=h.clock()) + assert await h.image() == before + assert not await worker.outbox.list_deliveries(account_id="acct_a") + await expire_queue_lease(h, worker, expansion=True) + restarted = fresh_workers(h)[queue] + assert await restarted.expand_one(account_id="acct_a") + assert len(await restarted.outbox.list_deliveries(account_id="acct_a")) == len(deliveries) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +@pytest.mark.parametrize("mutation", ["algorithm", "legacy", "transient", "permanent", "cancel"]) +async def test_current_signing_failure_never_sends_or_exposes_private_data( + backend, queue, mutation, tmp_path, monkeypatch, caplog +): + from cryptography.hazmat.primitives.asymmetric import ec + + from adcp.reporting.outbox.routing import ( + ReportingLegacyAuthentication, + ReportingSigningMaterial, + ) + from adcp.webhook_sender import ScopePermanentlyUnknown, ScopeTransientlyUnavailable + + async with queued_production(backend, tmp_path, monkeypatch) as h: + worker = h.workers[queue] + assert await worker.expand_one(account_id="acct_a") + quarantine = await h.queue() + + async def incompatible(**kwargs): + if mutation == "transient": + raise ScopeTransientlyUnavailable() + if mutation == "permanent": + raise ScopePermanentlyUnknown() + if mutation == "cancel": + raise asyncio.CancelledError() + return ReportingSigningMaterial( + ec.derive_private_key(1, ec.SECP256R1()), + "https://seller.example.test/keys#changed", + "ecdsa-p256-sha256", + frozenset({"ecdsa-p256-sha256"}), + ) + + with monkeypatch.context() as patch: + if mutation == "legacy": + for key, subscription in tuple(h.subscriptions.values.items()): + h.subscriptions.values[key] = replace( + subscription, + signing_scope_id=None, + authentication=ReportingLegacyAuthentication("Bearer", "fixture-only"), + ) + else: + patch.setattr(h.signing, "resolve", incompatible) + if mutation == "cancel": + with pytest.raises(asyncio.CancelledError): + await worker.deliver_one(account_id="acct_a") + else: + assert await worker.deliver_one(account_id="acct_a") + assert not h.receiver.received + assert not caplog.records + deliveries = await worker.outbox.list_deliveries(account_id="acct_a") + assert all(r.state != "complete" for r in deliveries) + if mutation == "cancel": + assert sum(r.state == "leased" for r in deliveries) == 1 + elif mutation == "permanent": + assert sum(r.state == "quarantined" for r in deliveries) == 1 + else: + assert {r.state for r in deliveries} == {"pending"} + assert await h.queue() == quarantine diff --git a/tests/conformance/reporting/test_reporting_production_packaging.py b/tests/conformance/reporting/test_reporting_production_packaging.py new file mode 100644 index 000000000..b529628d7 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_packaging.py @@ -0,0 +1,56 @@ +"""Python 3.10 VCS/sdist installed B2.4, with and without optional PostgreSQL.""" + +import asyncio +import os +import shutil + +import pytest + +from ._generation_support import assert_c_collated_rolling_database +from ._production_packaging import installed_production +from .test_reporting_materializer_packaging import b1_wheels, built_distribution +from .test_reporting_notification_packaging import run_step + +__all__ = ["b1_wheels", "built_distribution"] + + +@pytest.mark.parametrize("kind", ["vcs", "sdist"]) +@pytest.mark.parametrize("drivers", [False, True], ids=["base", "pg"]) +async def test_floor_installed_production_contract(request, kind, drivers): + interpreter = os.environ.get("ADCP_PYTHON310") + if interpreter is None: + pytest.skip("ADCP_PYTHON310 supplies the installed floor runtime") + if drivers: + assert_c_collated_rolling_database() + root, wheels, _ = request.getfixturevalue("b1_wheels") + _, _, source = request.getfixturevalue("built_distribution") + label = kind + ("-pg" if drivers else "-base") + environment = root / ("production-python310-" + label) + await asyncio.to_thread( + run_step, + [interpreter, "-m", "venv", str(environment)], + label=label + "-environment", + cwd=root, + ) + python = environment / "bin/python" + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + await asyncio.to_thread( + run_step, + [*installer, str(wheels[kind]) + ("[pg]" if drivers else "")], + label=label + "-install", + cwd=root, + timeout=180, + ) + await asyncio.to_thread( + installed_production, + root, + python, + wheels[kind], + source, + label=label, + driver_absent=not drivers, + ) diff --git a/tests/conformance/reporting/test_reporting_production_progress.py b/tests/conformance/reporting/test_reporting_production_progress.py new file mode 100644 index 000000000..8fbab7ca4 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_progress.py @@ -0,0 +1,603 @@ +"""Bounded committed-period progress, with a settled first acquisition window.""" + +import asyncio +import hashlib +import json +import sqlite3 +import sys +from contextlib import asynccontextmanager +from dataclasses import replace +from datetime import timedelta + +import pytest + +from ._generation_support import END, START +from ._production_support import production_harness +from .test_reporting_production_lock_order import source_turn + + +async def sample_configurations(h, *, count=32): + selected = h.production.offerings[0] + blocked = [h.item.config] + [ + replace(h.item.config, delivery_config_id=f"busy-{i:02}") for i in range(count - 1) + ] + outside = replace(h.item.config, account_id="acct_b") + for configuration in [*blocked, outside]: + selected.producer._source.bind_generation(configuration) + binding = replace(h.item.binding, generation_key=configuration.generation_key) + h.item.writer.grant(binding) + await h.store.admit_production_configuration( + configuration, binding, offering_id=selected.offering_id + ) + for account in ("acct_a", "acct_b"): + await h.production.activate(account_id=account) + return blocked, outside + + +async def lease_source(support, worker, *, index=0): + token = support._producer_turn.set(support.offerings[index].producer) + try: + return await support.store.lease_period_close(worker_id=worker, now=END, lease_seconds=30) + finally: + support._producer_turn.reset(token) + + +def observe_samples(monkeypatch): + psycopg = pytest.importorskip("psycopg") + original = psycopg.AsyncConnection.execute + samples = [] + + async def execute(connection, query, *args, **kwargs): + result = await original(connection, query, *args, **kwargs) + if isinstance(query, str) and query.startswith( + "SELECT c.account_id,c.delivery_config_id,c.delivery_config_version," + ): + assert query.endswith("LIMIT 32") + assert 0 <= result.rowcount <= 32 + samples.append(result.rowcount) + return result + + monkeypatch.setattr(psycopg.AsyncConnection, "execute", execute) + return samples + + +async def bounded_turn(call, samples): + start = len(samples) + result = await asyncio.wait_for(call(), 5) + fetched = samples[start:] + assert 1 <= len(fetched) <= 2 and sum(fetched) <= 32 + return result + + +def account_image(image, account): + return { + table: [row for row in rows if row[0].get("account_id") == account] + for table, rows in image.items() + } + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_busy_account_window_advances_wraps_and_revisits_after_unlock( + notifications, tmp_path, monkeypatch +): + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + count=0, + source_publication=True, + second_source=True, + notifications=notifications, + ) as h: + support, store = h.production, h.store + blocked, outside = await sample_configurations(h) + other = replace(h.item.config, delivery_config_id="other-source") + unadmitted = replace(h.item.config, delivery_config_id="000-unadmitted") + await store.put_configuration(unadmitted) + support.offerings[1].producer._source.bind_generation(other) + binding = replace(h.item.binding, generation_key=other.generation_key) + h.item.writer.grant(binding) + await store.admit_production_configuration( + other, binding, offering_id=support.offerings[1].offering_id + ) + source = support.offerings[0].producer._source + samples = observe_samples(monkeypatch) + before = await h.image() + async with h.pool.connection() as holder, holder.transaction(): + await store._lock_account(holder, "acct_a") + first = await bounded_turn(lambda: source_turn(support), samples) + assert first.leased is None and samples[-1] == 32 + # Another producer has its own hint; its blocked turn cannot erase + # the first producer's progress beyond the full 32-row window. + assert ( + await bounded_turn(lambda: lease_source(support, "other", index=1), samples) is None + ) + assert await h.image() == before + second = await bounded_turn(lambda: source_turn(support), samples) + assert second.leased is not None, "locked prefix hid the eligible second account" + assert second.leased.generation_key == outside.generation_key + assert len(second.obligations_committed) == len(second.revisions_committed) == 1 + assert not second.slices_failed + assert account_image(await h.image(), "acct_a") == account_image(before, "acct_a") + assert len(source.requests) == 1 + assert source.requests[0].identity.account_id == "acct_b" + assert not support.offerings[1].producer._source.requests + # Successful acquisition resets discovery to the durable rank. + assert (await bounded_turn(lambda: source_turn(support), samples)).leased is None + held = await bounded_turn(lambda: lease_source(support, "tail-held"), samples) + assert held is not None and held.generation_key == outside.generation_key + held_image = await h.image() + assert (await bounded_turn(lambda: source_turn(support), samples)).leased is None + start = len(samples) + assert (await bounded_turn(lambda: source_turn(support), samples)).leased is None + assert samples[start:] == [0, 32] # empty tail wraps once, never an unbounded scan + assert await h.image() == held_image + try: + returned = await bounded_turn(lambda: source_turn(support), samples) + assert returned.leased.generation_key in {c.generation_key for c in blocked} + assert len(returned.obligations_committed) == len(returned.revisions_committed) == 1 + assert not returned.slices_failed + finally: + await store.release_period_close(held, worker_id="tail-held") + assert len(source.requests) == 2 + assert {r.identity.account_id for r in source.requests} == {"acct_a", "acct_b"} + assert len({r.identity.source_execution_key for r in source.requests}) == 2 + async with h.pool.connection() as c: + turns = await ( + await c.execute( + "SELECT account_id,delivery_config_id" + " FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id" + ) + ).fetchall() + assert turns == [ + ("acct_a", returned.leased.delivery_config_id), + ("acct_b", outside.delivery_config_id), + ] + assert await ( + await c.execute("SELECT count(*) FROM reporting_production_source_probe_turns") + ).fetchone() == (0,) + assert await ( + await c.execute( + "SELECT count(*) FROM reporting_configurations" + " WHERE lease_worker_id IS NOT NULL" + ) + ).fetchone() == (0,) + assert not (await h.queue())[0] + print( + json.dumps( + { + "busy_account_progress": { + "notifications": notifications, + "bound": 32, + "blocked_configurations": 32, + "publications": 2, + "separate_producer_hints": True, + "wrap_and_unlock": True, + "blocked_state_unchanged": True, + } + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_fresh_stores_and_concurrent_workers_preserve_bounded_sampling_and_acquisition( + notifications, tmp_path, monkeypatch +): + psycopg = pytest.importorskip("psycopg") + path = tmp_path / "destination.sqlite" + async with production_harness( + "postgres", + path, + count=0, + source_publication=True, + notifications=notifications, + ) as h: + # Two fresh services each perform a real startup turn. Even after + # those acquisitions, more than a full window belongs to the busy A. + blocked, outside = await sample_configurations(h, count=65) + async with h.pool.connection() as holder, holder.transaction(): + await h.store._lock_account(holder, "acct_a") + assert (await source_turn(h.production)).leased is None + await h.production.aclose() + bindings = [ + (c, h.production.offerings[0].offering_id, "catalog-7391") for c in [*blocked, outside] + ] + async with ( + production_harness( + "postgres", + path, + count=0, + source_publication=True, + notifications=notifications, + existing_pool=h.pool, + source_bindings=bindings, + ) as fresh, + production_harness( + "postgres", + path, + count=0, + source_publication=True, + notifications=notifications, + existing_pool=h.pool, + source_bindings=bindings, + ) as peer, + ): + assert fresh.store is not h.store and peer.store is not fresh.store + sources = [v.production.offerings[0].producer._source for v in (fresh, peer)] + assert [len(s.requests) for s in sources] == [1, 1] + before = await h.image() + samples = observe_samples(monkeypatch) + original = psycopg.AsyncConnection.execute + holding, release = asyncio.Event(), asyncio.Event() + winner = None + + async def coordinate(connection, query, *args, **kwargs): + result = await original(connection, query, *args, **kwargs) + if ( + asyncio.current_task() is winner + and isinstance(query, str) + and query.startswith("UPDATE reporting_configurations SET lease_worker_id") + ): + # The real statement, including its inherited trigger, + # has executed under the account lock, but not committed. + holding.set() + await asyncio.wait_for(release.wait(), 5) + return result + + monkeypatch.setattr(psycopg.AsyncConnection, "execute", coordinate) + async with h.pool.connection() as holder, holder.transaction(): + await h.store._lock_account(holder, "acct_a") + for current in (fresh, peer): + turn = await bounded_turn(lambda: source_turn(current.production), samples) + assert turn.leased is None and samples[-1] == 32 + assert await h.image() == before + winner = asyncio.create_task(source_turn(fresh.production)) + try: + await asyncio.wait_for(holding.wait(), 5) + losing = await bounded_turn(lambda: source_turn(peer.production), samples) + assert losing.leased is None and not losing.revisions_committed + # Observe committed rows without requesting the account + # lock deliberately held by the winning transaction. + async with h.pool.connection() as observer: + assert await ( + await observer.execute( + "SELECT count(*) FROM reporting_revisions WHERE account_id=%s", + ("acct_b",), + ) + ).fetchone() == (0,) + release.set() + won = await asyncio.wait_for(winner, 5) + finally: + release.set() + if not winner.done(): + winner.cancel() + await asyncio.gather(winner, return_exceptions=True) + assert won.leased.generation_key == outside.generation_key + assert len(won.obligations_committed) == len(won.revisions_committed) == 1 + assert not won.slices_failed + assert account_image(await h.image(), "acct_a") == account_image(before, "acct_a") + # The losing worker resumes; it cannot duplicate B's committed + # source execution or fabricate an acquisition for the held A. + repeat = await source_turn(peer.production) + assert not repeat.slices_failed + snapshot = await h.store.read_status_snapshot(account_id="acct_b") + assert len(snapshot.obligations) == len(snapshot.revisions) == 1 + requests = [r for s in sources for r in s.requests if r.identity.account_id == "acct_b"] + assert len(requests) == 1 + print( + json.dumps( + { + "busy_account_fresh_concurrent": { + "notifications": notifications, + "bound": 32, + "blocked_configurations": 65, + "fresh_store_instances": 2, + "coordinated_actual_trigger": True, + "second_account_publications": 1, + } + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_rejected_source_window_does_not_starve_later_admitted_generation( + backend, tmp_path, monkeypatch +): + path = tmp_path / "destination.sqlite" + async with production_harness(backend, path, count=0, source_publication=True) as h: + support, store, item = h.production, h.store, h.item + source = support.offerings[0].producer._source + blocked = [replace(item.config, delivery_config_id=f"blocked-{i:02}") for i in range(32)] + for configuration in blocked: + source.bind_generation(configuration) + binding = replace(item.binding, generation_key=configuration.generation_key) + item.writer.grant(binding) + await store.admit_production_configuration( + configuration, binding, offering_id=support.offerings[0].offering_id + ) + outside = replace(item.config, account_id="acct_b") + await store.put_configuration(outside) + await support.activate(account_id=item.config.account_id) + for configuration in blocked: + del source.bindings[configuration.generation_key] + before = await store.list_configurations(account_id=item.config.account_id) + if h.pool is not None: + import psycopg + + before_image = await h.image() + original_execute = psycopg.AsyncConnection.execute + + async def fail_after_probe(connection, query, *args, **kwargs): + result = await original_execute(connection, query, *args, **kwargs) + if isinstance(query, str) and query.startswith( + "INSERT INTO reporting_production_source_probe_turns" + ): + raise RuntimeError("injected probe commit failure") + return result + + with monkeypatch.context() as patch: + patch.setattr(psycopg.AsyncConnection, "execute", fail_after_probe) + with pytest.raises(RuntimeError, match="injected probe commit failure"): + await source_turn(support) + assert await h.image() == before_image + first = await source_turn(support) + assert not first.slices_failed + if h.pool is not None: + async with h.pool.connection() as c: + probes = await ( + await c.execute( + "SELECT account_id,delivery_config_id" + " FROM reporting_production_source_probe_turns" + " ORDER BY account_id,delivery_config_id" + ) + ).fetchall() + assert probes == [(item.config.account_id, v.delivery_config_id) for v in blocked] + assert await ( + await c.execute("SELECT count(*) FROM adcp_reporting_configuration_lease_turns") + ).fetchone() == (0,) + await support.aclose() + # A fresh store/service in PG uses persisted rejection progress. Memory + # keeps its state across a new service, without claiming process durability. + async with production_harness( + backend, + path, + count=0, + source_publication=True, + existing_store=store if backend == "memory" else None, + existing_pool=h.pool, + ) as fresh: + second = await source_turn(fresh.production) + assert not second.slices_failed + # start() itself executes a bounded producer turn. Count that + # actual acquisition as well as the explicitly requested turn. + fresh_source = fresh.production.offerings[0].producer._source + assert len(first.revisions_committed) + len(fresh_source.requests) == 1 + for completed in (first, second): + if completed.leased is not None: + assert completed.leased.generation_key == item.config.generation_key + repeat = await source_turn(fresh.production) + assert not repeat.obligations_committed and not repeat.revisions_committed + snapshot = await fresh.store.read_status_snapshot(account_id=item.config.account_id) + assert len(snapshot.obligations) == len(snapshot.revisions) == 1 + assert snapshot.obligations[0].generation_key == item.config.generation_key + assert not (await fresh.store.read_status_snapshot(account_id="acct_b")).obligations + assert ( + await fresh.store.list_configurations(account_id=item.config.account_id) == before + ) + if h.pool is not None: + after_image = await fresh.image() + for table in ( + "reporting_production_generations", + "reporting_production_destination_bindings", + "reporting_materializer_work", + "reporting_materializer_notification_events", + ): + assert after_image[table] == before_image[table] + print( + json.dumps( + { + "rejected_window": 32, + "backend": backend, + "publications": 1, + "fresh_store": backend == "postgres", + } + ), + flush=True, + ) + + +def turn_document(turn, source): + assert not turn.slices_failed + return { + "obligations": turn.obligations_committed, + "revisions": turn.revisions_committed, + "executions": [r.identity.source_execution_key for r in source.requests], + } + + +@asynccontextmanager +async def restarted_process(h, path, *, pause): + from .test_reporting_materializer_process import Child + + class ProgressChild(Child): + async def event(self, point): + line = await asyncio.wait_for(self.process.stdout.readline(), 120) + assert line, f"producer exited before {point}; retained diagnostic: {log}" + result = json.loads(line) + assert result["point"] == point, result + return result + + log = path.with_name( + "producer-restart-paused.log" if pause else "producer-restart-finished.log" + ) + with log.open("wb") as diagnostic: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "tests.conformance.reporting._production_progress_process", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=diagnostic, + ) + child = ProgressChild(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "path": str(path), + "pause": pause, + } + ) + yield child + finally: + await child.kill() + print( + json.dumps( + { + "producer_restart_diagnostic": str(log), + "bytes": log.stat().st_size, + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_bounded_producer_advances_past_processed_first_window(backend, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", count=0, periods=131 + ) as h: + support, item = h.production, h.item + producer = support.offerings[0].producer + source = producer._source + assert producer._max_periods_per_turn == 64 + h.source_clock.advance(timedelta(hours=131)) + # Same caller-selected ID in another account and a different retained + # generation must not acquire a position just because this source runs. + untouched = ( + replace(item.config, account_id="acct_b"), + replace(item.config, delivery_config_version=2), + ) + for config in untouched: + await h.store.put_configuration(config) + await support.activate(account_id=item.config.account_id) + print( + json.dumps({"progress_backend": backend, "phase": "activated", "bound": 64}), flush=True + ) + first = await source_turn(support) + assert not first.slices_failed + # The first period was already published before activation. The + # remaining first window is acquired once, using the real sealed source. + assert len(first.obligations_committed) == 63 + assert len(first.revisions_committed) == 63 + assert len(source.requests) == 63 + first_document = turn_document(first, source) + print( + json.dumps({"progress_backend": backend, "phase": "first_window", "count": 64}), + flush=True, + ) + await support.aclose() + path = tmp_path / "destination.sqlite" + if backend == "memory": + # This is a new service/source/projector over the reference store's + # retained state, not a claim of memory durability across processes. + async with production_harness( + backend, path, count=0, periods=131, existing_store=h.store + ) as fresh: + fresh.source_clock.advance(timedelta(hours=131, seconds=61)) + new_source = fresh.production.offerings[0].producer._source + second = turn_document(await source_turn(fresh.production), new_source) + previous_requests = len(new_source.requests) + third = turn_document(await source_turn(fresh.production), new_source) + third["executions"] = third["executions"][previous_requests:] + repeat = await source_turn(fresh.production) + assert not repeat.obligations_committed and not repeat.revisions_committed + assert len(new_source.requests) == 67 + else: + async with restarted_process(h, path, pause=True) as child: + second = await child.event("committed_before_release") + async with h.pool.connection() as c: + row = await ( + await c.execute( + "SELECT c.lease_worker_id,p.closed_through" + " FROM reporting_configurations c" + " JOIN reporting_production_source_progress p" + " USING(account_id,delivery_config_id,delivery_config_version)" + " WHERE c.account_id=%s AND c.delivery_config_id=%s" + " AND c.delivery_config_version=%s", + ( + item.config.account_id, + item.config.delivery_config_id, + item.config.delivery_config_version, + ), + ) + ).fetchone() + assert row[0] is not None and row[1] == START + timedelta(hours=128) + await child.kill() + assert child.process.returncode == -9 + async with restarted_process(h, path, pause=False) as child: + third = await child.event("done") + assert await asyncio.wait_for(child.process.wait(), 10) == 0 + assert len(second["obligations"]) == len(second["revisions"]) == 64 + assert len(third["obligations"]) == len(third["revisions"]) == 3 + executions = first_document["executions"] + second["executions"] + third["executions"] + assert len(executions) == len(set(executions)) == 130 + snapshot = await h.store.read_status_snapshot(account_id=item.config.account_id) + assert len(snapshot.obligations) == len(snapshot.revisions) == 131 + assert max(o.period.end for o in snapshot.obligations) == START + timedelta(hours=131) + assert {o.generation_key for o in snapshot.obligations} == {item.config.generation_key} + assert not (await h.store.read_status_snapshot(account_id="acct_b")).obligations + assert ( + await h.store.get_revision( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + ) + == item.revision + ) + with sqlite3.connect(tmp_path / "source.seals") as connection: + assert connection.execute("SELECT count(*) FROM seals").fetchone() == (130,) + if h.pool is None: + assert h.store._production_closed == { + item.config.generation_key: START + timedelta(hours=131) + } + assert {w.state for w in h.store._production_source_work.values()} == {"settled"} + assert not {c.generation_key for c in untouched} & set(h.store._production_closed) + else: + async with h.pool.connection() as c: + assert await ( + await c.execute( + "SELECT account_id,delivery_config_id,delivery_config_version," + " closed_through" + " FROM reporting_production_source_progress" + ) + ).fetchall() == [ + ( + item.config.account_id, + item.config.delivery_config_id, + 1, + START + timedelta(hours=131), + ) + ] + assert await ( + await c.execute( + "SELECT state,count(*) FROM reporting_production_source_work GROUP BY state" + ) + ).fetchall() == [("settled", 131)] + print( + json.dumps( + { + "progress_backend": backend, + "periods": 131, + "bound": 64, + "unique_acquisitions": 130, + "restart": "SIGKILL" if h.pool else "new-service", + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_production_readiness.py b/tests/conformance/reporting/test_reporting_production_readiness.py new file mode 100644 index 000000000..b0419c1bd --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_readiness.py @@ -0,0 +1,381 @@ +"""Actual route identity, positive schema proof and optional delivery lifecycle.""" + +import asyncio + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.server.a2a_server import create_a2a_server + +from ._production_support import production_harness +from ._production_transport import MountedProduction + + +@pytest.mark.parametrize("transport", ["mcp", "a2a"]) +@pytest.mark.parametrize("mutation", ["remove", "replace"]) +async def test_warm_proof_rechecks_the_actual_mount(transport, mutation, tmp_path): + async with production_harness("postgres", tmp_path / "destination.sqlite") as h: + support = h.production + app = create_a2a_server(support.handler) if transport == "a2a" else h.mount + proof = next(p for p in support._mounts if p.mount() is app) + dispatcher = proof.dispatcher() + mapping = ( + dispatcher._tool_manager._tools if transport == "mcp" else dispatcher._tool_callers + ) + original = mapping.pop("get_reporting_status") + if mutation == "replace": + mapping["get_reporting_status"] = mapping["get_adcp_capabilities"] + try: + assert await support.reporting_delivery() == {} + with pytest.raises(ReportingNotificationError, match="component_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + mapping["get_reporting_status"] = original + assert (await support.reporting_delivery())["managed_delivery"] is True + + +async def test_shared_scan_cancellation_and_post_scan_dynamic_check(tmp_path, monkeypatch): + import adcp.reporting.production.schema as schema + + async with production_harness("postgres", tmp_path / "destination.sqlite") as h: + support = h.production + original = schema.validate_production_schema + entered, release = asyncio.Event(), asyncio.Event() + scans = 0 + + async def paused(*args, **kwargs): + nonlocal scans + scans += 1 + entered.set() + await asyncio.wait_for(release.wait(), 5) + await original(*args, **kwargs) + + support.invalidate_schema_validation() + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", paused) + canceled = asyncio.create_task(support.reporting_delivery()) + await asyncio.wait_for(entered.wait(), 5) + callers = [asyncio.create_task(support.reporting_delivery()) for _ in range(4)] + canceled.cancel() + with pytest.raises(asyncio.CancelledError): + await canceled + removed = h.mount._tool_manager._tools.pop("get_reporting_status") + try: + release.set() + assert await asyncio.wait_for(asyncio.gather(*callers), 10) == [{}] * 4 + finally: + h.mount._tool_manager._tools["get_reporting_status"] = removed + assert scans == 1 + assert (await support.reporting_delivery())["managed_delivery"] is True + await h.store.materializer_ready() + assert scans == 1 + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("mutation", ["closed", "replaced"]) +@pytest.mark.parametrize("reconciled", [False, True]) +async def test_warm_mounted_capability_rechecks_pool_lifecycle_and_identity( + notifications, mutation, reconciled, tmp_path, monkeypatch +): + from contextlib import AsyncExitStack + + import adcp.reporting.production.schema as schema + + pool_type = pytest.importorskip("psycopg_pool").AsyncConnectionPool + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + notifications=notifications, + notification_delivery=notifications, + reconciled=reconciled, + ) as h: + support = h.production + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client, AsyncExitStack() as stack: + before = {} + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, before[transport] = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert before[transport]["media_buy"]["reporting_delivery"]["managed_delivery"] + if reconciled: + assert before[transport]["media_buy"]["reporting_delivery"][ + "reconciled_billing" + ] + replacement = None + if mutation == "replaced": + replacement = await stack.enter_async_context( + pool_type( + h.pool.conninfo, kwargs=h.pool.kwargs, min_size=1, max_size=2, open=False + ) + ) + await replacement.wait(timeout=5) + assert replacement.closed is False + else: + await h.pool.close() + assert h.pool.closed is True + assert not support._task.done() + scans = 0 + + async def no_scan(*args, **kwargs): + nonlocal scans + scans += 1 + raise AssertionError("component refusal must precede a catalog scan") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + if replacement is not None: + patch.setattr(h.store, "_pool", replacement) + for transport in before: + _, refused = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "reporting_delivery" not in refused.get("media_buy", {}) + assert "webhook_signing" not in refused + expected_code = ( + "notification_chain_unready" + if replacement is not None and notifications + else "reporting_production_component_unready" + ) + with pytest.raises(ReportingNotificationError, match=expected_code): + await support.activate(account_id="acct_a") + assert scans == 0 + if mutation == "replaced": + for transport in before: + _, restored = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert restored == before[transport] + + +@pytest.mark.parametrize("queue", ["projection", "core", "ready"]) +@pytest.mark.parametrize("replacement_open", [False, True]) +async def test_warm_mounted_capability_rechecks_each_queue_pool( + queue, replacement_open, tmp_path, monkeypatch +): + from contextlib import AsyncExitStack + + import adcp.reporting.production.schema as schema + + pool_type = pytest.importorskip("psycopg_pool").AsyncConnectionPool + # The projection queue also participates without optional HTTP workers. + delivery = queue != "projection" + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + notifications=True, + notification_delivery=delivery, + reconciled=True, + ) as h: + mount = MountedProduction(h) + mount.authorize(h.item) + outbox = ( + h.projection.outbox + if queue == "projection" + else h.production.notification_workers[0 if queue == "core" else 2].outbox + ) + replacement = pool_type( + h.pool.conninfo, kwargs=h.pool.kwargs, min_size=1, max_size=1, open=False + ) + async with mount.client() as client, AsyncExitStack() as stack: + if replacement_open: + await stack.enter_async_context(replacement) + await replacement.wait(timeout=5) + assert replacement.closed is not replacement_open + baseline = {} + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, baseline[transport] = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert baseline[transport]["media_buy"]["reporting_delivery"]["reconciled_billing"] + scans = 0 + + async def no_scan(*args, **kwargs): + nonlocal scans + scans += 1 + raise AssertionError("queue wiring checks must precede catalog proof") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + patch.setattr(outbox, "_pool", replacement) + for transport in baseline: + _, refused = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "reporting_delivery" not in refused.get("media_buy", {}) + assert "webhook_signing" not in refused + assert scans == 0 + for transport in baseline: + _, restored = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert restored == baseline[transport] + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_warm_mounted_discovery_does_not_checkout_a_saturated_valid_pool( + notifications, tmp_path, monkeypatch +): + import adcp.reporting.production.schema as schema + + async with production_harness( + "postgres", + tmp_path / "destination.sqlite", + notifications=notifications, + notification_delivery=notifications, + reconciled=True, + ) as h: + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client: + expected = {} + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, expected[transport] = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert expected[transport]["media_buy"]["reporting_delivery"]["reconciled_billing"] + connections = [] + try: + for _ in range(h.pool.max_size): + connections.append(await h.pool.getconn(timeout=5)) + assert h.pool.get_stats()["pool_available"] == 0 + assert h.pool.closed is False + scans = checkouts = 0 + + async def no_scan(*args, **kwargs): + nonlocal scans + scans += 1 + raise AssertionError("warm discovery must reuse the catalog proof") + + def no_connection(*args, **kwargs): + nonlocal checkouts + checkouts += 1 + raise AssertionError("warm discovery must not request a pool connection") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + patch.setattr(h.pool, "connection", no_connection) + for transport in expected: + _, result = await asyncio.wait_for( + mounted.call(client, "get_adcp_capabilities", {}, transport=transport), + 5, + ) + assert result == expected[transport] + assert scans == checkouts == 0 + finally: + for connection in connections: + await h.pool.putconn(connection) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_optional_delivery_is_owned_and_broken_enabled_chain_fails_closed(backend, tmp_path): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + support = h.production + assert support._notification_task is not None + assert not support._notification_task.done() + if backend == "postgres": + fields = await support.reporting_delivery() + assert fields["managed_delivery"] is True + assert fields["ledger_notification"] == "reporting.ledger_changed" + assert fields["readiness_notification"] == "reporting.delivery_ready" + assert fields["status_notification"] == "reporting.status_changed" + worker = support.notification_workers[-1] + outbox = worker.outbox + worker.outbox = support.notification_workers[0].outbox + try: + assert await support.reporting_delivery() == {} + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + worker.outbox = outbox + await support.activate(account_id=h.item.config.account_id) + assert h.subscriptions.lists + assert {event for _, event in h.subscriptions.lists} == { + "reporting.ledger_changed", + "reporting.status_changed", + "reporting.delivery_ready", + } + await support.aclose() + assert support._notification_task is None + assert await support.reporting_delivery() == {} + + +async def test_warm_catalog_proof_does_not_cache_signing_wiring(tmp_path, monkeypatch): + import adcp.reporting.production.schema as schema + + async with production_harness( + "postgres", tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + support = h.production + assert (await support.reporting_delivery())["managed_delivery"] + mount = MountedProduction(h) + mount.authorize(h.item) + + async def no_scan(*args, **kwargs): + pytest.fail("warm immutable catalog proof was unnecessarily repeated") + + with monkeypatch.context() as patch: + patch.setattr(schema, "validate_production_schema", no_scan) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, valid = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert valid["webhook_signing"]["algorithms"] == ["ed25519"] + # These are normative 3.2 obligations even though the + # additive schema fields remain optional for old agents. + assert valid["webhook_signing"]["delivery_retry_horizon_seconds"] == 86400 + assert valid["identity"]["brand_json_url"] == ( + "https://seller.example.test/brand.json" + ) + # Each mutation follows a successful schema proof. Every + # request must inspect the current concrete participant. + for target, name, replacement in ( + (support.notification_workers[0], "signing", h.signing), + (h.signing, "resolve", None), + (h.subscriptions, "get_active", None), + ): + with monkeypatch.context() as mutation: + mutation.setattr(target, name, replacement) + _, invalid = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "webhook_signing" not in invalid + assert "reporting_delivery" not in invalid.get("media_buy", {}) + with pytest.raises(ReportingNotificationError): + await support.activate(account_id="acct_a") + _, restored = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert restored == valid + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_current_key_contract_is_rechecked_before_account_activation( + backend, tmp_path, monkeypatch, caplog +): + from dataclasses import replace + + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + from ._reliable_support import notification_subscription + + h.subscriptions.put(notification_subscription(principal=h.item.binding.consumer_id)) + original = h.signing.resolve + + async def incompatible(**kwargs): + material = await original(**kwargs) + return replace(material, advertised_algorithms={"ed25519", "ecdsa-p256-sha256"}) + + before = await h.image() + with monkeypatch.context() as patch: + patch.setattr(h.signing, "resolve", incompatible) + with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): + await h.production.activate(account_id="acct_a") + assert await h.image() == before + assert not caplog.records + assert await h.production.activate(account_id="acct_a") diff --git a/tests/conformance/reporting/test_reporting_production_restart.py b/tests/conformance/reporting/test_reporting_production_restart.py new file mode 100644 index 000000000..ec30f2ebb --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_restart.py @@ -0,0 +1,108 @@ +"""Real database/process restart of first-attempt windows in all three queues.""" + +import asyncio +import hashlib +import json +import sys +from contextlib import asynccontextmanager +from datetime import datetime, timedelta +from pathlib import Path + +import pytest + +import adcp.reporting.production.delivery_window as window_module + +from .test_reporting_materializer_process import Child +from .test_reporting_production_notifications import ( + pin_current_clock, + queued_production, + retained_windows, +) + + +@asynccontextmanager +async def delivery_child(h, path, *, queue, pause, at): + origin = Path(window_module.__file__).resolve() + log = path / ("accepted-crash.log" if pause else "cold-retry.log") + with log.open("wb") as diagnostic: + process = await asyncio.create_subprocess_exec( + sys.executable, + "-m", + "tests.conformance.reporting._production_delivery_process", + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=diagnostic, + ) + child = Child(process) + try: + await child.send( + { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "queue": queue, + "pause": pause, + "at": at.isoformat(), + "consumer": h.item.binding.consumer_id, + "installed": "site-packages" in str(origin), + "module_sha256": hashlib.sha256(origin.read_bytes()).hexdigest(), + } + ) + yield child + finally: + await child.kill() + print( + json.dumps( + { + "production_delivery_process_log": str(log), + "bytes": log.stat().st_size, + "sha256": hashlib.sha256(log.read_bytes()).hexdigest(), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("queue", [0, 1, 2], ids=["core", "status-v2", "ready-v2"]) +@pytest.mark.parametrize("offset", [-1, 0], ids=["before", "exact"]) +async def test_pg_sigkill_after_accepted_http_keeps_first_retry_window( + queue, offset, tmp_path, monkeypatch +): + async with queued_production("postgres", tmp_path, monkeypatch) as h: + await pin_current_clock(h) + # This process receives exactly this registration. Multi-recipient + # expansion/rollback is exercised separately; retaining registrations + # unknown to this child would suppress a different first delivery. + for key in tuple(h.subscriptions.values): + if key[0] == "acct_a" and key[1] != "buyer": + del h.subscriptions.values[key] + assert await h.workers[queue].expand_one(account_id="acct_a") + async with delivery_child(h, tmp_path, queue=queue, pause=True, at=h.clock()) as child: + first = await child.event("accepted_before_ack") + assert len(await retained_windows(h)) == 1 + await child.kill() + assert child.process.returncode == -9 + saved = await retained_windows(h) + at = datetime.fromisoformat(first["expires_at"]) + timedelta(microseconds=offset) + async with delivery_child(h, tmp_path, queue=queue, pause=False, at=at) as child: + restored = await child.event("done") + assert await asyncio.wait_for(child.process.wait(), 10) == 0 + assert restored["http_calls"] == int(offset < 0) + assert restored["activity_count"] == (2 if offset < 0 else 1) + assert (restored["state"], restored["error_code"]) == ( + ("complete", None) if offset < 0 else ("suppressed", "lease_expired") + ) + for key in ("key_sha256", "body_sha256", "started_at", "expires_at", "origin"): + assert restored[key] == first[key] + assert await retained_windows(h) == saved + print( + json.dumps( + { + "production_delivery_cold_restart": { + "queue": queue, + "offset_microseconds": offset, + **restored, + } + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_production_rolling.py b/tests/conformance/reporting/test_reporting_production_rolling.py new file mode 100644 index 000000000..afc99fbb9 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_rolling.py @@ -0,0 +1,409 @@ +"""Actual approved B2.3 and hardening binaries across installed B2.4 activation.""" + +import asyncio +import hashlib +import json +import os +import shutil +import subprocess +import zipfile +from contextlib import asynccontextmanager +from dataclasses import asdict, replace +from datetime import timedelta +from pathlib import Path + +import pytest + +from adcp.reporting.ledger import revision_content_sha256 +from adcp.reporting.ledger.models import derive_period + +from ._feed_support import feed_harness, feed_request, mixed_case, walk +from ._production_packaging import copied_fixtures, inspect_distribution, source_basis +from ._production_support import production_harness +from .test_reporting_feed_hardening_installed import approved_b23 +from .test_reporting_feed_installed_pg import ( + b1_wheels, + built_distribution, + feed_wheels, + installed_feed, +) +from .test_reporting_feed_process import feed_process +from .test_reporting_materializer_process import Child +from .test_reporting_materializer_rolling import build_frozen +from .test_reporting_notification_packaging import ROOT, run_step + +__all__ = ["approved_b23", "b1_wheels", "built_distribution", "feed_wheels", "installed_feed"] +HARDENING = "a09878f67ab397a4b51b3314e3e8a5e87cf96da5" + + +@pytest.fixture(scope="module", params=["b23", "hardening"]) +def production_parent(request, tmp_path_factory): + if request.param == "b23": + return request.getfixturevalue("approved_b23") + root, _, _, identity = build_frozen("hardening-b24", tmp_path_factory, request, sha=HARDENING) + interpreter = os.environ.get("ADCP_PYTHON310") + if interpreter is None: + pytest.skip("ADCP_PYTHON310 supplies the installed floor artifact") + environment = root / "python310" + run_step( + [interpreter, "-m", "venv", str(environment)], label="hardening-floor-environment", cwd=root + ) + python = environment / "bin/python" + wheel = next((root / "dist").glob("*.whl")) + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, f"{wheel}[pg]", "asgi-lifespan==2.1.0"], + label="hardening-floor-install", + cwd=root, + timeout=180, + ) + from .test_reporting_feed_packaging import feed_modules + + with zipfile.ZipFile(wheel) as archive: + modules = {} + for name in {*feed_modules(), "adcp.reporting.outbox.status_pg"}: + member = name.replace(".", "/") + ".py" + if member not in archive.namelist(): + member = name.replace(".", "/") + "/__init__.py" + raw = archive.read(member) + assert raw == subprocess.check_output( + ["git", "show", f"{HARDENING}:src/{member}"], cwd=ROOT + ) + modules[name] = hashlib.sha256(raw).hexdigest() + script, helper = root / "feed_process.py", root / "receipt_transport.py" + shutil.copy2(Path(__file__).with_name("_feed_process.py"), script) + shutil.copy2(Path(__file__).with_name("_receipt_transport.py"), helper) + return ( + root, + python, + script, + helper, + { + **identity, + "modules": modules, + "python": [3, 10], + "tree": "528cf5fddb61c72a284dc0a9b22b169752db7691", + }, + wheel, + ) + + +@pytest.fixture(scope="module") +def production_install(installed_feed, feed_wheels, built_distribution, production_parent): + root, python, _, _, current = installed_feed + parent_label = production_parent[4]["sha"][:12] + label = parent_label + "-" + current["distribution"] + _, wheels, _ = feed_wheels + _, _, source = built_distribution + modules, assets = inspect_distribution(wheels[current["distribution"]], source) + fixture_root = copied_fixtures(root, label + "-restart") + script = fixture_root / "production_process.py" + shutil.copy2(Path(__file__).with_name("_production_installed_process.py"), script) + installer = ( + [shutil.which("uv"), "pip", "install", "--python", str(python)] + if shutil.which("uv") + else [str(python), "-m", "pip", "install"] + ) + run_step( + [*installer, "pytest==9.1.1", "pytest-asyncio==1.4.0", "respx==0.23.1"], + label="production-process-fixtures", + cwd=root, + timeout=180, + ) + return ( + python, + script, + { + **current, + "fixtures": str(fixture_root), + "modules": modules, + "assets": assets, + "source_basis": source_basis( + wheels[current["distribution"]], + source, + evidence=Path(os.environ.get("ADCP_PRODUCTION_EVIDENCE", root / "evidence")), + label=label + "-rolling", + ), + }, + ) + + +@asynccontextmanager +async def installed_child(python, script, settings, path): + log = path / settings.get( + "diagnostic_name", "activation.log" if settings["pause"] else "continuation.log" + ) + try: + with log.open("xb") as diagnostic: + process = await asyncio.create_subprocess_exec( + str(python), + "-I", + str(script), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=diagnostic, + ) + + class ActivationChild(Child): + async def event(self, point): + line = await asyncio.wait_for(self.process.stdout.readline(), 120) + assert ( + line + ), f"installed process exited before {point}; diagnostic retained at {log}" + result = json.loads(line) + assert result["point"] == point + return result + + child = ActivationChild(process) + try: + await child.send(settings) + yield child + finally: + await child.kill() + finally: + raw = log.read_bytes() + retained = None + if os.environ.get("ADCP_PRODUCTION_EVIDENCE"): + evidence = Path(os.environ["ADCP_PRODUCTION_EVIDENCE"]) + evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + retained = evidence / (settings["evidence_key"] + "-" + log.name) + with retained.open("xb") as stream: + stream.write(raw) + print( + json.dumps( + { + "installed_activation_process_log": str(log), + "retained_log": str(retained) if retained else None, + "bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + ), + flush=True, + ) + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_actual_parent_page_one_to_installed_activation_sigkill_and_complete_walk( + production_parent, production_install, notifications, tmp_path +): + old_root, old_python, old_script, old_helper, old, old_wheel = production_parent + python, script, current = production_install + evidence_key = f"{old['sha'][:12]}-{current['distribution']}-{int(notifications)}" + async with feed_harness("postgres", notifications=notifications) as h: + case, receipt_request, receipt_response = await mixed_case(h) + # Seed public ordinary records on the parent schema. The memory fixture + # supplies only deterministic input values and a provider grant; the + # installed parent, below, creates the actual attempt and durable work. + async with production_harness( + "memory", + tmp_path / "destination.sqlite", + notifications=notifications, + count=0, + reconciled=True, + identity_prefix="b24-", + ) as seed: + item = seed.item + await h.store.put_configuration(item.config) + await h.store.commit_obligation(item.obligation) + await h.store.put_destination_binding(item.binding) + await h.store.commit_revision(item.revision, item.rows) + key = asdict(item.verifier.key) + legacy_script = old_root / "production_legacy_process.py" + shutil.copy2(Path(__file__).with_name("_production_legacy_process.py"), legacy_script) + legacy_module = subprocess.check_output( + ["git", "show", f"{old['sha']}:src/adcp/reporting/outbox/status_pg.py"], cwd=ROOT + ) + materializer_module = subprocess.check_output( + ["git", "show", f"{old['sha']}:src/adcp/reporting/materializer/pg.py"], cwd=ROOT + ) + legacy_settings = { + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "notifications": notifications, + "module_sha256": hashlib.sha256(legacy_module).hexdigest(), + "materializer_module_sha256": hashlib.sha256(materializer_module).hexdigest(), + "verification_key": key, + "evidence_key": evidence_key, + } + async with installed_child( + old_python, + legacy_script, + { + **legacy_settings, + "pending": True, + "pause": True, + "diagnostic_name": "historical-pending.log", + "revision_id": item.revision.reporting_revision_id, + }, + tmp_path, + ) as child: + pending = await child.event("pending") + await child.kill() + assert child.process.returncode == -9 + async with h.pool.connection() as connection: + assert ( + await ( + await connection.execute("SELECT to_regclass('reporting_production_accounts')") + ).fetchone() + )[0] is None + pending_before_migration = (await h.image())["reporting_materializer_work"] + options = { + "python": old_python, + "script": old_script, + "helper": old_helper, + "installed": old, + } + async with feed_process(h, case, feed_request(case), pause="committed", **options) as child: + first = (await child.event("committed"))["result"] + await child.kill() + assert child.process.returncode == -9 + snapshot = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=case.binding.principal + ) + expected = await walk(h.store, feed_request(case), case.binding.principal, first=first) + # Change actual live evidence while the original binary's frozen pages + # stay open. No reconstruction of its original representation is used. + await h.store.set_revision_readable( + account_id="acct_a", + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + settings = { + **current, + "conninfo": h.pool.conninfo, + "kwargs": h.pool.kwargs, + "destination": str(tmp_path / "destination.sqlite"), + "notifications": notifications, + "caller": {"account_id": "acct_a", "consumer_id": case.binding.consumer_id}, + "receipt_request": receipt_request, + "receipt_response": receipt_response, + "historical_pending": pending, + "evidence_key": evidence_key, + "pause": True, + } + async with installed_child(python, script, settings, tmp_path) as child: + activated = await child.event("activated") + await child.kill() + assert child.process.returncode == -9 + assert activated["pending_continuation"]["state"] == "verified" + assert activated["pending_continuation"]["external_id"] == pending["external_id"] + assert ( + await h.store.read_reporting_feed_snapshot( + snapshot.snapshot_id, caller=case.binding.principal + ) + == snapshot + ) + # Run the actual inherited binary's projector primitive and sweeper on + # this newly activated schema, with no child SDK imported into it. + # An actually eligible revision for the next period makes this an old + # reservation exclusion test, rather than an idle-worker observation. + selected = await h.store.get_revision( + account_id="acct_a", reporting_revision_id="b24-production-revision" + ) + assert selected is not None and selected.row_count == 0 + original_obligation = await h.store.get_obligation( + account_id="acct_a", reporting_obligation_id=selected.reporting_obligation_id + ) + period = derive_period( + item.config.schedule, account_timezone=item.config.account_timezone, ordinal=1 + ) + new_obligation = replace( + original_obligation, + reporting_obligation_id="b24-fence-next-period", + period=period, + scope_resolved_at=period.end, + automated_recovery_deadline_at=period.expected_at + + item.config.automated_recovery_window, + ) + await h.store.commit_obligation(new_obligation) + revision_id = "b24-fence-next-revision" + await h.store.commit_revision( + replace( + selected, + reporting_revision_id=revision_id, + reporting_obligation_id=new_obligation.reporting_obligation_id, + data_through=period.end, + observed_at=period.end, + finalized_at=period.end, + created_at=period.end + timedelta(seconds=1), + revision_content_sha256=revision_content_sha256( + reporting_revision_id=revision_id, + row_count=0, + control_totals=selected.control_totals, + reporting_rows=[], + control_total_evidence=selected.managed_control_totals, + ), + ), + [], + ) + before_old = await h.image() + fenced = json.loads( + await asyncio.to_thread( + run_step, + [str(old_python), "-I", str(legacy_script)], + label="actual-parent-projector-fence", + cwd=old_root, + value=legacy_settings, + timeout=90, + ) + ) + assert fenced["historical_projection"] == "trigger_fenced" + assert fenced["historical_materializer"] == "reservation_trigger_fenced" + assert await h.image() == before_old + settings.update( + pause=False, + new_revision_after_snapshot=revision_id, + continuation=feed_request( + case, pagination={"cursor": first["pagination"]["cursor"], "max_results": 1} + ), + new_continuation=feed_request( + case, + pagination={"cursor": activated["first"]["pagination"]["cursor"], "max_results": 1}, + ), + ) + async with installed_child(python, script, settings, tmp_path) as child: + result = await child.event("done") + assert await asyncio.wait_for(child.process.wait(), 10) == 0 + assert result["legacy"] == { + "pages": expected[0][1:], + "binding": snapshot.binding, + "version": snapshot.representation_version, + "ownership_mode": snapshot.ownership_mode, + } + assert result["new"] == activated["new_remaining"] + assert result["fresh_external_writes"] == 1 + assert result["new"]["version"] == 2 and result["new"]["ownership_mode"] == "bindings" + assert ( + await h.store.read_reporting_feed_snapshot( + snapshot.snapshot_id, caller=case.binding.principal + ) + == snapshot + ) + print( + json.dumps( + { + "b24_actual_parent_activation_restart": old["sha"], + "parent_tree": old["tree"], + "parent_wheel_sha256": hashlib.sha256(old_wheel.read_bytes()).hexdigest(), + "current": current, + "notifications": notifications, + "historical_pending": pending, + "pending_continuation": activated["pending_continuation"], + "pending_before_migration_sha256": hashlib.sha256( + json.dumps(pending_before_migration, sort_keys=True).encode() + ).hexdigest(), + "parent_fence": fenced, + "legacy_page_count": len(expected[0]), + "new_page_count": len(result["new"]["pages"]) + 1, + "legacy_snapshot_sha256": hashlib.sha256( + json.dumps(expected[0], sort_keys=True).encode() + ).hexdigest(), + "new_origins": result["origins"], + } + ), + flush=True, + ) diff --git a/tests/conformance/reporting/test_reporting_production_signing_schema.py b/tests/conformance/reporting/test_reporting_production_signing_schema.py new file mode 100644 index 000000000..fd1c578bd --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_signing_schema.py @@ -0,0 +1,118 @@ +"""3.2 normative declarations, distinct from intentionally optional JSON fields.""" + +import hashlib +import json +from copy import deepcopy + +import pytest + +from adcp.server.responses import capabilities_response +from adcp.validation import schema_loader + +from ._production_support import production_harness +from ._production_transport import MountedProduction + +PIN = "3.2.0-rc.3" +HORIZON = "/properties/webhook_signing/properties/delivery_retry_horizon_seconds" +IDENTITY = "/properties/identity/description" +CACHED = ( + ( + "protocol/get-adcp-capabilities-response.json", + 213991, + "b8bb9cbd19491f352a277b0a5e7a39d88ab1290481e0f48cc025e61be6e52be1", + ), + ( + "bundled/protocol/get-adcp-capabilities-response.json", + 802990, + "bb852633ddf0873d935ab296b284b8cdf84f1857126b6ee01d4af1338750e1c8", + ), +) + + +@pytest.mark.parametrize("relative,size,digest", CACHED) +def test_exact_cached_schema_accepts_omission_despite_normative_32_requirement( + relative, size, digest +): + root = schema_loader._resolve_schema_root(PIN) + assert root is not None + path = root.root / relative + original = path.read_bytes() + assert len(original) == size and hashlib.sha256(original).hexdigest() == digest + schema = json.loads(original) + props = schema["properties"] + horizon = props["webhook_signing"]["properties"]["delivery_retry_horizon_seconds"] + identity = props["identity"]["description"] + assert "A webhook-emitting AdCP 3.2 agent MUST populate" in horizon["description"] + assert "Retries do not extend the horizon" in horizon["description"] + assert "brand_json_url` MUST be present" in identity + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None and validator.schema == schema + omitted = capabilities_response(["media_buy"], sandbox=False, idempotency={"supported": False}) + omitted["webhook_signing"] = { + "supported": True, + "profile": "adcp/webhook-signing/v1", + "algorithms": ["ed25519"], + "legacy_hmac_fallback": False, + } + # This is an executed ACCEPTANCE, never the #1179 cached rejection. + validator.validate(omitted) + complete = deepcopy(omitted) + complete["webhook_signing"]["delivery_retry_horizon_seconds"] = 86400 + complete["identity"] = {"brand_json_url": "https://seller.example.test/brand.json"} + validator.validate(complete) + for field, value in ( + ("delivery_retry_horizon_seconds", 86399), + ("delivery_retry_horizon_seconds", 604801), + ("delivery_retry_horizon_seconds", "86400"), + ("algorithms", ["hs256"]), + ("profile", "invented-signing-profile"), + ): + invalid = deepcopy(complete) + invalid["webhook_signing"][field] = value + assert not validator.is_valid(invalid) + invalid = deepcopy(complete) + invalid["identity"]["brand_json_url"] = "http://seller.example.test/brand.json" + assert not validator.is_valid(invalid) + assert path.read_bytes() == original + print( + json.dumps( + { + "cached_optional_signing_declarations": { + "version": PIN, + "file": str(path), + "uri": f"https://adcontextprotocol.org/schemas/{PIN}/{relative}", + "bytes": size, + "sha256": digest, + "dialect": schema["$schema"], + "normative_pointers": {HORIZON: horizon, IDENTITY: identity}, + "omitted_payload": omitted, + "unmodified_schema_result": "accepted", + "semantic_32_result": "missing required horizon and operator declaration", + } + }, + sort_keys=True, + ) + ) + + +async def test_actual_public_declarations_and_schema_optional_omissions_on_all_mounts(tmp_path): + async with production_harness( + "postgres", tmp_path / "destination.sqlite", notifications=True, notification_delivery=True + ) as h: + mount = MountedProduction(h) + mount.authorize(h.item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, raw = await mount.call(client, "get_adcp_capabilities", {}, transport=transport) + assert raw["webhook_signing"]["delivery_retry_horizon_seconds"] == 86400 + assert raw["identity"]["brand_json_url"] == ( + "https://seller.example.test/brand.json" + ) + for relative, _, _ in CACHED: + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None + validator.validate(raw) + omitted = deepcopy(raw) + del omitted["webhook_signing"]["delivery_retry_horizon_seconds"] + del omitted["identity"] + validator.validate(omitted) diff --git a/tests/conformance/reporting/test_reporting_production_transactions.py b/tests/conformance/reporting/test_reporting_production_transactions.py new file mode 100644 index 000000000..cfd739cb3 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_transactions.py @@ -0,0 +1,406 @@ +"""Admitted finish atomicity and permanent quarantine across production activation.""" + +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ( + ReportingMaterializationAttempt, + ReportingMaterializationRecord, + ReportingObligationDeliveryRecord, +) +from adcp.reporting.materializer import ReportingDestinationRequest, ReportingWriterError +from adcp.reporting.materializer.memory import InMemoryReportingMaterializerStore +from adcp.reporting.materializer.work import ReportingMaterializerLease + +from ._feed_support import feed_request, walk +from ._production_support import production_harness +from ._projection_support import drain +from .test_reporting_materializer_transactions import postgres_failure +from .test_reporting_production_lock_order import source_turn + + +async def production_queue(h): + if h.pool is None: + state = h.store._production_outbox + return ( + ((), ()) + if state is None + else ( + tuple(state.events.values()), + tuple(w.state for w in state.expansions.values()), + ) + ) + async with h.pool.connection() as c: + events = await ( + await c.execute("SELECT snapshot FROM reporting_production_notification_events") + ).fetchall() + work = await ( + await c.execute("SELECT state FROM reporting_production_notification_expansions") + ).fetchall() + return tuple(r[0] for r in events), tuple(r[0] for r in work) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("position", ["outcome", "status_head", "account_head", "boundary", "ack"]) +async def test_production_finish_fault_restores_every_row_head_capture_and_frozen_page( + backend, notifications, position, monkeypatch, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=notifications, count=1 + ) as h: + item = h.item + await h.production.activate(account_id=item.config.account_id) + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) and lease.admission_epoch == 2 + prepared, verified = await item.verified(lease) + first = await h.store.read_reporting_feed(feed_request(item), caller=item.binding.principal) + frozen = await walk(h.store, feed_request(item), item.binding.principal, first=first) + old_queue, ordinary = await h.queue(), await h.ordinary_events() + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if h.pool is not None: + prefixes = { + "outcome": "INSERT INTO reporting_reconciliation_records", + "status_head": "INSERT INTO reporting_production_status_heads", + "account_head": "UPDATE reporting_materializer_accounts SET captured_sequence=", + "boundary": "INSERT INTO reporting_production_status_boundaries", + "ack": "UPDATE reporting_production_work SET state='acked'", + } + with postgres_failure(monkeypatch, prefixes[position]) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + import adcp.reporting.materializer.memory as memory + + cls = InMemoryReportingMaterializerStore + if position == "outcome": + original = cls._commit_record_unlocked + + def fail(self, record, **kwargs): + result = original(self, record, **kwargs) + if isinstance(record, ReportingMaterializationRecord): + raise OSError("injected production finish") + return result + + target, method = cls, "_commit_record_unlocked" + elif position in {"status_head", "account_head"}: + + def fail(*args, **kwargs): + raise OSError("injected production finish") + + target, method = memory, "ReportingMaterializerBoundary" + else: + method = "_materializer_dirty" if position == "boundary" else "_park" + target, original = cls, getattr(cls, method) + + def fail(self, *args, **kwargs): + original(self, *args, **kwargs) + raise OSError("injected production finish") + + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + assert ( + await walk(h.store, feed_request(item), item.binding.principal, first=first) == frozen + ) + assert (await finish()).state == "verified" + assert len(await item.outcomes()) == 1 + assert len(await h.store.read_production_boundaries(caller=item.binding.principal)) == 1 + assert await h.store.read_materializer_boundaries(caller=item.binding.principal) == () + assert len((await production_queue(h))[0]) == int(notifications) + assert await h.queue() == old_queue and await h.ordinary_events() == ordinary + committed = await h.image() + assert (await finish()).state == "verified" + assert await h.image() == committed + assert item.writer.writes == 1 + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("position", ["event", "expansion", "missing_event"]) +async def test_enabled_production_enqueue_is_required_inside_verified_finish( + backend, position, monkeypatch, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=True, count=1 + ) as h: + item = h.item + await h.production.activate(account_id=item.config.account_id) + lease = await item.claim() + prepared, verified = await item.verified(lease) + before = await h.image() + + async def finish(): + return await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + + if h.pool is not None and position != "missing_event": + table = "events" if position == "event" else "expansions" + with postgres_failure( + monkeypatch, f"INSERT INTO reporting_production_notification_{table}" + ) as hit: + with pytest.raises(ReportingWriterError): + await finish() + assert len(hit) == 1 + else: + with monkeypatch.context() as patch: + if h.pool is not None: + import adcp.reporting.materializer.pg as pg + + async def empty(connection, event): + pass + + patch.setattr(pg, "enqueue_materializer_event_on", empty) + else: + import adcp.reporting.outbox.memory as memory + + if position == "event": + + def fail(*args, **kwargs): + raise OSError("injected production event") + + patch.setattr(memory, "_Work", fail) + else: + original = memory.NotificationState.enqueue + + def enqueue(self, event): + if position != "missing_event": + original(self, event) + raise OSError("injected production expansion") + + patch.setattr(memory.NotificationState, "enqueue", enqueue) + with pytest.raises(ReportingWriterError): + await finish() + assert await h.image() == before + assert await production_queue(h) == ((), ()) + assert (await finish()).state == "verified" + events, work = await production_queue(h) + assert len(events) == 1 and work == ("pending",) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_explicit_old_pending_import_keeps_epoch_and_external_identity_after_activation( + backend, notifications, tmp_path, monkeypatch +): + path = tmp_path / "destination.sqlite" + async with production_harness(backend, path, notifications=notifications, count=1) as h: + item = h.item + await h.store.bind_obligation_delivery( + ReportingObligationDeliveryRecord( + item.scope, + "USD", + item.revision.created_at + timedelta(days=400), + item.revision.created_at, + ) + ) + attempt = ReportingMaterializationAttempt( + item.scope, + item.revision.reporting_revision_id, + "preactivation-import", + 1, + item.revision.created_at, + ) + await h.store.commit_materialization_attempt(attempt) + external = ReportingDestinationRequest.from_binding( + item.binding, attempt, item.verifier.key + ).external_id + await h.store.import_pending_materialization( + scope=item.scope, + reporting_materialization_id=attempt.reporting_materialization_id, + original_external_id=external, + keys=item.keys, + ) + await h.production.activate(account_id=item.config.account_id) + original = await h.works() + await h.production.aclose() + observed = [] + original_finish = type(h.store).finish_materialization + + async def observe_finish(self, lease, **kwargs): + result = await original_finish(self, lease, **kwargs) + observed.append((lease, kwargs, result)) + return result + + monkeypatch.setattr(type(h.store), "finish_materialization", observe_finish) + async with production_harness( + backend, + path, + notifications=notifications, + count=1, + existing_store=h.store if h.pool is None else None, + existing_pool=h.pool, + ) as fresh: + # start() waits for its first real worker turn. The cold support + # has already resumed and completed the original pending effect. + assert await fresh.works() == tuple((key, "acked", gen) for key, _, gen in original) + assert len(observed) == 1 + lease, arguments, result = observed[0] + await fresh.production.activate(account_id=item.config.account_id) + assert lease.admission_epoch == 0 + assert lease.attempt == attempt and lease.request.external_id == external + assert result.state == "verified" + assert fresh.item.writer.writes == 1 + assert await production_queue(fresh) == ((), ()) + events, work = await fresh.queue() + assert len(events) == int(notifications) + assert work == (("quarantined",) if notifications else ()) + assert await fresh.store.read_production_boundaries(caller=item.binding.principal) == () + assert ( + len(await fresh.store.read_materializer_boundaries(caller=item.binding.principal)) + == 1 + ) + frozen = await fresh.image() + assert ( + await fresh.store.finish_materialization(lease, **arguments) + ).state == "verified" + assert await fresh.image() == frozen + await drain(fresh.projection, item.config.account_id) + assert await production_queue(fresh) == ((), ()) + assert await fresh.queue() == (events, work) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("source_changes", [None, "readability", "configuration"]) +async def test_producer_lease_bookkeeping_preserves_io_but_real_source_change_fences_it( + backend, notifications, source_changes, tmp_path +): + async with production_harness( + backend, tmp_path / "destination.sqlite", notifications=notifications, count=1 + ) as h: + item = h.item + await h.production.activate(account_id=item.config.account_id) + lease = await item.claim() + assert isinstance(lease, ReportingMaterializerLease) and lease.admission_epoch == 2 + prepared, verified = await item.verified(lease) + first = await h.store.read_reporting_feed(feed_request(item), caller=item.binding.principal) + frozen = await walk(h.store, feed_request(item), item.binding.principal, first=first) + # The source takes and releases its real fair configuration lease + # while a verified external effect is still awaiting its fenced finish. + turn = await source_turn(h.production) + assert turn.leased is not None and not turn.revisions_committed + producer = h.production.offerings[0].producer + token = h.production._producer_turn.set(producer) + try: + + async def acquire(worker, seconds): + return await h.store.lease_period_close( + worker_id=worker, + now=h.source_clock() + timedelta(seconds=seconds), + lease_seconds=1, + ) + + crashed = await acquire("crashed-producer", 0) + assert crashed is not None + assert await acquire("duplicate-producer", 0) is None + recovered = await acquire("recovered-producer", 2) + assert recovered is not None + await h.store.release_period_close(crashed, worker_id="crashed-producer") + assert await acquire("duplicate-producer", 2) is None + await h.store.release_period_close(recovered, worker_id="recovered-producer") + for seconds in (3, 4): + repeated = await acquire("scheduled-producer", seconds) + assert repeated is not None + await h.store.release_period_close(repeated, worker_id="scheduled-producer") + finally: + h.production._producer_turn.reset(token) + assert ( + await walk(h.store, feed_request(item), item.binding.principal, first=first) == frozen + ) + if source_changes == "readability": + await h.store.set_revision_readable( + account_id=item.config.account_id, + reporting_revision_id=item.revision.reporting_revision_id, + readable=False, + ) + elif source_changes == "configuration": + from dataclasses import replace + + await h.store.put_configuration(replace(item.config, deactivated_at=h.clock())) + result = await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + assert result.state == ("failed" if source_changes else "verified") + outcomes = await item.outcomes() + assert len(outcomes) == 1 + assert ( + outcomes[0].reporting_materialization_id == lease.attempt.reporting_materialization_id + ) + assert item.writer.writes == 1 + assert len((await production_queue(h))[0]) == int(notifications and not source_changes) + assert await h.queue() == ((), ()) + committed = await h.image() + assert ( + await h.store.finish_materialization(lease, prepared=prepared, verified=verified) + == result + ) + assert await h.image() == committed + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("release", [False, True]) +async def test_pg_lease_bookkeeping_fault_rolls_back_configuration_candidate_and_fairness( + notifications, release, monkeypatch, tmp_path +): + async with production_harness( + "postgres", tmp_path / "destination.sqlite", notifications=notifications, count=1 + ) as h: + await h.production.activate(account_id=h.item.config.account_id) + pending = await h.item.claim() + assert isinstance(pending, ReportingMaterializerLease) + producer = h.production.offerings[0].producer + token = h.production._producer_turn.set(producer) + try: + + async def acquire(): + return await h.store.lease_period_close( + worker_id="bookkeeping-fault", now=h.source_clock(), lease_seconds=30 + ) + + lease = await acquire() if release else None + before = await h.image() + async with h.pool.connection() as c: + ranks = await ( + await c.execute( + "SELECT * FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + with postgres_failure( + monkeypatch, "UPDATE reporting_materializer_candidates SET generation=generation-1" + ) as hit: + with pytest.raises(OSError, match="injected transaction boundary"): + if release: + await h.store.release_period_close(lease, worker_id="bookkeeping-fault") + else: + await acquire() + assert len(hit) == 1 + assert await h.image() == before + async with h.pool.connection() as c: + assert ( + await ( + await c.execute( + "SELECT * FROM adcp_reporting_configuration_lease_turns" + " ORDER BY account_id,delivery_config_id,delivery_config_version" + ) + ).fetchall() + == ranks + ) + lease = lease if release else await acquire() + assert lease is not None + await h.store.release_period_close(lease, worker_id="bookkeeping-fault") + # A stale or duplicate release cannot remove another generation. + image = await h.image() + await h.store.release_period_close(lease, worker_id="bookkeeping-fault") + assert await h.image() == image + prepared, verified = await h.item.verified(pending) + assert ( + await h.store.finish_materialization(pending, prepared=prepared, verified=verified) + ).state == "verified" + finally: + h.production._producer_turn.reset(token) diff --git a/tests/conformance/reporting/test_reporting_projection_capture.py b/tests/conformance/reporting/test_reporting_projection_capture.py new file mode 100644 index 000000000..58fd5428b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_capture.py @@ -0,0 +1,143 @@ +"""Ordered capture, checkpoint, activation, rollback and frozen feed integration.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ownership import page_revision_ownership + +from ._feed_support import feed_request, walk +from ._generation_support import END +from ._projection_support import drain, inputs, projections +from ._receipt_support import adjustment_for, receipt_case, request_for + +__all__ = ["projections"] + + +async def test_captured_receipt_adjustment_and_readability_cycles_are_not_collapsed(projections): + h = projections + s = await receipt_case(h) + account = s.obligation.account_id + assert await h.projection.activate(account_id=account) + assert await h.projection.baseline_ready(account_id=account) + starting = next( + c.generation + for c in await h.projection.checkpoints(account_id=account) + if c.scope.reporting_obligation_id == s.obligation.reporting_obligation_id + and c.scope.consumer_id == s.binding.consumer_id + ) + baseline = await inputs(h, account) + assert len(baseline) == 1 + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + rejected = ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "projection-corruption", + "corrupt", + END + timedelta(seconds=30), + ) + await h.store.record_materialization_check(rejected) + await h.store.record_materialization_check( + replace( + rejected, + check_id="projection-recovery", + state="readable", + checked_at=END + timedelta(seconds=31), + ) + ) + item = await adjustment_for(h, s) + await h.store.ingest_receipt_batch( + { + "account": {"account_id": account}, + "idempotency_key": "projection-adjustment-0001", + "adjustment_receipts": [item], + }, + caller=s.binding.principal, + ) + frozen = await inputs(h, account) + assert len(frozen) == 6 + # All five transitions are pending while today's rows already look recovered. + assert len(frozen[0].reconciliation) + 4 == len(frozen[-1].reconciliation) + transitions = await drain(h.projection, account) + assert len(transitions) == 5 + checkpoints = await h.projection.checkpoints(account_id=account) + obligation = next( + c + for c in checkpoints + if c.scope.reporting_obligation_id == s.obligation.reporting_obligation_id + and c.scope.consumer_id == s.binding.consumer_id + ) + assert obligation.generation == starting + 5 + assert obligation.snapshot["health"] == "complete" + assert ( + all(t.events > 0 for t in transitions) + if h.projection.policy["notifications_enabled"] + else all(t.events == 0 for t in transitions) + ) + assert await inputs(h, account) == frozen + before = await h.image() + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + assert await h.image() == before + assert not await h.projection.activate(account_id=account) + + +async def test_activation_preserves_legacy_snapshot_and_new_pages_have_exact_local_ownership( + projections, +): + h = projections + s = await receipt_case(h) + req = feed_request(s) + first = await h.store.read_reporting_feed(req, caller=s.binding.principal) + legacy = await walk(h.store, req, s.binding.principal, first=first) + assert await h.projection.activate(account_id=s.obligation.account_id) + resumed = await walk(h.store, req, s.binding.principal, first=first) + assert resumed == legacy + new = await walk(h.store, req, s.binding.principal) + for page in new[0]: + bindings = page_revision_ownership(page) + assert bindings is not None + assert set(bindings) == {r["reporting_revision_id"] for r in page.get("revisions", [])} + assert all(owner == s.obligation.reporting_obligation_id for owner in bindings.values()) + assert new[0][0]["changes_checkpoint"] != first["changes_checkpoint"] + + +async def test_capture_failure_rolls_back_every_source_collection_or_row(projections, monkeypatch): + h = projections + s = await receipt_case(h) + await h.projection.activate(account_id=s.obligation.account_id) + before = await h.image() + if h.pool is None: + + def fail(self, account_id): + raise ReportingNotificationError("status_projection_history_corrupt") + + monkeypatch.setattr(type(h.store), "_capture_projection", fail) + with pytest.raises(ReportingNotificationError): + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + monkeypatch.undo() + else: + from psycopg import sql + + async with h.pool.connection() as c: + await c.execute( + "CREATE FUNCTION test_projection_failure() RETURNS trigger LANGUAGE plpgsql" + " AS $$BEGIN RAISE EXCEPTION 'injected'; END$$" + ) + await c.execute( + "CREATE TRIGGER test_projection_failure" + " BEFORE INSERT ON reporting_projection_inputs" + " FOR EACH ROW EXECUTE FUNCTION test_projection_failure()" + ) + try: + with pytest.raises(Exception): + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + finally: + async with h.pool.connection() as c: + await c.execute( + sql.SQL("DROP TRIGGER test_projection_failure ON reporting_projection_inputs") + ) + await c.execute("DROP FUNCTION test_projection_failure()") + assert await h.image() == before diff --git a/tests/conformance/reporting/test_reporting_projection_history.py b/tests/conformance/reporting/test_reporting_projection_history.py new file mode 100644 index 000000000..0a8b0949f --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_history.py @@ -0,0 +1,261 @@ +"""Retained preactivation inputs, incremental cutover and permanent quarantine.""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone + +import pytest + +from adcp.reporting.ledger import ( + ReportingAdjustmentReceiptRecord, + ReportingAdjustmentRecord, + ReportingControlTotalRecord, + ReportingRevisionReceiptRecord, +) +from adcp.reporting.ledger.delivery import adjustment_to_wire +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.projection.history import checkpoint_document, checkpoint_key + +from ._durable_materializer_support import durable_case +from ._projection_support import projection_harness, projections + +__all__ = ["projections"] + + +async def history_image(h, account): + if h.pool is None: + state = h.store._projection_accounts[account] + return [(sequence, document) for sequence, document in state.historical_steps] + async with h.pool.connection() as c: + return await ( + await c.execute( + "SELECT account_sequence,input FROM reporting_projection_legacy_steps" + " WHERE account_id=%s ORDER BY account_sequence,scope_key", + (account,), + ) + ).fetchall() + + +async def prepare_history(h): + h.clock.now = datetime.now(timezone.utc) + first = await durable_case( + h.store, required="official", finality="official", reconciliation_mode="consumer_receipt" + ) + second = await durable_case( + h.store, + required="official", + finality="official", + reconciliation_mode="consumer_receipt", + consumer="https://buyer.example.test/other", + ) + for _ in range(2): + lease = await first.claim() + case = first if lease.scope == first.scope else second + prepared, evidence = await case.verified(lease) + await h.store.finish_materialization(lease, prepared=prepared, verified=evidence) + outcome = (await first.outcomes())[0] + h.clock.now = datetime.now(timezone.utc) + receipt = ReportingRevisionReceiptRecord( + first.scope, + "historical-receipt", + first.revision.reporting_revision_id, + outcome.reporting_materialization_id, + "accepted", + first.binding.verification_profile, + first.revision.row_count, + first.revision.managed_control_totals, + h.clock(), + observed_canonical_content_digest=first.revision.canonical_content_digest, + ) + await h.store.record_revision_receipt(receipt) + adjustment = ReportingAdjustmentRecord( + "historical-adjustment", + first.config.account_id, + first.revision.reporting_revision_id, + "source_correction", + first.obligation.period.end, + first.obligation.period.end + timedelta(days=30), + (("spend", "-0.50"),), + h.clock(), + h.clock(), + managed_control_total_deltas=( + ReportingControlTotalRecord("spend", "-0.50", "decimal", "USD"), + ), + ) + await h.store.commit_adjustment(adjustment) + rejected = ReportingAdjustmentReceiptRecord( + first.scope, + "historical-adjustment-rejected", + adjustment.reporting_adjustment_id, + first.revision.reporting_revision_id, + "rejected", + adjustment_to_wire(adjustment)["canonical_adjustment_sha256"], + h.clock(), + rejection_codes=("CONTROL_TOTAL_MISMATCH",), + ) + await h.store.record_adjustment_receipt(rejected) + await h.store.record_adjustment_receipt( + replace( + rejected, + reporting_receipt_id="historical-adjustment-accepted", + status="accepted", + supersedes_reporting_receipt_id=rejected.reporting_receipt_id, + rejection_codes=(), + ) + ) + h.clock.now = datetime.now(timezone.utc) + return first, second + + +async def original_inputs(h, cases): + result = [] + for case in cases: + records = ( + *await h.store.read_materializer_boundaries(caller=case.scope.principal), + *await h.store.read_receipt_boundaries(caller=case.scope.principal), + ) + result.extend(b.to_storage() for b in records) + return tuple(result) + + +async def test_captured_history_replays_after_interruption_without_promoting_readiness( + projections, monkeypatch +): + await history_replay(projections, monkeypatch, legacy_baseline=False) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_captured_history_preserves_the_existing_c_baseline(backend, monkeypatch): + async with projection_harness(backend, notifications=True) as h: + await history_replay(h, monkeypatch, legacy_baseline=True) + + +async def history_replay(h, monkeypatch, *, legacy_baseline): + first, second = await prepare_history(h) + account = first.config.account_id + baselines = () + if legacy_baseline: + if h.pool is None: + from adcp.reporting.outbox.status_memory import InMemoryStatusNotificationStore + + old = InMemoryStatusNotificationStore(h.store) + else: + from adcp.reporting.outbox.status_pg import PgStatusNotificationStore + + old = PgStatusNotificationStore(h.store) + assert await old.baseline(account_id=account) + baselines = await old.checkpoints(account_id=account) + assert baselines + original_queue = await h.queue() + originals = await original_inputs(h, (first, second)) + assert len(originals) == 5 + assert await h.projection._begin_activation(account_id=account) + assert not await h.projection.baseline_ready(account_id=account) + if h.pool is None: + archived = h.store._projection_accounts[account].baselines + assert archived == {checkpoint_key(c): c for c in baselines} + else: + async with h.pool.connection() as c: + archived = dict( + await ( + await c.execute( + "SELECT scope_key,checkpoint FROM reporting_projection_legacy_baselines" + " WHERE account_id=%s", + (account,), + ) + ).fetchall() + ) + assert archived == {checkpoint_key(c): checkpoint_document(c) for c in baselines} + assert (await h.projection.project_one(account_id=account)).did_work + before = await history_image(h, account) + assert before + # An interrupted invocation has a committed first input; the next complete + # transaction fails after preparing its replay, and must advance nothing. + if h.pool is None: + import adcp.reporting.projection.memory as module + + original = module.project_boundary + + def failure(*args, **kwargs): + original(*args, **kwargs) + raise RuntimeError("injected replay transaction failure") + + with monkeypatch.context() as patch: + patch.setattr(module, "project_boundary", failure) + with pytest.raises(RuntimeError): + await h.projection.project_one(account_id=account) + else: + async with h.pool.connection() as c: + await c.execute( + "CREATE FUNCTION fail_history() RETURNS trigger LANGUAGE plpgsql AS" + " $$BEGIN RAISE EXCEPTION 'injected replay failure'; END$$" + ) + await c.execute( + "CREATE TRIGGER fail_history BEFORE INSERT ON reporting_projection_legacy_steps" + " FOR EACH ROW EXECUTE FUNCTION fail_history()" + ) + try: + with pytest.raises(Exception): + await h.projection.project_one(account_id=account) + finally: + async with h.pool.connection() as c: + await c.execute("DROP TRIGGER fail_history ON reporting_projection_legacy_steps") + await c.execute("DROP FUNCTION fail_history()") + assert await history_image(h, account) == before + projection = type(h.projection)( + h.store, + consumer_status_enabled=False, + revision_ownership=True, + ) + assert not await projection.activate(account_id=account) + assert await projection.baseline_ready(account_id=account) + history = await history_image(h, account) + personal = [ + doc["checkpoint"] + for _, doc in history + if doc["checkpoint"]["scope"]["consumer_id"] == first.binding.consumer_id + and doc["checkpoint"]["scope"]["reporting_obligation_id"] + == first.obligation.reporting_obligation_id + ] + assert [c["snapshot"]["health"] for c in personal] == [ + "action_required", + "complete", + "action_required", + "complete", + ] + assert [c["generation"] for c in personal] == [1, 2, 3, 4] + assert all(doc["admission_epoch"] == 0 for _, doc in history) + assert {doc["checkpoint"]["scope"]["consumer_id"] for _, doc in history} == { + first.binding.consumer_id, + second.binding.consumer_id, + } + assert await h.queue() == original_queue + current = await projection.checkpoints(account_id=account) + assert ( + next( + c + for c in current + if c.scope.consumer_id == first.binding.consumer_id + and c.scope.reporting_obligation_id == first.obligation.reporting_obligation_id + ).generation + >= 4 + ) + assert await original_inputs(h, (first, second)) == originals + assert not await projection.activate(account_id=account) + + +async def test_missing_retained_capture_refuses_activation(projections): + h = projections + first, _ = await prepare_history(h) + account = first.config.account_id + if h.pool is None: + h.store._materializer_account_heads[account] += 1 + else: + async with h.pool.connection() as c: + await c.execute( + "UPDATE reporting_materializer_accounts SET captured_sequence=captured_sequence+1" + " WHERE account_id=%s", + (account,), + ) + with pytest.raises(ReportingNotificationError, match="status_projection_history_corrupt"): + await h.projection.activate(account_id=account) + assert not await h.projection.baseline_ready(account_id=account) diff --git a/tests/conformance/reporting/test_reporting_projection_memory_rollback.py b/tests/conformance/reporting/test_reporting_projection_memory_rollback.py new file mode 100644 index 000000000..e4964a77b --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_memory_rollback.py @@ -0,0 +1,160 @@ +"""Shared frozen inputs never share mutable transaction or public response state.""" + +from copy import deepcopy +from dataclasses import FrozenInstanceError, replace +from datetime import timedelta, tzinfo + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.projection.capture import ReportingProjectionInput + +from ._feed_support import feed_request, walk +from ._projection_support import projection_harness +from ._receipt_support import receipt_case, request_for + + +class InjectedMutationError(Exception): + pass + + +async def test_shared_capture_rejects_a_mutable_timezone(): + class MutableZone(tzinfo): + offset = timedelta(0) + + def utcoffset(self, dt): + return self.offset + + def dst(self, dt): + return timedelta(0) + + async with projection_harness("memory", notifications=False) as h: + case = await receipt_case(h) + await h.projection.activate(account_id=case.obligation.account_id) + captured = h.store._projection_accounts[case.obligation.account_id].inputs[0] + zone = MutableZone() + core = replace(captured.core, as_of=captured.core.as_of.replace(tzinfo=zone)) + with pytest.raises(ReportingNotificationError, match="status_projection_history_corrupt"): + ReportingProjectionInput(core, captured.reconciliation, captured.document) + + +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("point", ["receipt_ordinal", "projection_capture", "feed_save"]) +async def test_memory_rollback_preserves_frozen_history_and_nested_public_values( + notifications, point, monkeypatch +): + async with projection_harness("memory", notifications=notifications) as h: + case = await receipt_case(h) + caller, account = case.binding.principal, case.obligation.account_id + await h.projection.activate(account_id=account) + request = feed_request(case) + first = await h.store.read_reporting_feed(request, caller=caller) + pages, records, checkpoint = await walk(h.store, request, caller, first=first) + public_checkpoints = await h.projection.checkpoints(account_id=account) + captured = h.store._projection_accounts[account].inputs[0] + captured_bytes = captured.document + assert deepcopy(captured) is captured + before = await h.image() + published = deepcopy((first, pages, records, checkpoint, public_checkpoints)) + + # Returned JSON is mutable by design; changing it must not lend a way + # to mutate retained bytes, a checkpoint, or a subsequent page. + first["ext"]["adcp"]["reporting_revision_ownership"]["bindings"].append( + {"reporting_revision_id": "caller-only", "reporting_obligation_id": "caller-only"} + ) + public_checkpoints[0].snapshot["issues"].append({"code": "CALLER_ONLY"}) + frozen = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + frozen_bytes = canonical_json_utf8_v1(frozen.to_storage()) + frozen.inputs["core"]["revisions"][0]["readable"] = False + assert await h.image() == before + first, pages, records, checkpoint, public_checkpoints = published + retained = deepcopy(published) + + injected = False + + def fail_with_nested_changes(): + nonlocal injected + injected = True + # A failure after mutating actual transaction-owned nested JSON + # must restore it, as well as ordinary immutable domain records. + state = h.store._status_notification_state + next(iter(state.checkpoints.values())).snapshot["issues"].append( + {"code": "TRANSACTION_ONLY", "details": {"partial": [1, 2]}} + ) + h.store._new_projection_collection = {"sequence_head": 1, "partial": ["new"]} + raise InjectedMutationError(point) + + name = { + "receipt_ordinal": "_append_receipt_result", + "projection_capture": "_capture_projection", + "feed_save": "_save_feed_snapshot", + }[point] + original = getattr(type(h.store), name) + + def fail_after(self, *args, **kwargs): + original(self, *args, **kwargs) + fail_with_nested_changes() + + with monkeypatch.context() as patch: + patch.setattr(type(h.store), name, fail_after) + with pytest.raises(Exception): + async with h.store.transaction(): + await h.store.set_revision_readable( + account_id=account, + reporting_revision_id=case.revision.reporting_revision_id, + readable=False, + ) + if point == "feed_save": + await h.store.read_reporting_feed(request, caller=caller) + else: + await h.store.ingest_receipt_batch(request_for(case), caller=caller) + assert injected + assert not hasattr(h.store, "_new_projection_collection") + assert await h.image() == before + assert (first, pages, records, checkpoint, public_checkpoints) == retained + assert await h.projection.checkpoints(account_id=account) == public_checkpoints + assert h.store._projection_accounts[account].inputs[0] is captured + assert captured.document == captured_bytes + resumed = await walk(h.store, request, caller, first=first) + assert resumed == (pages, records, checkpoint) + reread = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + assert canonical_json_utf8_v1(reread.to_storage()) == frozen_bytes + with pytest.raises(FrozenInstanceError): + captured.core.revisions[0].readable = False + with pytest.raises(ReportingNotificationError): + ReportingProjectionInput( + replace(captured.core, configurations=list(captured.core.configurations)), + captured.reconciliation, + captured.document, + ) + # Restore the fault and prove the real transaction can still commit. + result = await h.store.ingest_receipt_batch(request_for(case), caller=caller) + assert result["results"][0]["result"] == "recorded" + assert captured.document == captured_bytes + + +@pytest.mark.parametrize("notifications", [False, True]) +async def test_failed_first_feed_capture_removes_new_collection(notifications, monkeypatch): + async with projection_harness("memory", notifications=notifications) as h: + case = await receipt_case(h) + await h.projection.activate(account_id=case.obligation.account_id) + assert not hasattr(h.store, "_reporting_feed_snapshots") + before = await h.image() + original = type(h.store)._save_feed_snapshot + + def fail(self, stored): + original(self, stored) + assert self._reporting_feed_snapshots + raise InjectedMutationError("first feed") + + with monkeypatch.context() as patch: + patch.setattr(type(h.store), "_save_feed_snapshot", fail) + with pytest.raises(Exception): + await h.store.read_reporting_feed(feed_request(case), caller=case.binding.principal) + assert await h.image() == before + assert not hasattr(h.store, "_reporting_feed_snapshots") diff --git a/tests/conformance/reporting/test_reporting_projection_notifications.py b/tests/conformance/reporting/test_reporting_projection_notifications.py new file mode 100644 index 000000000..9cb55f5f1 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_notifications.py @@ -0,0 +1,72 @@ +"""New status queue identity, old-worker exclusion and same-transaction rollback.""" + +import pytest + +from adcp.reporting.outbox.status_memory import InMemoryReportingStatusOutbox + +from ._projection_support import projection_harness +from ._receipt_support import receipt_case, request_for + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("fault", [False, True]) +async def test_status_queue_isolation_and_atomic_projection(backend, fault, monkeypatch): + async with projection_harness(backend, notifications=True) as h: + case = await receipt_case(h) + account = case.obligation.account_id + if h.pool is None: + old = InMemoryReportingStatusOutbox(h.store) + else: + from adcp.reporting.outbox.status_pg import PgReportingStatusOutbox + + old = PgReportingStatusOutbox(pool=h.pool) + assert await old.list_events(account_id=account) == () + original_readiness = await h.queue() + await h.projection.activate(account_id=account) + assert await h.projection.outbox.list_events(account_id=account) == () + await h.store.ingest_receipt_batch(request_for(case), caller=case.binding.principal) + before = await h.image() + if fault: + if h.pool is None: + original = h.projection._enqueue_status + + def fail(event): + original(event) + raise RuntimeError("injected status enqueue failure") + + method = "_enqueue_status" + else: + original = h.projection._enqueue_status_on + + async def fail(connection, event): + await original(connection, event) + raise RuntimeError("injected status enqueue failure") + + method = "_enqueue_status_on" + with monkeypatch.context() as patch: + patch.setattr(h.projection, method, fail) + with pytest.raises(RuntimeError, match="injected status enqueue failure"): + await h.projection.project_one(account_id=account) + assert await h.image() == before + turn = await h.projection.project_one(account_id=account) + assert turn.did_work and turn.events > 0 + events = await h.projection.outbox.list_events(account_id=account) + assert len(events) == turn.events + assert all(e.notification_type == "reporting.status_changed" for e in events) + assert await old.list_events(account_id=account) == () + assert ( + await old.claim_expansion(account_id=account, now=h.clock(), lease_seconds=30) is None + ) + assert await h.queue() == original_readiness + # Persist a real expansion lease, then verify a competing incarnation + # cannot claim it and a separately constructed new outbox resumes it. + current = h.projection.outbox + lease = await current.claim_expansion(account_id=account, now=h.clock(), lease_seconds=30) + assert lease is not None + restarted = type(current)(h.store) if h.pool is None else type(current)(pool=h.pool) + await restarted.complete_expansion(lease, (), now=h.clock()) + assert await current.list_events(account_id=account) == events + assert await h.queue() == original_readiness + await h.store.ingest_receipt_batch(request_for(case), caller=case.binding.principal) + assert not (await h.projection.project_one(account_id=account)).did_work + assert await current.list_events(account_id=account) == events diff --git a/tests/conformance/reporting/test_reporting_projection_schedule.py b/tests/conformance/reporting/test_reporting_projection_schedule.py new file mode 100644 index 000000000..a0b683ea6 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_schedule.py @@ -0,0 +1,174 @@ +"""Captured generation forecasts agree with actual period creation (#1179).""" + +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from itertools import islice + +import pytest + +from adcp.reporting.ledger import ProducerOfferings, ReportingProducer, ReportingScheduleSpec +from adcp.reporting.ledger.schedule import committed_periods, next_reporting_expectation +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.types import GetReportingStatusResponse +from adcp.validation.schema_loader import get_validator + +from ._generation_support import START, UncalledSource, configuration, revision_for +from ._reliable_support import reliable_factory + + +@pytest.fixture( + params=[("memory", False), ("memory", True), ("postgres", False), ("postgres", True)] +) +async def schedules(request): + backend, notifications = request.param + async with reliable_factory(backend, notifications=notifications) as harness: + yield harness + + +def hour(value): + return START + timedelta(hours=value) + + +@pytest.mark.parametrize( + "activation,deactivation,now,expected,closed", + [ + (0, None, 0.5, 2, 0), # mid-period, no obligation or coverage yet + (0, None, 1, 2, 1), # closes at 01:00, due at 02:00 + (0, None, 1.5, 2, 1), + (0, None, 2, 3, 2), # strictly future expectation at the exact SLA + (0, None, 2.5, 3, 2), + (3, None, 0.5, 5, 0), # already committed future activation + (0.5, None, 0.5, 3, 0), # first full period only + (1, None, 1, 3, 0), + (0, 3, 0.5, 2, 0), # future deactivation + (0, 1, 2, None, 1), # stop exactly at the next start + (0, 0.5, 0.5, 2, 0), # begun full period remains owed + (0, 0.5, 2, None, 1), + (1, 1, 0.5, None, 0), # no committed active interval + (None, None, 0.5, None, 0), + ], +) +async def test_producer_and_public_summary_share_all_activation_and_due_boundaries( + schedules, activation, deactivation, now, expected, closed +): + h = schedules + h.clock.now = hour(now) + config = replace( + configuration(), + activated_at=hour(activation) if activation is not None else None, + deactivated_at=hour(deactivation) if deactivation is not None else None, + ) + await h.store.put_configuration(config) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + obligations = await producer.close_elapsed_periods(config, now=h.clock()) + assert len(obligations) == closed + assert await producer.close_elapsed_periods(config, now=h.clock()) == [] + # Completing all existing evidence must not erase tomorrow's commitment. + for obligation in obligations: + revision, rows = revision_for(obligation, suffix=obligation.reporting_obligation_id) + await h.store.commit_revision(revision, rows) + handler = ReportingStatusHandler(h.store) + for consumer in ("buyer-one", "buyer-two"): + raw = await handler.handle({}, caller=ReportingStatusCaller("acct_a", consumer)) + GetReportingStatusResponse.model_validate(raw) + validator = get_validator("get_reporting_status", "sync") + assert validator is not None + validator.validate(raw) + assert raw["health"] == "complete" + assert raw["obligation_counts"]["total"] == closed + assert raw.get("next_expected_at") == ( + hour(expected).isoformat().replace("+00:00", "Z") if expected is not None else None + ) + if not closed: + assert raw["coverage"]["media_buy_ids"] == [] + assert raw["issues"] == [] + + +async def test_nearest_generation_and_account_filters_use_captured_not_current_configuration( + schedules, +): + h = schedules + h.clock.now = hour(0.5) + first = replace(configuration(), deactivated_at=None) + second = replace( + first, delivery_config_version=2, schedule=replace(first.schedule, delivery_sla="PT10M") + ) + foreign = replace( + first, account_id="acct_b", schedule=replace(first.schedule, delivery_sla="PT0S") + ) + await h.store.put_configuration(first) + await h.store.put_configuration(second) + await h.store.put_configuration(foreign) + handler = ReportingStatusHandler(h.store) + caller = ReportingStatusCaller("acct_a", "buyer") + captured = await h.store.read_status_snapshot(account_id="acct_a") + expected = hour(1) + timedelta(minutes=10) + original = handler.render_snapshot({}, caller=caller, snapshot=captured) + assert original["next_expected_at"] == expected.isoformat().replace("+00:00", "Z") + await h.store.put_configuration(replace(second, deactivated_at=hour(0))) + h.clock.now = hour(5) + assert handler.render_snapshot({}, caller=caller, snapshot=captured) == original + for filters in ( + {"delivery_config_ids": ["absent"]}, + {"feed_purposes": ["billing"]}, + {"media_buy_ids": ["foreign-buy"]}, + {"period": {"start": hour(-2).isoformat(), "end": hour(0).isoformat()}}, + ): + assert "next_expected_at" not in handler.render_snapshot( + filters, caller=caller, snapshot=captured + ) + + +@pytest.mark.parametrize( + "date,hours", [("2026-03-08T05:00:00+00:00", 23), ("2026-11-01T04:00:00+00:00", 25)] +) +def test_civil_days_across_dst_preserve_the_captured_timezone_and_sla(date, hours): + start = datetime.fromisoformat(date) + config = replace( + configuration(), + activated_at=start, + deactivated_at=start + timedelta(days=3), + account_timezone="America/New_York", + schedule=ReportingScheduleSpec("P1D", "PT1H", "account_timezone", period_anchor=start), + ) + first = next(committed_periods(config)) + assert first.start == start + assert first.end - first.start == timedelta(hours=hours) + assert ( + next_reporting_expectation((config,), (), as_of=start + timedelta(hours=1)) + == first.expected_at + ) + assert next_reporting_expectation((config,), (), as_of=first.expected_at) > first.expected_at + + +@pytest.mark.parametrize("duration", ["PT1H", "PT10M"]) +def test_nonexistent_civil_slots_are_skipped_without_duplicate_or_reordered_periods(duration): + start = datetime(2026, 3, 8, 5, tzinfo=timezone.utc) + config = replace( + configuration(), + activated_at=start, + deactivated_at=start + timedelta(hours=5), + schedule=ReportingScheduleSpec( + duration, "PT0S", "custom_timezone", "America/New_York", start + ), + ) + periods = list(committed_periods(config)) + assert all(a.end == b.start for a, b in zip(periods, periods[1:])) + assert len({p.period_key for p in periods}) == len(periods) + for period in periods: + near = period.start + (period.end - period.start) / 2 + assert next_reporting_expectation((config,), (), as_of=near) == period.expected_at + + +def test_empty_scope_and_explicit_anchor_before_or_after_activation(): + assert next_reporting_expectation((), (), as_of=START) is None + config = replace( + configuration(), + deactivated_at=None, + schedule=replace(configuration().schedule, period_anchor=hour(10)), + ) + assert [p.start for p in islice(committed_periods(config), 2)] == [START, hour(1)] + offset = START.astimezone(timezone(timedelta(hours=5, minutes=30))) + assert next_reporting_expectation((config,), (), as_of=offset) == hour(2) diff --git a/tests/conformance/reporting/test_reporting_projection_timestamps.py b/tests/conformance/reporting/test_reporting_projection_timestamps.py new file mode 100644 index 000000000..7b1918d48 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_timestamps.py @@ -0,0 +1,114 @@ +"""PostgreSQL JSON timestamps retain their exact instant on the Python floor.""" + +from copy import deepcopy +from dataclasses import replace +from datetime import datetime, timedelta, timezone + +import pytest + +from adcp.reporting._timestamp import aware_timestamp +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.ledger.notification_models import ReportingNotificationError +from adcp.reporting.ledger.status_snapshot import snapshot_from_storage +from adcp.reporting.projection.capture import decode_projection_input +from adcp.validation.schema_loader import get_named_validator + +from ._generation_support import configuration +from ._projection_support import projection_harness + + +@pytest.mark.parametrize("microsecond", [1, 100000, 120000, 123000, 123400, 123450, 123456]) +@pytest.mark.parametrize("offset", ["+00:00:00", "-00:00:00", "+05:45:03", "-02:30:01"]) +def test_fractional_offsets_preserve_the_exact_instant(microsecond, offset): + hours, minutes, seconds = map(int, offset[1:].split(":")) + delta = timedelta(hours=hours, minutes=minutes, seconds=seconds, microseconds=microsecond) + if offset[0] == "-": + delta = -delta + value = "2026-09-01T12:00:00.12345" + offset + "." + f"{microsecond:06d}".rstrip("0") + expected = datetime(2026, 9, 1, 12, 0, 0, 123450, tzinfo=timezone.utc) - delta + assert aware_timestamp(value) == expected + + +@pytest.mark.parametrize("fraction", ["", ".1", ".12345", ".123456"]) +def test_utc_z_is_aware_and_lossless(fraction): + microsecond = int(fraction.lstrip(".").ljust(6, "0")) + assert aware_timestamp("2026-09-01T12:00:00" + fraction + "Z") == datetime( + 2026, 9, 1, 12, 0, 0, microsecond, tzinfo=timezone.utc + ) + + +@pytest.mark.parametrize("microsecond", [0, 100000, 120000, 123000, 123400, 123450, 123456]) +@pytest.mark.parametrize("zone", ["UTC", "Asia/Kathmandu", "America/St_Johns", "Europe/Paris"]) +async def test_actual_pg_json_precision_and_offset_survive_captured_decode(microsecond, zone): + # Paris before 1911 also exercises a real seconds-bearing historical offset. + at = datetime(1890 if zone == "Europe/Paris" else 2026, 9, 1, 12, tzinfo=timezone.utc) + at = at.replace(microsecond=microsecond) + async with projection_harness("postgres") as h: + config = configuration() + config = replace( + config, + schedule=replace(config.schedule, period_anchor=at - timedelta(days=1)), + activated_at=at - timedelta(days=1), + deactivated_at=at + timedelta(days=1), + ) + await h.store.put_configuration(config) + async with h.pool.connection() as c: + await c.execute("SELECT set_config('TimeZone',%s,false)", (zone,)) + row = await ( + await c.execute( + "SELECT reporting_projection_document(%s,%s), to_jsonb(%s::timestamptz)", + (config.account_id, at, at), + ) + ).fetchone() + document, timestamp = row + assert document["as_of"] == document["core"]["as_of"] == timestamp + fraction = timestamp.split("T", 1)[1].split("+", 1)[0].split("-", 1)[0] + fraction = fraction.partition(".")[2] + assert fraction == (f"{microsecond:06d}".rstrip("0") if microsecond else "") + raw = canonical_json_utf8_v1(document) + core = snapshot_from_storage(document["core"]) + decoded = decode_projection_input(document) + assert core == decoded.core + assert core.as_of == at and core.as_of.microsecond == microsecond + assert core.configurations == (config,) + assert decoded.document == raw == canonical_json_utf8_v1(document) + # Exercise the actual PostgreSQL representation through the public + # named-schema path as well. RFC 3339 excludes the seconds-bearing + # historical Paris offset that the private lossless decoder permits. + validator = get_named_validator( + "core/reporting-delivery-config-state.json", version="3.2.0-rc.3" + ) + assert validator is not None + field = validator.evolve(schema=validator.schema["properties"]["activated_at"]) + assert field.is_valid(timestamp) is (zone != "Europe/Paris") + assert canonical_json_utf8_v1(document) == raw + + +@pytest.mark.parametrize( + "timestamp", + [ + "2026-09-01T12:00:00.12345", + "2026-09-01T12:00:00", + "2026-09-01T12:00:00.1234567+00:00", + "2026-09-01T12:00:00.12x45+00:00", + "2026-09-01T25:00:00.12345+00:00", + "2026-02-30T12:00:00.12345+00:00", + "2026-09-01T12:00:00.12345+25:00", + "2026-09-01T12:00:00.12345+01:60", + "2026-09-01T12:00:00.12345+00:09:60", + "not-a-timestamp", + ], +) +async def test_sql_boundary_rejects_invalid_naive_or_precision_losing_timestamps(timestamp): + async with projection_harness("postgres") as h: + async with h.pool.connection() as c: + row = await ( + await c.execute( + "SELECT reporting_projection_document('acct_a',%s)", + (datetime(2026, 9, 1, tzinfo=timezone.utc),), + ) + ).fetchone() + document = deepcopy(row[0]) + document["as_of"] = document["core"]["as_of"] = timestamp + with pytest.raises(ReportingNotificationError, match="status_projection_history_corrupt"): + decode_projection_input(document) diff --git a/tests/conformance/reporting/test_reporting_projection_wire.py b/tests/conformance/reporting/test_reporting_projection_wire.py new file mode 100644 index 000000000..37c7ade4d --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_wire.py @@ -0,0 +1,77 @@ +"""Authenticated tier reads use the same captured private inputs in every view.""" + +import json +from functools import partial + +import pytest + +from adcp.reporting.ownership import page_revision_ownership + +from ._feed_support import MountedFeed, feed_request, second_consumer +from ._projection_support import projection_harness +from ._receipt_support import receipt_case +from ._receipt_transport import error_code + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("feedback", [False, True]) +async def test_mounted_private_summary_revision_and_page_walk(backend, notifications, feedback): + async with projection_harness(backend, notifications=notifications, feedback=feedback) as h: + first = await receipt_case(h) + second = await second_consumer(h, first, "https://buyer.example/second") + await h.projection.activate(account_id=first.obligation.account_id) + mounted = MountedFeed(h, feedback=feedback, hydrated=True, registry_kind="oauth") + mounted.authorize(first) + mounted.authorize(second, token="token-two") + async with mounted.client() as client: + for call in (mounted.mcp, mounted.a2a, partial(mounted.a2a, v1=True)): + summaries = [] + for item, token, receipt_count in ( + (first, "token-one", 0), + (second, "token-two", 1), + ): + request = feed_request(item, view="summary") + del request["pagination"] + _, summary = await call(client, request, token=token) + assert summary.get("status") == "completed", summary + summaries.append(summary) + request.update( + view="revision", + reporting_revision_id=item.revision.reporting_revision_id, + ) + _, exact = await call(client, request, token=token) + assert exact.get("status") == "completed", exact + assert len(exact["receipts"]) == receipt_count + assert len(exact["materializations"]) == 1 + assert page_revision_ownership(exact) == { + item.revision.reporting_revision_id: item.obligation.reporting_obligation_id + } + assert exact["revision"]["revision_content_sha256"] == ( + item.revision.revision_content_sha256 + ) + records = [] + request = feed_request(item) + for _ in range(20): + _, page = await call(client, request, token=token) + assert page.get("status") == "completed", page + page_revision_ownership(page) + records.extend(page["receipts"]) + if not page["pagination"]["has_more"]: + break + request["pagination"]["cursor"] = page["pagination"]["cursor"] + else: + pytest.fail("bounded mounted walk did not finish") + assert len(records) == receipt_count + assert summaries[0]["ledger_snapshot_id"] != summaries[1]["ledger_snapshot_id"] + assert summaries[0]["health"] != summaries[1]["health"] + assert second.binding.consumer_id not in json.dumps(summaries[0]) + mounted.grants.remove((first.obligation.account_id, first.binding.consumer_id)) + for view in ("summary", "revision", "periods"): + request = feed_request(first, view=view) + if view != "periods": + del request["pagination"] + if view == "revision": + request["reporting_revision_id"] = first.revision.reporting_revision_id + _, denied = await mounted.mcp(client, request) + assert error_code(denied) == "UNAUTHORIZED" diff --git a/tests/conformance/reporting/test_reporting_schedule_schema.py b/tests/conformance/reporting/test_reporting_schedule_schema.py new file mode 100644 index 000000000..849bc1602 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_schedule_schema.py @@ -0,0 +1,251 @@ +"""#1179: execute the exact cached rejection and the version-scoped SDK correction. + +This is Python evidence, not approval of another SDK or a cross-language pin. +Every designated blocking compatible lane must independently succeed with this +scenario; an unsupported-schema outcome is only valid in an unsupported lane. +""" + +import hashlib +import json +from copy import deepcopy +from dataclasses import replace +from datetime import timedelta +from types import SimpleNamespace + +import pytest +from jsonschema import Draft7Validator, FormatChecker +from jsonschema.validators import validator_for + +from adcp.reporting.ledger import InMemoryReportingLedgerStore +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.types import GetReportingStatusResponse +from adcp.validation import schema_loader + +from ._feed_support import MountedFeed, feed_harness +from ._generation_support import START, configuration +from ._projection_support import projection_harness + +PIN = "3.2.0-rc.3" +RULE = "/allOf/2/then/not" +CACHED = ( + ( + "media-buy/get-reporting-status-response.json", + 30561, + "498774fa2a15ce1487183d3df4f982436936427d104879279524ed5d38d8e726", + ), + ( + "bundled/media-buy/get-reporting-status-response.json", + 314212, + "578ea8233c4022c528dee3b53bcb88884b36033c71d77a0eb00bee4bc67c72e3", + ), + ( + "mcp/2026-07-28/profiles/production/media-buy/get-reporting-status-response.json", + 171397, + "6b68df71ef16d3317a20e2b4f23f4c385f1f9bbb36bddc80515c719e56868b74", + ), +) + + +def formats(): + checker = FormatChecker() + checker.checks("date-time")(schema_loader._is_rfc3339_date_time) + return checker + + +async def deterministic_summary(): + store = InMemoryReportingLedgerStore(clock=lambda: START + timedelta(minutes=30)) + await store.put_configuration(replace(configuration(), deactivated_at=None)) + return await ReportingStatusHandler(store).handle( + {}, caller=ReportingStatusCaller("acct_a", "https://buyer.example.test/agent") + ) + + +def assert_original_rejection(raw, relative=CACHED[0][0]): + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None + if relative.startswith("mcp/"): + validator = validator_for(validator.schema)(validator.schema, format_checker=formats()) + without_expectation = {k: v for k, v in raw.items() if k != "next_expected_at"} + validator.validate(without_expectation) + errors = list(validator.iter_errors(raw)) + assert len(errors) == 1, errors + error = errors[0] + assert error.validator == "not" + assert list(error.absolute_schema_path) == ["allOf", 2, "then", "not"] + return { + "validator": error.validator, + "schema_pointer": RULE, + "instance_path": list(error.absolute_path), + "message": error.message, + } + + +def invalid_summaries(raw): + for field in ("scope_closed", "coverage_complete"): + for value in (False, None): + invalid = deepcopy(raw) + if value is None: + del invalid["scope"][field] + else: + invalid["scope"][field] = value + yield invalid + for value in (None, 5, "2026-09-01", "2026-09-01T02:00:00", "not-a-time"): + yield {**raw, "next_expected_at": value} + yield {**raw, "health": "pretend-complete"} + yield {**raw, "status": "pretend-completed"} + yield {**raw, "view": "periods"} # summary cannot evade the periods requirements + invalid = deepcopy(raw) + invalid["obligation_counts"]["total"] = "0" + yield invalid + + +@pytest.mark.parametrize("relative,expected_bytes,expected_sha256", CACHED) +async def test_exact_unmodified_cached_rule_rejects_the_required_payload( + relative, expected_bytes, expected_sha256 +): + root = schema_loader._resolve_schema_root(PIN) + assert root is not None + file = root.root / relative + original = file.read_bytes() + assert len(original) == expected_bytes + assert hashlib.sha256(original).hexdigest() == expected_sha256 + schema = json.loads(original) + assert schema["$schema"] == ( + "https://json-schema.org/draft/2020-12/schema" + if relative.startswith("mcp/") + else "http://json-schema.org/draft-07/schema#" + ) + validator = schema_loader.get_named_validator(relative, version=PIN) + assert validator is not None and validator.schema == schema + raw = await deterministic_summary() + assert raw["health"] == "complete" + assert raw["scope"]["scope_closed"] is raw["scope"]["coverage_complete"] is True + assert raw["next_expected_at"] == "2026-09-01T02:00:00Z" + rejection = assert_original_rejection(raw, relative) + effective = schema_loader._effective_task_schema( + schema, "get_reporting_status", "sync", bundle_key=PIN + ) + reconstructed = deepcopy(effective) + reconstructed["allOf"][2]["then"]["not"] = {"required": ["next_expected_at"]} + assert reconstructed == schema # only this pointer differs; all guards survive + assert file.read_bytes() == original + print( + json.dumps( + { + "cached_rule_reproduction": { + "schema_file": str(file), + "schema_uri": f"https://adcontextprotocol.org/schemas/{PIN}/{relative}", + "version": PIN, + "bytes": len(original), + "sha256": expected_sha256, + "dialect": schema["$schema"], + "payload": raw, + "exact_rejection": rejection, + "effective_correction": {"removed_pointer": RULE, "other_changes": []}, + } + }, + sort_keys=True, + ) + ) + + +async def test_effective_validation_advertisement_and_remaining_constraints(): + raw = await deterministic_summary() + GetReportingStatusResponse.model_validate(raw) + validator = schema_loader.get_validator("get_reporting_status", "sync", version=PIN) + assert validator is not None + validators = [validator] + for load in ( + schema_loader.get_schema, + schema_loader.get_portable_schema, + schema_loader.get_mcp_schema, + ): + schema = load("get_reporting_status", "sync", version=PIN) + assert schema is not None + validators.append(validator_for(schema)(deepcopy(schema), format_checker=formats())) + saved = deepcopy(schema) + schema.clear() + assert load("get_reporting_status", "sync", version=PIN) == saved + for validator in validators: + validator.validate(raw) + for invalid in invalid_summaries(raw): + assert not validator.is_valid(invalid), invalid + + +@pytest.mark.parametrize("mutation", ["other-pin", "changed-if", "changed-then", "moved-rule"]) +def test_correction_is_limited_to_the_known_rule_and_version(mutation): + schema = schema_loader.get_named_schema_document(CACHED[0][0], version=PIN) + assert schema is not None + version = PIN + if mutation == "other-pin": + version = "3.2.0-beta.4" + elif mutation == "changed-if": + schema["allOf"][2]["if"]["properties"]["health"]["const"] = "healthy" + elif mutation == "changed-then": + schema["allOf"][2]["then"]["required"] = ["unreviewed-condition"] + else: + schema["allOf"].insert(0, {}) + original = deepcopy(schema) + assert ( + schema_loader._effective_task_schema( + schema, "get_reporting_status", "sync", bundle_key=version + ) + == original + ) + assert schema == original + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("mode", ["core", "projection"]) +@pytest.mark.parametrize("notifications", [False, True]) +@pytest.mark.parametrize("version", [None, PIN]) +async def test_complete_future_expectation_on_actual_summary_mounts( + backend, mode, notifications, version +): + factory = feed_harness if mode == "core" else projection_harness + async with factory(backend, notifications=notifications) as h: + h.clock.now = START + timedelta(minutes=30) + h.store._clock = h.clock # supported deterministic store clock, including PG captures + config = replace(configuration(), deactivated_at=None) + await h.store.put_configuration(config) + if mode == "projection": + await h.projection.activate(account_id=config.account_id) + mounted = MountedFeed(h, version=version, hydrated=True, registry_kind="oauth") + identity = SimpleNamespace( + obligation=config, + binding=SimpleNamespace(consumer_id="https://buyer.example.test/agent"), + ) + mounted.authorize(identity) + request = { + "adcp_version": "3.2-rc.3", + "account": {"account_id": config.account_id}, + "view": "summary", + } + async with mounted.client() as client: + _, inventory = await mounted.mcp(client, inventory=True) + output = next( + t["outputSchema"] for t in inventory["tools"] if t["name"] == "get_reporting_status" + ) + advertised = Draft7Validator(output, format_checker=formats()) + for path in ("/.well-known/agent.json", "/.well-known/agent-card.json"): + card = await client.get(path) + assert card.status_code == 200 + assert "get_reporting_status" in {s["id"] for s in card.json()["skills"]} + results = [] + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + if transport == "mcp": + _, raw = await mounted.mcp(client, request) + else: + _, raw = await mounted.a2a(client, request, v1=transport == "a2a-1.0") + assert raw.get("health") == "complete", raw + assert raw["next_expected_at"] == "2026-09-01T02:00:00Z" + assert_original_rejection(raw) + schema_loader.get_validator("get_reporting_status", "sync", version=PIN).validate( + raw + ) + advertised.validate(raw) + GetReportingStatusResponse.model_validate(raw) + for invalid in invalid_summaries(raw): + assert not advertised.is_valid(invalid), invalid + results.append(raw) + assert results[0] == results[1] == results[2] diff --git a/tests/conformance/reporting/test_reporting_tier_projection.py b/tests/conformance/reporting/test_reporting_tier_projection.py new file mode 100644 index 000000000..19d58831c --- /dev/null +++ b/tests/conformance/reporting/test_reporting_tier_projection.py @@ -0,0 +1,152 @@ +"""One captured tier projection over the real memory/PostgreSQL financial graph.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger import ReportingMaterializationCheck +from adcp.reporting.ledger.notification_models import ReportingStatusScope +from adcp.reporting.ledger.reconciliation_projection import project_reconciliation +from adcp.reporting.ledger.status_projection import StatusProjectionInput, project_status_scope +from adcp.reporting.outbox.status import advance_checkpoint + +from ._generation_support import END +from ._receipt_support import adjustment_for, receipt_case, receipts, request_for + +__all__ = ["receipts"] + + +async def captured(h, s, *, feedback=False): + core = await h.store.read_status_snapshot(account_id=s.obligation.account_id) + private = await h.store.read_reconciliation_snapshot(caller=s.binding.principal) + core = replace(core, as_of=h.clock.now) + scope = ReportingStatusScope( + core.account_id, + s.binding.generation_key, + consumer_id=s.binding.consumer_id, + reporting_obligation_id=s.obligation.reporting_obligation_id, + feed_purpose=s.obligation.feed_purpose, + ) + return StatusProjectionInput( + core, scope, reconciliation=private.records, consumer_status_enabled=feedback + ) + + +def projected(value): + result = project_status_scope(value) + assert len(result.obligations) == 1 + tier = result.obligations[0].reconciliation + assert tier is not None + return result, tier + + +async def test_receipts_adjustments_and_reversible_health_keep_each_ordered_generation(receipts): + h = receipts + s = await receipt_case(h) + first = await captured(h, s) + result, tier = projected(first) + assert tier.wire["reconciliation_status"] == "pending" + assert tier.wire["successful_materialization_count"] == 1 + previous, _ = advance_checkpoint(None, result, fired_at=h.clock.now, source_sequence=1) + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + accepted = await captured(h, s) + result, tier = projected(accepted) + assert tier.satisfied and tier.wire["reconciliation_status"] == "accepted" + previous, event = advance_checkpoint(previous, result, fired_at=h.clock.now, source_sequence=2) + assert event is not None and previous.generation == 2 + item = await adjustment_for(h, s) + pending_adjustment = await captured(h, s) + result, tier = projected(pending_adjustment) + assert not tier.satisfied and tier.wire["pending_adjustment_count"] == 1 + previous, event = advance_checkpoint(previous, result, fired_at=h.clock.now, source_sequence=3) + assert event is not None and previous.generation == 3 + await h.store.ingest_receipt_batch( + { + "account": {"account_id": s.obligation.account_id}, + "idempotency_key": "adjustment-batch-0001", + "adjustment_receipts": [item], + }, + caller=s.binding.principal, + ) + completed = await captured(h, s) + result, tier = projected(completed) + assert tier.satisfied and tier.wire["pending_adjustment_count"] == 0 + previous, event = advance_checkpoint(previous, result, fired_at=h.clock.now, source_sequence=4) + assert event is not None and previous.generation == 4 + # Replay every old captured input after later mutable state has changed. + assert projected(first)[1].wire["reconciliation_status"] == "pending" + assert projected(accepted)[1].wire["pending_adjustment_count"] == 0 + assert projected(pending_adjustment)[1].wire["pending_adjustment_count"] == 1 + + +@pytest.mark.parametrize("later", ["corrupt", "expired", "unreadable", "success", "failure"]) +async def test_accepted_artifact_evidence_survives_later_health_and_attempts(receipts, later): + h = receipts + s = await receipt_case(h) + await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) + if later == "corrupt": + await h.store.record_materialization_check( + ReportingMaterializationCheck( + s.attempt.scope, + s.attempt.reporting_materialization_id, + "later-corruption", + "corrupt", + END + timedelta(seconds=20), + ) + ) + elif later == "expired": + h.clock.now = s.outcome.resource.expires_at + elif later == "unreadable": + await h.store.set_revision_readable( + account_id=s.obligation.account_id, + reporting_revision_id=s.revision.reporting_revision_id, + readable=False, + ) + else: + attempt = replace(s.attempt, reporting_materialization_id="materialization-2", attempt=2) + await h.store.commit_materialization_attempt(attempt) + outcome = replace( + s.outcome, + reporting_materialization_id=attempt.reporting_materialization_id, + status=s.outcome.status if later == "success" else "failed", + resource=s.outcome.resource if later == "success" else None, + verification=s.outcome.verification if later == "success" else None, + failure_code=None if later == "success" else "WRITE_FAILED", + ) + await h.store.commit_materialization(outcome) + _, tier = projected(await captured(h, s)) + assert tier.wire["reconciliation_status"] == "accepted" + assert tier.wire["accepted_receipt_count"] == 1 + assert tier.wire["successful_materialization_count"] == (2 if later == "success" else 1) + assert tier.satisfied is (later in {"success", "failure"}) + assert tier.wire[ + "resource_retained_until" + ] == s.delivery.resource_retained_until.isoformat().replace("+00:00", "Z") + + +async def test_official_selection_precedes_available_snapshot_artifacts(receipts): + h = receipts + s = await receipt_case( + h, finality="snapshot", billing=False, reconciliation_mode="delivery_only" + ) + private = await h.store.read_reconciliation_snapshot(caller=s.binding.principal) + official = replace( + s.revision, + reporting_revision_id="unmaterialized-official", + finality="official", + finality_basis="source_final", + finality_policy_id="policy-1", + finalized_at=END, + ) + result = project_reconciliation( + replace(s.obligation, required_finality="official"), + (s.revision, official), + (), + private.records, + consumer_id=s.binding.consumer_id, + as_of=h.clock.now, + ) + assert not result.satisfied + assert result.wire["successful_materialization_count"] == 1 + assert result.wire["reconciliation_status"] == "not_required" diff --git a/tests/fixtures/public_api_snapshot.json b/tests/fixtures/public_api_snapshot.json index b6e016a83..a19e66e16 100644 --- a/tests/fixtures/public_api_snapshot.json +++ b/tests/fixtures/public_api_snapshot.json @@ -1362,6 +1362,8 @@ "ReportPlanOutcomeResponse", "ReportUsageRequest", "ReportUsageResponse", + "ReportingAdjustment", + "ReportingAdjustmentReceipt", "ReportingAuthoritativeParty", "ReportingBucket", "ReportingCanonicalContentDigest", diff --git a/tests/test_reporting_capability_models.py b/tests/test_reporting_capability_models.py new file mode 100644 index 000000000..d8ae20449 --- /dev/null +++ b/tests/test_reporting_capability_models.py @@ -0,0 +1,174 @@ +"""Public reporting promises are nullable, explicit and tier-consistent (#1180).""" + +from __future__ import annotations + +import importlib +import json +import shutil +from pathlib import Path +from typing import get_args + +import pytest +from pydantic import BaseModel, ValidationError + +from adcp.types import GetAdcpCapabilitiesResponse, ReportingDeliveryCapabilities +from tests.test_reporting_ledger import _OFFERING + +PROMISES = { + "receipt_task": "sync_reporting_receipts", + "readiness_notification": "reporting.delivery_ready", + "status_notification": "reporting.status_changed", + "ledger_notification": "reporting.ledger_changed", +} + + +def capability(**fields): + return { + "supported": True, + "offerings": [_OFFERING], + "automated_recovery_window_seconds": 0, + "status_retention_days": 7, + **fields, + } + + +def response(raw): + return { + "status": "completed", + "adcp": { + "major_versions": [3], + "idempotency": {"supported": True, "replay_ttl_seconds": 3600}, + }, + "supported_protocols": ["media_buy"], + "media_buy": {"reporting_delivery": raw}, + } + + +@pytest.fixture(params=["public", "bundled"]) +def graph(request): + if request.param == "public": + return ReportingDeliveryCapabilities, GetAdcpCapabilitiesResponse + # Codegen's self-contained clone is deliberately tested as a separate graph. + module = importlib.import_module( + "adcp.types.generated_poc.bundled.protocol.get_adcp_capabilities_response" + ) + return module.ReportingDelivery, module.GetAdcpCapabilitiesResponse + + +@pytest.mark.parametrize( + "flags", + [ + {}, + {"managed_delivery": False}, + {"reconciled_billing": False}, + {"managed_delivery": False, "reconciled_billing": False}, + ], +) +def test_core_attributes_schema_and_json_round_trip_do_not_synthesize_promises(graph, flags): + model, envelope = graph + value = model.model_validate(capability(**flags)) + fields = set(value.model_fields_set) + schema = model.model_json_schema() + for name in PROMISES: + field = model.model_fields[name] + assert field.default is None + assert type(None) in get_args(field.annotation) + assert getattr(value, name) is None + assert schema["properties"][name]["default"] is None + assert {"type": "null"} in schema["properties"][name]["anyOf"] + for raw in ( + value.model_dump(), + value.model_dump(mode="json"), + json.loads(value.model_dump_json()), + ): + assert not PROMISES.keys() & raw.keys() + assert model.model_validate(raw).model_dump(mode="json") == value.model_dump(mode="json") + restored = model.model_validate_json(value.model_dump_json()) + assert restored.model_dump(mode="json") == value.model_dump(mode="json") + nested = envelope.model_validate(response(capability(**flags))) + for raw in (nested.model_dump(mode="json"), json.loads(nested.model_dump_json())): + assert not PROMISES.keys() & raw["media_buy"]["reporting_delivery"].keys() + restored = envelope.model_validate_json(json.dumps(raw)) + assert restored.media_buy.reporting_delivery.receipt_task is None + assert restored.media_buy.reporting_delivery.readiness_notification is None + assert fields == value.model_fields_set + + +@pytest.mark.parametrize( + "field,tier", + [("readiness_notification", "managed_delivery"), ("receipt_task", "reconciled_billing")], +) +@pytest.mark.parametrize("flag", [None, False]) +def test_inverse_tier_validation_runs_in_both_standalone_and_nested_graphs( + graph, field, tier, flag +): + model, envelope = graph + raw = capability(**{field: PROMISES[field], **({tier: flag} if flag is not None else {})}) + with pytest.raises(ValidationError, match=f"{field} requires {tier}"): + model.model_validate(raw) + with pytest.raises(ValidationError, match=f"{field} requires {tier}"): + envelope.model_validate(response(raw)) + + +@pytest.mark.parametrize("managed", [None, False]) +def test_reconciled_is_cumulative_even_without_a_receipt_task(graph, managed): + model, envelope = graph + raw = capability(reconciled_billing=True, managed_delivery=managed) + for cls, body in ((model, raw), (envelope, response(raw))): + with pytest.raises(ValidationError, match="reconciled_billing requires managed_delivery"): + cls.model_validate(body) + + +@pytest.mark.parametrize("field", PROMISES) +def test_each_explicit_supported_literal_survives_without_implying_other_promises(graph, field): + model, envelope = graph + flags = {"managed_delivery": True} if field == "readiness_notification" else {} + if field == "receipt_task": + flags = {"managed_delivery": True, "reconciled_billing": True} + raw = capability(**flags, **{field: PROMISES[field]}) + for cls, body in ((model, raw), (envelope, response(raw))): + value = cls.model_validate(body) + for output in (value.model_dump(mode="json"), json.loads(value.model_dump_json())): + block = output if cls is model else output["media_buy"]["reporting_delivery"] + assert {key: block[key] for key in PROMISES if key in block} == {field: PROMISES[field]} + with pytest.raises(ValidationError): + cls.model_validate( + {**raw, field: "unsupported"} + if cls is model + else response({**raw, field: "unsupported"}) + ) + + +def test_subclasses_inside_a_non_sdk_parent_omit_absent_promises(): + class Reporting(ReportingDeliveryCapabilities): + pass + + class OrdinaryParent(BaseModel): + reporting: Reporting + + value = OrdinaryParent.model_validate({"reporting": capability()}) + assert all(getattr(value.reporting, name) is None for name in PROMISES) + for raw in (value.model_dump(), json.loads(value.model_dump_json())): + assert not PROMISES.keys() & raw["reporting"].keys() + value = OrdinaryParent.model_validate( + {"reporting": capability(status_notification=PROMISES["status_notification"])} + ) + assert value.model_dump()["reporting"]["status_notification"] == PROMISES["status_notification"] + + +def test_post_generation_repair_is_idempotent_for_both_actual_model_layouts(tmp_path, monkeypatch): + from scripts import post_generate_fixes + + root = Path(__file__).parents[1] / "src/adcp/types/generated_poc" + targets = ( + "core/reporting_delivery_capabilities.py", + "bundled/protocol/get_adcp_capabilities_response.py", + ) + for relative in targets: + target = tmp_path / relative + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(root / relative, target) + monkeypatch.setattr(post_generate_fixes, "OUTPUT_DIR", tmp_path) + before = [(tmp_path / name).read_bytes() for name in targets] + post_generate_fixes.fix_reporting_capability_defaults() + assert before == [(tmp_path / name).read_bytes() for name in targets] diff --git a/tests/test_reporting_production_public.py b/tests/test_reporting_production_public.py new file mode 100644 index 000000000..87322a1b3 --- /dev/null +++ b/tests/test_reporting_production_public.py @@ -0,0 +1,50 @@ +"""All public exports work without optional drivers; concrete PG classes are lazy.""" + +import subprocess +import sys + +import pytest + + +@pytest.mark.parametrize("module", ["adcp.reporting.production", "adcp.reporting.projection"]) +@pytest.mark.parametrize("drivers", [False, True]) +def test_public_exports_and_optional_database_drivers(module, drivers): + script = """ +import importlib, sys +drivers = sys.argv[2] == 'True' +if not drivers: + class NoDrivers: + def find_spec(self, fullname, path=None, target=None): + if fullname.split('.')[0] in {'psycopg', 'psycopg_pool'}: + raise ModuleNotFoundError(fullname) + sys.meta_path.insert(0, NoDrivers()) +module = importlib.import_module(sys.argv[1]) +assert len(module.__all__) == len(set(module.__all__)) +assert module.__name__ + '.pg' not in sys.modules +for name in module.__all__: + assert getattr(module, name) is not None, name +if not drivers: + assert 'psycopg' not in sys.modules + assert 'psycopg_pool' not in sys.modules + name = ('PgReportingProductionStore' if module.__name__.endswith('production') + else 'PgReportingProjectionStore') + try: + getattr(module, name)(pool=object()) + except ImportError as error: + assert 'pg' in str(error) + else: + raise AssertionError('PG store constructed without its driver') +try: + getattr(module, 'NoSuchReportingExport') +except AttributeError: + pass +else: + raise AssertionError('unknown public export was accepted') +""" + result = subprocess.run( + [sys.executable, "-I", "-c", script, module, str(drivers)], + capture_output=True, + text=True, + timeout=60, + ) + assert result.returncode == 0, result.stdout + result.stderr diff --git a/tests/test_reporting_revision_ownership.py b/tests/test_reporting_revision_ownership.py new file mode 100644 index 000000000..5ab6699f9 --- /dev/null +++ b/tests/test_reporting_revision_ownership.py @@ -0,0 +1,243 @@ +"""Ownership is checked across the whole public, bounded periods walk.""" + +from copy import deepcopy +from datetime import datetime, timezone + +import pytest + +from adcp.reporting import ( + ReportingReconciliationError, + evaluate_reporting_ledger, + load_reporting_ledger, +) +from adcp.reporting.ownership import ( + ReportingOwnershipError, + page_revision_ownership, + with_revision_ownership, +) +from adcp.types import GetReportingStatusRequest, GetReportingStatusResponse +from adcp.types.core import TaskResult, TaskStatus +from tests.test_reporting_reconciliation import REVISION, _obligation, _response + +ARRAYS = ( + "periods", + "revisions", + "materializations", + "receipts", + "adjustments", + "adjustment_receipts", +) +OWNER = "obligation-billing" +REVISION_ID = REVISION["reporting_revision_id"] + + +class Pages: + def __init__(self, pages): + self.pages, self.calls = pages, 0 + + async def get_reporting_status(self, request): + page = self.pages[self.calls % len(self.pages)] + self.calls += 1 + return TaskResult( + status=TaskStatus.COMPLETED, + data=GetReportingStatusResponse.model_validate(deepcopy(page)), + ) + + +def pages(raw=None): + raw = deepcopy(raw or _response()) + owners = {REVISION_ID: OWNER, "revision-other": "obligation-other"} + records = [(name, record) for name in ARRAYS for record in raw.get(name, [])] + result = [] + for index, (name, record) in enumerate(records): + page = {k: v for k, v in raw.items() if k not in ARRAYS} + page.update({a: [] for a in ARRAYS}) + page[name] = [record] + page["pagination"] = {"has_more": index + 1 < len(records), "total_count": len(records)} + if page["pagination"]["has_more"]: + page["pagination"]["cursor"] = f"cursor-{index}" + page["changes_checkpoint"] = "constant-checkpoint" + result.append(with_revision_ownership(page, owners)) + return result + + +async def load(values, **bounds): + return await load_reporting_ledger( + Pages(values), + GetReportingStatusRequest.model_validate( + {"account": {"account_id": "account-1"}, "view": "periods"} + ), + **bounds, + ) + + +def test_page_local_merge_preserves_keys_and_input_and_explicit_empty_mode(): + raw = { + "revisions": [{"reporting_revision_id": "r"}], + "ext": {"vendor": {"a": 1}, "adcp": {"other": "retained"}}, + } + original = deepcopy(raw) + result = with_revision_ownership(raw, {"r": "o", "not-on-page": "o"}) + assert raw == original + assert result["ext"]["vendor"] == {"a": 1} + assert result["ext"]["adcp"]["other"] == "retained" + assert page_revision_ownership(result) == {"r": "o"} + assert with_revision_ownership(result, {"r": "o"}) == result + assert page_revision_ownership(with_revision_ownership({"revisions": []}, {})) == {} + assert page_revision_ownership({"revisions": []}) is None + + +@pytest.mark.parametrize( + "reserved", + [ + None, + [], + "bad", + {}, + {"version": True, "bindings": []}, + {"version": 2, "bindings": []}, + {"version": 1, "bindings": {}}, + {"version": 1, "bindings": [], "unknown": 1}, + { + "version": 1, + "bindings": [{"reporting_revision_id": "r", "reporting_obligation_id": "o"}], + }, + ], +) +def test_malformed_reserved_values_and_extra_page_binding_fail(reserved): + with pytest.raises(ReportingOwnershipError): + page_revision_ownership( + {"revisions": [], "ext": {"adcp": {"reporting_revision_ownership": reserved}}} + ) + + +@pytest.mark.parametrize("ext", [None, [], "bad", {"adcp": None}, {"adcp": []}, {"adcp": "bad"}]) +def test_non_object_reserved_namespaces_are_not_legacy(ext): + with pytest.raises(ReportingOwnershipError): + with_revision_ownership({"revisions": [], "ext": ext}, {}) + + +def test_duplicate_missing_and_conflicting_binding_are_rejected(): + page = with_revision_ownership({"revisions": [{"reporting_revision_id": "r"}]}, {"r": "o"}) + duplicate = deepcopy(page) + bindings = duplicate["ext"]["adcp"]["reporting_revision_ownership"]["bindings"] + bindings.append(deepcopy(bindings[0])) + with pytest.raises(ReportingOwnershipError): + page_revision_ownership(duplicate) + with pytest.raises(ReportingOwnershipError): + with_revision_ownership(page, {"r": "other"}) + with pytest.raises(ReportingOwnershipError): + with_revision_ownership({"revisions": [{"reporting_revision_id": "r"}]}, {}) + + +@pytest.mark.parametrize("name", ["revision", "reporting_revision"]) +def test_exact_view_checks_supplied_binding_without_proving_an_unknown_owner(name): + raw = { + name: {"reporting_revision_id": "r"}, + "reporting_revision_binding": {"reporting_revision_id": "r"}, + } + page = with_revision_ownership(raw, {"r": "owner-needs-periods-walk"}) + assert page_revision_ownership(page) == {"r": "owner-needs-periods-walk"} + for other in {"revision", "reporting_revision", "revisions"} - {name}: + with pytest.raises(ReportingOwnershipError): + page_revision_ownership({**page, other: []}) + for binding in (None, {}, {"reporting_revision_id": "other"}): + with pytest.raises(ReportingOwnershipError): + page_revision_ownership({**page, "reporting_revision_binding": binding}) + + +def test_a2a_struct_numeric_version_preserves_integer_semantics(): + page = with_revision_ownership({"revisions": []}, {}) + reserved = page["ext"]["adcp"]["reporting_revision_ownership"] + reserved["version"] = 1.0 + assert page_revision_ownership(page) == {} + for bad in (True, "1", 1.5, float("nan"), float("inf")): + reserved["version"] = bad + with pytest.raises(ReportingOwnershipError): + page_revision_ownership(page) + + +async def test_page_size_one_dependencies_and_repeated_identical_metadata(): + values = pages() + # Revisions precede their owner and materialization on this wire walk. + values[0]["periods"], values[1]["periods"] = [], values[0]["periods"] + values[0]["revisions"], values[1]["revisions"] = values[1]["revisions"], [] + for i in (0, 1): + values[i].pop("ext") + values[i] = with_revision_ownership(values[i], {REVISION_ID: OWNER}) + repeated = deepcopy(values[0]) + repeated["pagination"]["cursor"] = "repeated-revision" + values.insert(1, repeated) + ledger = await load(values) + assert ledger.revision_ownership == {REVISION_ID: OWNER} + assert len(ledger.revisions) == len(ledger.obligations) == len(ledger.materializations) == 1 + + +@pytest.mark.parametrize( + "mutation", + [ + "mixed", + "unknown-owner", + "foreign-account", + "semantic", + "count", + "missing-revision", + "changed-owner", + ], +) +async def test_full_walk_rejects_inconsistent_ownership(mutation): + values = pages() + if mutation == "mixed": + values[-1].pop("ext") + elif mutation == "unknown-owner": + values[1]["ext"]["adcp"]["reporting_revision_ownership"]["bindings"][0][ + "reporting_obligation_id" + ] = "unknown" + elif mutation == "foreign-account": + values[0]["periods"][0]["account_id"] = "other-account" + elif mutation == "semantic": + values[1]["revisions"][0]["media_buy_ids"] = ["unknown-buy"] + elif mutation == "count": + values[0]["periods"][0]["revision_count"] = 2 + elif mutation == "missing-revision": + values[1]["revisions"] = [] + else: + extra = deepcopy(values[1]) + extra["pagination"]["cursor"] = "extra" + extra["ext"]["adcp"]["reporting_revision_ownership"]["bindings"][0][ + "reporting_obligation_id" + ] = "unknown" + values.insert(2, extra) + with pytest.raises(ReportingReconciliationError, match="ownership"): + await load(values) + + +async def test_explicit_ownership_separates_identical_scopes_and_legacy_stays_conservative(): + raw = _response() + raw["periods"].append(_obligation("obligation-other")) + raw["revisions"].append({**deepcopy(REVISION), "reporting_revision_id": "revision-other"}) + raw["periods"][1].update( + destination_ref=None, + materialization_count=None, + successful_materialization_count=None, + reconciliation_mode="delivery_only", + reconciliation_status="not_required", + health="complete", + ) + owned = await load(pages(raw)) + result = evaluate_reporting_ledger(owned, now=datetime(2026, 9, 3, tzinfo=timezone.utc)) + assert result.obligations[1].reporting_revision_id == "revision-other" + assert result.obligations[1].definitive + legacy_pages = pages(raw) + for page in legacy_pages: + page.pop("ext") + legacy = await load(legacy_pages) + assert legacy.revision_ownership is None + assert not evaluate_reporting_ledger(legacy).obligations[1].definitive + + +@pytest.mark.parametrize("bounds", [{"max_pages": 1}, {"max_records": 1}]) +async def test_walk_budgets_are_enforced(bounds): + with pytest.raises(ReportingReconciliationError) as error: + await load(pages(), **bounds) + assert error.value.code == "LEDGER_LIMIT_EXCEEDED" diff --git a/tests/test_schema_datetime_formats.py b/tests/test_schema_datetime_formats.py new file mode 100644 index 000000000..e2d7f333a --- /dev/null +++ b/tests/test_schema_datetime_formats.py @@ -0,0 +1,136 @@ +"""Public date-time format validation is independent of Python parser precision. + +These exercise the registered named and task validators on the actual cached +schemas. Persisted reporting timestamps have a separate, lossless decoder; +its microsecond precision and historical offset rules are not this contract. +""" + +from copy import deepcopy + +import pytest + +from adcp.validation.schema_loader import get_named_validator, get_validator +from adcp.validation.schema_validator import validate_request + +PIN = "3.2.0-rc.3" +FRACTIONS = ("", ".1", ".12", ".123", ".1234", ".12345", ".123456", ".123456789012") +OFFSETS = ("Z", "z", "+00:00", "-00:00", "+05:45", "-03:30", "+23:59") +VALID = tuple( + "2026-04-01T12:00:00" + fraction + offset for fraction in FRACTIONS for offset in OFFSETS +) + ( + "2024-02-29t12:00:00.00001z", + "2000-02-29T12:00:00Z", + "0001-01-01T00:00:00Z", + "9999-12-31T23:59:59.999999999Z", +) +INVALID = ( + "2026-02-29T12:00:00Z", + "1900-02-29T12:00:00Z", + "2026-02-30T12:00:00.12345Z", + "0000-01-01T00:00:00Z", + "2026-00-01T00:00:00Z", + "2026-13-01T00:00:00Z", + "2026-04-00T00:00:00Z", + "2026-04-31T00:00:00Z", + "2026-04-01T12:00:00.12345", + "2026-04-01T12:00:00", + "2026-04-01", + "2026-04-01T12:00:00.Z", + "2026-04-01T12:00:00,12345Z", + "2026-04-01T24:00:00Z", + "2026-04-01T12:60:00Z", + # Keep the existing checker's seconds 00..59 boundary; this correction + # does not commission leap-second support or new offset syntax. + "2016-12-31T23:59:60Z", + "2026-04-01T12:00:00+24:00", + "2026-04-01T12:00:00+05:60", + "2026-04-01T12:00:00+05:45:03", + "2026-04-01T12:00:00+0545", + "2026-04-01 12:00:00Z", + "20260401T120000Z", + "2026-W14-3T12:00:00Z", + "2026-04-01T12:00:00.\u0661Z", + "2026-04-01T1\u0662:00:00Z", + "2026-04-01T12:00:00+0\u0661:00", + "2026-04-01T12:00:00Z\n", + "2026-04-01T12:00:00Z trailing", + "not-a-timestamp", + None, + 5, + True, + {}, +) + + +def reporting_timestamp_field(): + validator = get_named_validator("core/reporting-delivery-config-state.json", version=PIN) + assert validator is not None + field = validator.schema["properties"]["activated_at"] + assert field["type"] == "string" and field["format"] == "date-time" + return validator.evolve(schema=field) + + +def request_with_timestamp(value): + return { + "proposal_id": "format-contract-proposal", + "total_budget": {"amount": 50000, "currency": "USD"}, + "start_time": value, + "end_time": "2030-06-30T23:59:59Z", + "idempotency_key": "format-contract-0001", + "brand": {"domain": "advertiser.example.test"}, + "account": {"account_id": "acct_format"}, + } + + +@pytest.mark.parametrize("value", VALID) +def test_named_reporting_format_accepts_fractional_precision_without_coercion(value): + original = value.encode() + reporting_timestamp_field().validate(value) + assert value.encode() == original + + +@pytest.mark.parametrize("value", INVALID) +def test_named_reporting_format_preserves_invalid_input_rejection(value): + assert not reporting_timestamp_field().is_valid(value) + + +@pytest.mark.parametrize("version", ["3.1", PIN]) +@pytest.mark.parametrize("value", VALID) +def test_public_request_format_preserves_exact_value_and_oneof(version, value): + payload = request_with_timestamp(value) + original = deepcopy(payload) + validator = get_validator("create_media_buy", "request", version=version) + assert validator is not None + validator.validate(payload) + outcome = validate_request("create_media_buy", payload, version=version) + assert outcome.valid, outcome.issues + assert payload == original + + +@pytest.mark.parametrize("version", ["3.1", PIN]) +@pytest.mark.parametrize("value", INVALID) +def test_public_request_format_preserves_invalid_input_rejection(version, value): + payload = request_with_timestamp(value) + original = deepcopy(payload) + outcome = validate_request("create_media_buy", payload, version=version) + assert not outcome.valid + assert any(issue.pointer == "/start_time" for issue in outcome.issues) + assert payload == original + + +@pytest.mark.parametrize("version", ["3.1", PIN]) +def test_public_request_asap_still_selects_only_the_const_branch(version): + payload = request_with_timestamp("asap") + assert validate_request("create_media_buy", payload, version=version).valid + payload["end_time"] = "asap" + assert not validate_request("create_media_buy", payload, version=version).valid + assert not reporting_timestamp_field().is_valid("asap") + + +@pytest.mark.parametrize("value", [None, 5, True, {}, []]) +def test_format_annotation_does_not_impose_a_string_type(value): + # JSON Schema format applies to strings. The actual cached field's type + # constraint, rather than a new format coercion, rejects nonstrings. + field = reporting_timestamp_field() + assert field.evolve(schema={"format": "date-time"}).is_valid(value) + assert not field.is_valid(value) diff --git a/tests/test_schema_loader_per_version.py b/tests/test_schema_loader_per_version.py index 4fb9c55f1..c93fb6be2 100644 --- a/tests/test_schema_loader_per_version.py +++ b/tests/test_schema_loader_per_version.py @@ -161,6 +161,81 @@ def test_root_relative_legacy_refs_resolve_from_offline_registry( assert not invalid.valid +def test_schedule_correction_does_not_change_a_different_version( + synthetic_legacy_bundle: tuple[str, Path], +) -> None: + """A similar legacy rule requires its own explicit compatibility decision.""" + version, root = synthetic_legacy_bundle + pinned = _loader_mod.get_named_schema_document( + "media-buy/get-reporting-status-response.json", version="3.2.0-rc.3" + ) + assert pinned is not None + schema = {"allOf": [{}, {}, pinned["allOf"][2]]} + file = root / "bundled" / "get-reporting-status-response.json" + original = json.dumps(schema).encode() + file.write_bytes(original) + value = { + "health": "complete", + "scope": {"scope_closed": True, "coverage_complete": True}, + "next_expected_at": "2026-10-01T01:00:00Z", + } + validator = get_validator("get_reporting_status", "sync", version=version) + assert validator is not None and not validator.is_valid(value) + for load in ( + _loader_mod.get_schema, + _loader_mod.get_portable_schema, + _loader_mod.get_mcp_schema, + ): + assert load("get_reporting_status", "sync", version=version) == schema + assert file.read_bytes() == original + + +def test_modular_schema_without_id_resolves_its_own_fragments_offline( + synthetic_legacy_bundle: tuple[str, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + """Canonical path refs do not require $id or a network round trip.""" + legacy_key, root = synthetic_legacy_bundle + child = { + "definitions": {"Choice": {"type": "string", "enum": ["producer_managed"]}}, + "type": "object", + "properties": {"orchestration": {"$ref": "#/definitions/Choice"}}, + "required": ["orchestration"], + } + (root / "core" / "method.json").write_text(json.dumps(child), encoding="utf-8") + request = { + "type": "object", + "definitions": {"Choice": {"type": "integer"}}, + "properties": { + "method": { + "$ref": f"https://adcontextprotocol.org/schemas/{legacy_key}/core/method.json" + }, + "count": {"$ref": "#/definitions/Choice"}, + }, + "required": ["method", "count"], + } + (root / "bundled" / "synthetic-tool-request.json").write_text( + json.dumps(request), encoding="utf-8" + ) + + def deny_remote(*args: object, **kwargs: object) -> None: + pytest.fail("validation attempted to retrieve a remote schema") + + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + from jsonschema import RefResolver + + monkeypatch.setattr(RefResolver, "resolve_remote", deny_remote) + valid = {"method": {"orchestration": "producer_managed"}, "count": 1} + assert validate_request("synthetic_tool", valid, version=legacy_key).valid + invalid = {"method": {"orchestration": "consumer_managed"}, "count": 1} + assert not validate_request("synthetic_tool", invalid, version=legacy_key).valid + assert not validate_request( + "synthetic_tool", {**valid, "count": "producer_managed"}, version=legacy_key + ).valid + + def test_get_validator_same_tool_different_versions_compiles_separately( synthetic_legacy_bundle: tuple[str, Path], ) -> None: From e3a44d281d019ebf6aec738c2cfe18f8bba97462 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Sat, 19 Sep 2026 02:23:48 +0000 Subject: [PATCH 2/6] fix(reporting): stop and diagnose failed production workers --- docs/reporting-production.md | 15 + src/adcp/reporting/production/_diagnostics.py | 43 +++ src/adcp/reporting/production/service.py | 20 +- .../reporting/_production_support.py | 3 +- ...est_reporting_production_worker_failure.py | 296 ++++++++++++++++++ 5 files changed, 373 insertions(+), 4 deletions(-) create mode 100644 src/adcp/reporting/production/_diagnostics.py create mode 100644 tests/conformance/reporting/test_reporting_production_worker_failure.py diff --git a/docs/reporting-production.md b/docs/reporting-production.md index 686394a23..acbd34705 100644 --- a/docs/reporting-production.md +++ b/docs/reporting-production.md @@ -266,6 +266,21 @@ migrating. Preserve pending work, immutable journals, snapshots and exact receip batch responses. Restart with the same admitted contracts, complete migration and let the owned bounded workers converge. Inspect typed closed failure codes; provider bodies and credential contexts are not persistence or diagnostic data. +An unexpected owned worker failure latches the composition unready, wakes the +shared stop signal and emits one `ERROR` record on `adcp.reporting.production` +with code `REPORTING_PRODUCTION_WORKER_STOPPED` and a closed boundary label +(`producer`, `materializer`, `projection`, `sweeper` or `notifications`). Route +that logger to the operator's alert sink. Records contain no exception text, +traceback, provider body, request identity or ambient logging context. Expected +`ReportingNotificationError` outcomes and cancellation stop the owned loops +without this unexpected-failure signal. A late in-flight error after an already +requested stop does not create a second alert or turn cancellation into an +unexpected-failure signal. Existing notification guards still +check readiness before sampling and before dispatch; already-reserved work may +finish under its existing transaction and deadline rules. Drain with `aclose()`, +repair the failed component and construct fresh support to recover; the failed +instance never silently restarts or regains its capability claim. A failing log +sink cannot prevent the stop latch or change the safe public error. The [receipt ingress](reporting-receipt-ingress.md), [frozen feed](reporting-frozen-feed.md) and original materializer recovery contracts continue to apply. diff --git a/src/adcp/reporting/production/_diagnostics.py b/src/adcp/reporting/production/_diagnostics.py new file mode 100644 index 000000000..f2c852cf1 --- /dev/null +++ b/src/adcp/reporting/production/_diagnostics.py @@ -0,0 +1,43 @@ +"""Closed operator signals for an unexpectedly stopped production worker.""" + +from __future__ import annotations + +import logging +from typing import Literal + +_WorkerBoundary = Literal[ + "producer", "materializer", "projection", "sweeper", "notifications", "worker" +] +_BOUNDARIES = frozenset( + {"producer", "materializer", "projection", "sweeper", "notifications", "worker"} +) +_LOGGER = logging.getLogger("adcp.reporting.production") + + +def _boundary_label(value: object) -> str: + return value if type(value) is str and value in _BOUNDARIES else "worker" + + +def _worker_stopped(*, boundary: _WorkerBoundary) -> None: + """No exception, provider/request object or ambient context enters the record.""" + if not _LOGGER.isEnabledFor(logging.ERROR): + return + record = logging.LogRecord( + _LOGGER.name, logging.ERROR, "", 0, "Reporting production worker stopped", (), None + ) + # Bypass ambient LogRecordFactory additions. Names may themselves carry + # adopter data, so this diagnostic does not retain them or a traceback. + record.threadName = None + record.processName = None + if hasattr(record, "taskName"): + record.taskName = None + record.__dict__.update( + code="REPORTING_PRODUCTION_WORKER_STOPPED", + boundary=_boundary_label(boundary), + ) + try: + _LOGGER.handle(record) + except Exception: + # The failed/stop latch has already committed. A broken operator sink + # cannot prevent sibling shutdown or replace the closed public error. + return diff --git a/src/adcp/reporting/production/service.py b/src/adcp/reporting/production/service.py index 99ce6aba7..54848a145 100644 --- a/src/adcp/reporting/production/service.py +++ b/src/adcp/reporting/production/service.py @@ -28,6 +28,7 @@ from adcp.reporting.materializer.service import ReportingMaterializerService from adcp.reporting.materializer.verification import ReportingDestinationIO from adcp.reporting.materializer.work import MaterializerContext, verification_key_id +from adcp.reporting.production._diagnostics import _worker_stopped, _WorkerBoundary from adcp.reporting.production.configuration import ( ReportingConfigurationAdmission, ReportingProductionConfigurationTask, @@ -568,18 +569,23 @@ async def start(self) -> None: self._assert_components() async def _run(self) -> None: + boundary: _WorkerBoundary = "producer" try: while not self._stop.is_set(): # Each producer leases a generation through the reviewed fair # indexed acquisition path; no account enumeration is required. for producer in dict.fromkeys(o.producer for o in self.offerings): + boundary = "producer" token = self._producer_turn.set(producer) try: await producer.run_worker() finally: self._producer_turn.reset(token) + boundary = "materializer" await self.materializer.run_once() + boundary = "projection" await self.projection.rebuild_one() + boundary = "sweeper" await self.projection.sweep_one() self._started.set() try: @@ -587,10 +593,14 @@ async def _run(self) -> None: except asyncio.TimeoutError: pass except asyncio.CancelledError: + self._stop.set() raise - except Exception: - # Retain only the closed lifecycle state, never provider bodies. + except Exception as error: + already_stopping = self._stop.is_set() self._failed = True + self._stop.set() + if not already_stopping and not isinstance(error, ReportingNotificationError): + _worker_stopped(boundary=boundary) finally: self._started.set() @@ -606,10 +616,14 @@ async def _run_notifications(self) -> None: except asyncio.TimeoutError: pass except asyncio.CancelledError: + self._stop.set() raise - except Exception: + except Exception as error: + already_stopping = self._stop.is_set() self._failed = True self._stop.set() + if not already_stopping and not isinstance(error, ReportingNotificationError): + _worker_stopped(boundary="notifications") finally: self._notification_started.set() diff --git a/tests/conformance/reporting/_production_support.py b/tests/conformance/reporting/_production_support.py index 161d4231b..192a6bfd4 100644 --- a/tests/conformance/reporting/_production_support.py +++ b/tests/conformance/reporting/_production_support.py @@ -371,6 +371,7 @@ async def production_harness( reconciled=False, feedback=False, identity_prefix="", + poll_seconds=60, ): from contextlib import AsyncExitStack @@ -625,7 +626,7 @@ async def authorize(account, context, consumer): ), resolve_account=authorize, notification_workers=workers, - poll_seconds=60, + poll_seconds=poll_seconds, ) h.production, h.projection, h.item = support, projection, item h.source_clock = source_clock diff --git a/tests/conformance/reporting/test_reporting_production_worker_failure.py b/tests/conformance/reporting/test_reporting_production_worker_failure.py new file mode 100644 index 000000000..337aa7203 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_worker_failure.py @@ -0,0 +1,296 @@ +"""Owned worker failures stop the composition and emit only closed diagnostics.""" + +import asyncio +import json +import logging + +import pytest + +from adcp.reporting.ledger.notification_models import ReportingNotificationError + +from ._production_support import production_harness +from ._production_transport import MountedProduction + + +def _fault_target(harness, boundary): + if boundary == "producer": + return harness.production.offerings[0].producer, "run_worker" + if boundary == "materializer": + return type(harness.production.materializer), "run_once" + if boundary == "projection": + return harness.projection, "rebuild_one" + if boundary == "sweeper": + return harness.projection, "sweep_one" + import adcp.reporting.production.notifications as notifications + + # The running loop binds notification_turn at startup; its live sampling + # call is the actual owned boundary to fault after a healthy first turn. + return notifications, "next_account" + + +_CASES = [ + (boundary, delivery) + for boundary in ("producer", "materializer", "projection", "sweeper", "notifications") + for delivery in (False, True) + if delivery or boundary != "notifications" +] + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("boundary,delivery", _CASES) +async def test_unexpected_worker_failure_has_closed_operator_diagnostic( + backend, boundary, delivery, tmp_path, monkeypatch, caplog +): + entered, release = asyncio.Event(), asyncio.Event() + private_marker = "private-worker-failure-canary" + + async def fail(*args, **kwargs): + entered.set() + await release.wait() + raise RuntimeError(private_marker) + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=delivery, + notification_delivery=delivery, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + mount = MountedProduction(h) + mount.authorize(h.item) + async with mount.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, before = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + if backend == "postgres": + assert before["media_buy"]["reporting_delivery"]["managed_delivery"] + else: + assert "reporting_delivery" not in before.get("media_buy", {}) + target, method = _fault_target(h, boundary) + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + original_factory = logging.getLogRecordFactory() + + def ambient_factory(*args, **kwargs): + record = original_factory(*args, **kwargs) + record.private_request_context = private_marker + return record + + try: + await asyncio.wait_for(entered.wait(), 5) + caplog.clear() + logging.setLogRecordFactory(ambient_factory) + task = ( + support._notification_task if boundary == "notifications" else support._task + ) + task.set_name(private_marker) + release.set() + await asyncio.wait_for(asyncio.shield(task), 5) + records = [r for r in caplog.records if r.name == "adcp.reporting.production"] + assert len(records) == 1, "unexpected owned failure needs one operator signal" + record = records[0] + assert record.levelno == logging.ERROR + assert record.code == "REPORTING_PRODUCTION_WORKER_STOPPED" + assert record.boundary == boundary + assert record.getMessage() == "Reporting production worker stopped" + assert not record.args and record.exc_info is None and record.stack_info is None + assert record.pathname == "" + assert record.threadName is None and record.processName is None + assert getattr(record, "taskName", None) is None + assert private_marker not in json.dumps(record.__dict__, default=str) + assert support._failed and support._stop.is_set() + tasks = [ + t for t in (support._task, support._notification_task) if t is not None + ] + await asyncio.wait_for(asyncio.gather(*tasks), 5) + assert all(t.done() for t in tasks) + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, after = await mount.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert "reporting_delivery" not in after.get("media_buy", {}) + assert "webhook_signing" not in after + with pytest.raises(ReportingNotificationError, match="component_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + logging.setLogRecordFactory(original_factory) + release.set() + support._stop.set() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("boundary", ["producer", "notifications"]) +@pytest.mark.parametrize("kind", ["domain", "cancellation"]) +async def test_expected_worker_stop_is_silent_and_drains_sibling( + backend, boundary, kind, tmp_path, monkeypatch, caplog +): + entered, release = asyncio.Event(), asyncio.Event() + + async def fail(*args, **kwargs): + entered.set() + await release.wait() + if kind == "cancellation": + raise asyncio.CancelledError + raise ReportingNotificationError("notification_chain_unready") + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=True, + notification_delivery=True, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + target, method = _fault_target(h, boundary) + with monkeypatch.context() as patch: + patch.setattr(target, method, fail) + try: + await asyncio.wait_for(entered.wait(), 5) + caplog.clear() + release.set() + task = support._notification_task if boundary == "notifications" else support._task + if kind == "cancellation": + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(task), 5) + else: + await asyncio.wait_for(asyncio.shield(task), 5) + assert support._stop.is_set(), "the owned sibling must be woken immediately" + tasks = [t for t in (support._task, support._notification_task) if t is not None] + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 5) + assert all(t.done() for t in tasks) + assert not [r for r in caplog.records if r.name == "adcp.reporting.production"] + assert await support.reporting_delivery() == {} + finally: + release.set() + support._stop.set() + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("delivery", [False, True]) +async def test_failing_operator_sink_cannot_prevent_owned_shutdown( + backend, delivery, tmp_path, monkeypatch +): + entered, release = asyncio.Event(), asyncio.Event() + observed = [] + + async def fail(): + entered.set() + await release.wait() + raise RuntimeError("private-provider-diagnostic-canary") + + def broken_sink(record): + observed.append(record) + raise RuntimeError("private-operator-diagnostic-canary") + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=delivery, + notification_delivery=delivery, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + with monkeypatch.context() as patch: + patch.setattr(support.offerings[0].producer, "run_worker", fail) + patch.setattr(logging.getLogger("adcp.reporting.production"), "handle", broken_sink) + try: + await asyncio.wait_for(entered.wait(), 5) + release.set() + tasks = [t for t in (support._task, support._notification_task) if t is not None] + await asyncio.wait_for(asyncio.gather(*tasks), 5) + assert support._failed and support._stop.is_set() + assert len(observed) == 1 + assert "canary" not in json.dumps(observed[0].__dict__, default=str) + assert await support.reporting_delivery() == {} + with pytest.raises(ReportingNotificationError, match="component_unready"): + await support.activate(account_id=h.item.config.account_id) + finally: + release.set() + support._stop.set() + + +@pytest.mark.parametrize("invalid", ["private-boundary-canary", []], ids=["string", "unhashable"]) +def test_operator_boundary_is_runtime_allowlisted(invalid, caplog): + from typing import get_args + + from adcp.reporting.production._diagnostics import ( + _BOUNDARIES, + _worker_stopped, + _WorkerBoundary, + ) + + assert set(get_args(_WorkerBoundary)) == _BOUNDARIES + _worker_stopped(boundary=invalid) + records = [r for r in caplog.records if r.name == "adcp.reporting.production"] + assert len(records) == 1 and records[0].boundary == "worker" + assert "canary" not in json.dumps(records[0].__dict__, default=str) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("stop", ["cancellation", "close"]) +async def test_stopped_composition_does_not_alert_on_late_inflight_failure( + backend, stop, tmp_path, monkeypatch, caplog +): + import adcp.reporting.production.notifications as notifications + + producer_entered, notification_entered = asyncio.Event(), asyncio.Event() + producer_release, notification_release = asyncio.Event(), asyncio.Event() + + async def producer_wait(): + producer_entered.set() + await producer_release.wait() + raise RuntimeError("private-late-worker-canary") + + async def notification_wait(*args, **kwargs): + notification_entered.set() + await notification_release.wait() + if stop == "cancellation": + raise asyncio.CancelledError + return None + + async with production_harness( + backend, + tmp_path / "destination.sqlite", + notifications=True, + notification_delivery=True, + count=0, + poll_seconds=0.02, + ) as h: + support = h.production + closing = None + with monkeypatch.context() as patch: + patch.setattr(support.offerings[0].producer, "run_worker", producer_wait) + patch.setattr(notifications, "next_account", notification_wait) + try: + await asyncio.wait_for( + asyncio.gather(producer_entered.wait(), notification_entered.wait()), 5 + ) + caplog.clear() + if stop == "cancellation": + notification_release.set() + with pytest.raises(asyncio.CancelledError): + await asyncio.wait_for(asyncio.shield(support._notification_task), 5) + else: + closing = asyncio.create_task(support.aclose()) + await asyncio.wait_for(support._stop.wait(), 5) + assert support._stop.is_set() + assert not support._task.done() + producer_release.set() + notification_release.set() + tasks = [t for t in (support._task, support._notification_task) if t is not None] + await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 5) + if closing is not None: + await asyncio.wait_for(closing, 5) + assert not [r for r in caplog.records if r.name == "adcp.reporting.production"] + assert await support.reporting_delivery() == {} + finally: + producer_release.set() + notification_release.set() + support._stop.set() + if closing is not None: + await asyncio.gather(closing, return_exceptions=True) From 8b2418a542341f804f75beb0c1817dc080c02987 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Thu, 24 Sep 2026 23:51:46 +0000 Subject: [PATCH 3/6] fix(reporting): integrate production settling and rc6 boundaries --- .github/workflows/ci.yml | 23 +- docs/reporting-production.md | 82 ++- src/adcp/reporting/feed/errors.py | 5 + src/adcp/reporting/feed/projection.py | 4 +- src/adcp/reporting/feed/request.py | 13 + src/adcp/reporting/feed/snapshot.py | 10 +- src/adcp/reporting/ledger/producer.py | 42 +- src/adcp/reporting/ledger/schedule.py | 32 +- src/adcp/reporting/ledger/status.py | 80 ++- src/adcp/reporting/ledger/status_server.py | 25 +- src/adcp/reporting/production/handler.py | 8 +- src/adcp/reporting/production/service.py | 10 +- src/adcp/reporting/receipts/handler.py | 20 +- tests/conformance/reporting/_feed_support.py | 5 +- .../reporting/_production_delivery_process.py | 3 +- .../reporting/_production_installed.py | 9 +- .../_production_installed_process.py | 9 +- .../reporting/_production_packaging.py | 14 +- .../reporting/_production_support.py | 9 +- .../reporting/_production_transport.py | 3 +- .../reporting/_receipt_transport.py | 6 +- .../reporting/test_reporting_production.py | 6 +- .../test_reporting_production_bindings.py | 9 +- .../test_reporting_production_lifecycle.py | 10 +- .../test_reporting_production_lock_order.py | 9 +- .../test_reporting_production_migration.py | 10 +- ...test_reporting_production_notifications.py | 62 +- .../test_reporting_production_packaging.py | 4 +- .../test_reporting_production_progress.py | 20 +- .../test_reporting_production_readiness.py | 24 +- .../test_reporting_production_restart.py | 6 +- .../test_reporting_production_rolling.py | 7 +- .../test_reporting_production_settling.py | 196 ++++++ .../test_reporting_production_transactions.py | 33 +- ...est_reporting_production_worker_failure.py | 9 +- .../test_reporting_projection_capture.py | 9 +- .../test_reporting_projection_history.py | 15 +- ...test_reporting_projection_notifications.py | 8 +- .../test_reporting_projection_rc6.py | 581 ++++++++++++++++++ .../test_reporting_projection_schedule.py | 3 +- .../test_reporting_projection_waiver_rc6.py | 72 +++ .../test_reporting_schedule_schema.py | 9 +- 42 files changed, 1317 insertions(+), 197 deletions(-) create mode 100644 tests/conformance/reporting/test_reporting_production_settling.py create mode 100644 tests/conformance/reporting/test_reporting_projection_rc6.py create mode 100644 tests/conformance/reporting/test_reporting_projection_waiver_rc6.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 45a1548c8..d2edf5f5b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -675,6 +675,8 @@ jobs: pg-reporting-production: name: Production reporting status, ownership and notification contracts runs-on: ubuntu-latest + permissions: + contents: read timeout-minutes: 35 services: postgres: @@ -682,7 +684,6 @@ jobs: env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_production_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: ["5432:5432"] options: >- --health-cmd pg_isready --health-interval 5s @@ -702,6 +703,8 @@ jobs: env: ADCP_PG_TEST_URL: postgresql://postgres@localhost:5432/adcp_production_test run: | + # Explicit pytest paths bypass --ignore; preselect and reject empty globs. + shopt -s nullglob source_tests=() for test in tests/conformance/reporting/test_reporting_production*.py; do case "$test" in @@ -709,9 +712,14 @@ jobs: esac source_tests+=("$test") done + projection_tests=(tests/conformance/reporting/test_reporting_projection*.py) + if (( ${#source_tests[@]} == 0 || ${#projection_tests[@]} == 0 )); then + echo "Production or projection test selection is empty" + exit 1 + fi python scripts/reporting_test_harness.py pytest \ "${source_tests[@]}" \ - tests/conformance/reporting/test_reporting_projection*.py \ + "${projection_tests[@]}" \ tests/conformance/reporting/test_reporting_tier_projection.py \ tests/conformance/reporting/test_reporting_schedule_schema.py \ tests/test_reporting_capability_models.py \ @@ -730,6 +738,8 @@ jobs: pg-reporting-production-installed: name: Installed production reporting (${{ matrix.cell }}, Python 3.10) runs-on: ubuntu-latest + permissions: + contents: read timeout-minutes: 40 strategy: fail-fast: false @@ -741,7 +751,6 @@ jobs: env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_production_installed_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: ["5432:5432"] options: >- --health-cmd pg_isready --health-interval 5s @@ -784,6 +793,8 @@ jobs: pg-reporting-production-compatibility: name: B2.3 and hardening to installed B2.4 activation and restart runs-on: ubuntu-latest + permissions: + contents: read timeout-minutes: 50 services: postgres: @@ -791,15 +802,15 @@ jobs: env: POSTGRES_HOST_AUTH_METHOD: trust POSTGRES_DB: adcp_production_rolling_test - POSTGRES_INITDB_ARGS: "--encoding=UTF8 --lc-collate=C --lc-ctype=C" ports: ["5432:5432"] options: >- --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 steps: - uses: actions/checkout@v6 - - name: Fetch both independently approved artifacts - run: git fetch --no-tags origin 50e35f0ae3540f19b40e8fc460f5870dfe018bf9 a09878f67ab397a4b51b3314e3e8a5e87cf96da5 + - name: Fetch integrated feed and hardening comparison artifacts + timeout-minutes: 1 + run: git fetch --no-tags origin 2d777ace7b4bf8be519ce0abd4fd0a25ed4f1da7 e16eb8cf3074cabd45aab42840950f05ad6d2b43 - uses: actions/setup-python@v6 id: production-python310 with: diff --git a/docs/reporting-production.md b/docs/reporting-production.md index acbd34705..d89178e36 100644 --- a/docs/reporting-production.md +++ b/docs/reporting-production.md @@ -169,36 +169,42 @@ applicable official adjustment. Later valid artifacts or health degradation do not erase accepted evidence or successful-materialization counts. Consumer rejection is never a materializer retry signal. -For #1179, a generation owes only complete periods whose start is at or after -activation and strictly before deactivation. A period already begun at -deactivation remains owed in full, including its SLA. The producer and feed use -this same rule. `next_expected_at` is the nearest strictly future committed -expectation in the selected captured scope, even when current health is complete. -The immutable Draft 7 cache for `3.2.0-rc.3` rejects that otherwise-valid response -at `/allOf/2/then/not`. The effective Python SDK validator and advertised MCP -schema remove only that exact known prohibition, only for this version. The -`if` and its requirements that `scope_closed` and `coverage_complete` are both -true remain enforced, as do types, formats and all other conditionals. A -different version or changed rule is left intact. Cached files and generated -status models are preserved; this is an explicit SDK correction, not a new -upstream schema version. The executable reproduction and constraint/mounted -regressions are in `test_reporting_schedule_schema.py`; the existing SDK issue -is [#1179](https://github.com/adcontextprotocol/adcp-client-python/issues/1179). - -Cross-language compatibility remains a separate blocking dependency. Every -selected compatible stable/skew client/server lane must successfully return -`complete` with the legitimate future expectation and correct semantics. -Unsupported-schema errors are acceptable only for explicitly unsupported -combinations. Python success does not establish TypeScript/protocol agreement -or release the later expert/four-quadrant gate. The coordinator owns upstream -schema/version coordination; neither suppress the expectation nor change an -otherwise-correct health value to satisfy the original prohibition. -It is absent when there is no applicable expectation. Captured timezones and -civil/DST boundaries survive later configuration changes. Producer turns keep -the 64-period maximum; durable cursors and bounded pending-acquisition queues -advance past completed windows without an unbounded history scan. Lease -acquisition takes the account lock before configuration rows and preserves -turn-primary fairness and account isolation. +A generation owes only complete periods whose start is at or after activation +and strictly before deactivation. A period already begun at deactivation remains +owed in full, including its SLA. The producer and captured feed use the same rule. +For the current `3.2-rc.6` contract, an open summary's `next_expected_at` is its +nearest future obligation due time. A **complete summary** instead reports the +nearest future period **start**, strictly after the captured `ledger_as_of`, +across the committed generations in scope. It is absent when no such period +exists. This forecast creates, counts and leases no future obligation, and does +not change scope closure, coverage or health. Complete periods responses do not +inherit the summary-only forecast. Civil-time and DST boundaries are retained. + +The immutable `3.2.0-rc.3` cache and exact-version Python correction for #1179 +remain available for historical validation. Direct historical projections and +stored rc.3 walks retain their original representation; they do not acquire the +new rc.6 forecast. The current SDK does not advertise rc.3 as a live client or +server pin. New mounts, advertised schemas and rendering use the supported +packaged version. Cursor/checkpoint version mismatches fail closed only after +caller and signed-position verification. Changing a production mount's captured +protocol pin invalidates readiness even after a successful schema proof. + +These are Python source and conformance boundaries, not TypeScript, release or +cross-language acceptance. The tests in `test_reporting_projection_rc6.py` cover +current mounted transports, clients and schema agreement, frozen historical +bytes and continuations, civil/DST boundaries and absence of future work. The +exact original rc.3 rejection and version-scoped correction remain separately +recorded in `test_reporting_schedule_schema.py`. + +Producer turns keep the 64-period maximum. Durable closing positions and bounded +pending queues rotate fairly without rescanning all history. A readable snapshot +completes an acquisition, not its declared settling policy: the queue retains +that obligation through restatement cadence and the settling window. It retires +a snapshot-only policy at its terminal boundary, or a closing policy only after +an official revision is actually committed. A not-ready official source leaves +work pending. Persisted observation checkpoints survive fresh workers and keep +no-op refreshes from replaying or changing execution identity. Lease acquisition +takes the account lock before configuration rows and preserves account isolation. Lease acquisition, release and recovery are bookkeeping, not new reporting observations or materializer targets. The production PostgreSQL path retains the inherited trigger and cancels only its lease-only candidate increment in @@ -314,3 +320,19 @@ or the later cross-language interoperability gate. | Private scope and buyer ownership | B2.3 inputs; B2.4 wire/walk | Exact page-local ownership, malformed/mixed/conflicting walks, cross-page dependencies and conservative legacy compatibility. | | Production tier capabilities | B2.4; #1180 | Full provider/source contracts, live component/mount checks, empty-seller discovery, polling and signing/retry truthfulness. | | Rolling compatibility | Every slice | Eleven distinct historical inputs, isolated manifests, populated/repeated/interrupted migration, installed floor runtimes and actual restarts. | + +## Integrated comparison and release-note limits + +The production rolling controls compare the integrated B2.3 artifact +`2d777ace7b4bf8be519ce0abd4fd0a25ed4f1da7` and integrated hardening artifact +`e16eb8cf3074cabd45aab42840950f05ad6d2b43`. Their frozen bytes, origins and +continuations must be established by fresh installed CI. Earlier snapshots are +not qualified by these comparisons. + +The release notes must retain all eight exclusions: pre-`17ee407a` A, +pre-`0f34c666` B, pre-`967b6e28` C, pre-`5487f2bd` B1, +pre-`3fd62121` B2.1, pre-`09fd87f7` B2.2, pre-`2d777ace` B2.3, +and pre-`e16eb8cf` hardening. In particular, old A whole-trigger startup after C +is not supported, and historical false notification-readiness results do not +become healthy through later source integration. These notes do not qualify +simultaneous old autonomous writers or release/activation acceptance. diff --git a/src/adcp/reporting/feed/errors.py b/src/adcp/reporting/feed/errors.py index e00d1d694..052a9b0ff 100644 --- a/src/adcp/reporting/feed/errors.py +++ b/src/adcp/reporting/feed/errors.py @@ -7,6 +7,7 @@ FeedErrorCode = Literal[ "INVALID_REQUEST", "INVALID_CHECKPOINT", + "REPORTING_FEED_VERSION_MISMATCH", "UNAUTHORIZED", "REPORTING_FEED_SCHEMA_UNREADY", "REPORTING_FEED_HISTORY_CORRUPT", @@ -17,6 +18,10 @@ _MESSAGES: dict[FeedErrorCode, str] = { "INVALID_REQUEST": "supply a periods request with valid reporting filters and pagination", "INVALID_CHECKPOINT": "restart the reporting walk; this position is unavailable for this scope", + "REPORTING_FEED_VERSION_MISMATCH": ( + "continue with the original walk's reporting representation;" + " start a new walk without a cursor or checkpoint when changing protocol version" + ), "UNAUTHORIZED": "the reporting account or authenticated consumer is unavailable", "REPORTING_FEED_SCHEMA_UNREADY": "install and verify the isolated reporting feed schema", "REPORTING_FEED_HISTORY_CORRUPT": "retained reporting feed evidence requires operator repair", diff --git a/src/adcp/reporting/feed/projection.py b/src/adcp/reporting/feed/projection.py index bad09482b..bb9468409 100644 --- a/src/adcp/reporting/feed/projection.py +++ b/src/adcp/reporting/feed/projection.py @@ -534,7 +534,9 @@ def capture_feed( "health": scope_result.health, "issues": [issue.to_wire() for issue in scope_result.issues], } - if representation_version == 2: + if representation_version == 2 and not ( + scope_result.health == "complete" and "adcp_version" in filters + ): configurations = tuple( c for c in scope_result.configurations diff --git a/src/adcp/reporting/feed/request.py b/src/adcp/reporting/feed/request.py index 3aaaa197e..de12580b3 100644 --- a/src/adcp/reporting/feed/request.py +++ b/src/adcp/reporting/feed/request.py @@ -12,6 +12,11 @@ from jsonschema import Draft7Validator, FormatChecker +from adcp._version import ( + is_adcp_version_at_least, + normalize_to_release_precision, + resolve_adcp_version, +) from adcp.reporting.feed.errors import ReportingFeedError from adcp.reporting.receipts.wire import _IDENTITY_FIELDS from adcp.validation.schema_loader import get_portable_schema @@ -151,6 +156,14 @@ def parse(cls, request: dict[str, Any]) -> FeedRequest: "reporting_revision_id", } } + version = normalize_to_release_precision( + request.get("adcp_version") or resolve_adcp_version(None) + ) + if is_adcp_version_at_least(version, "3.2-rc.6"): + # Existing snapshots omit this marker and keep their rc.3 + # bytes. The persisted filter binding prevents cross-version + # replay under an incompatible advertised output schema. + filters["adcp_version"] = version for name in ( "delivery_config_ids", "media_buy_ids", diff --git a/src/adcp/reporting/feed/snapshot.py b/src/adcp/reporting/feed/snapshot.py index 069a1539d..ef89a406b 100644 --- a/src/adcp/reporting/feed/snapshot.py +++ b/src/adcp/reporting/feed/snapshot.py @@ -238,7 +238,6 @@ def check( position = decoded[3] if ( snapshot.caller != caller - or snapshot.filters_json != request.filters_json or decoded[1] != kind or decoded[2] != snapshot.snapshot_id or type(position) is not int @@ -248,6 +247,15 @@ def check( or not hmac.compare_digest(token, self.token(kind, position)) ): raise ReportingFeedError("INVALID_CHECKPOINT") + if snapshot.filters_json != request.filters_json: + # Only an authenticated, correctly signed position can disclose + # this actionable version boundary. Other callers/tokens retain + # the indistinguishable INVALID_CHECKPOINT error above. + if json.loads(snapshot.filters_json).get("adcp_version") != request.filters.get( + "adcp_version" + ): + raise ReportingFeedError("REPORTING_FEED_VERSION_MISMATCH") + raise ReportingFeedError("INVALID_CHECKPOINT") return position def page(self, offset: int, limit: int) -> dict[str, Any]: diff --git a/src/adcp/reporting/ledger/producer.py b/src/adcp/reporting/ledger/producer.py index aaacafe71..62134481e 100644 --- a/src/adcp/reporting/ledger/producer.py +++ b/src/adcp/reporting/ledger/producer.py @@ -591,8 +591,15 @@ async def _acquire_progress( ) if obligation is None or obligation.generation_key != configuration.generation_key: raise LedgerConflictError("HISTORY_UNAVAILABLE", "producer history is unavailable") + policy = self._settling_policy(configuration, obligation) + finished = policy is None try: - await self.acquire_obligation(configuration, obligation, turn=turn, now=now) + if policy is None: + await self.acquire_obligation(configuration, obligation, turn=turn, now=now) + else: + finished = await self._acquire_with_settling_policy( + configuration, obligation, policy=policy, turn=turn, now=now + ) except (ReportingCurrencyError, LedgerConflictError) as error: if isinstance(error, LedgerConflictError) and error.code not in { "HISTORY_UNAVAILABLE", @@ -601,9 +608,13 @@ async def _acquire_progress( raise turn.slices_failed.append(identifier) self._note_escalation(obligation, turn, now=now) - await progress.finish_producer_acquisition( - configuration, reporting_obligation_id=identifier - ) + # Retain the existing corrupt-history parking check. A transient + # currency failure during settling must not retire a readable leaf. + finished = policy is None or isinstance(error, LedgerConflictError) + if finished: + await progress.finish_producer_acquisition( + configuration, reporting_obligation_id=identifier + ) def _settling_policy( self, @@ -650,14 +661,19 @@ async def _acquire_with_settling_policy( policy: _SettlingPolicy, turn: WorkerTurn, now: datetime, - ) -> None: + ) -> bool: + """Return whether policy-controlled acquisition may leave the pending queue. + + A readable snapshot completes one acquisition, not the settling policy. + The progress store retains its rotating work item until the policy ends. + """ checkpoint_store = self._restatement_store() revisions = await self._store.list_revisions( account_id=obligation.account_id, reporting_obligation_id=obligation.reporting_obligation_id, ) if any(item.finality == "official" for item in revisions): - return + return True if not revisions: await self.acquire_obligation( configuration, @@ -667,7 +683,7 @@ async def _acquire_with_settling_policy( target_finality="snapshot", track_settling=True, ) - return + return False checkpoint = await checkpoint_store.get_restatement_checkpoint( account_id=obligation.account_id, @@ -695,10 +711,10 @@ async def _acquire_with_settling_policy( target_finality="snapshot", track_settling=True, ) - return + return False if policy.official_close_lag is None or self._offerings.official_offering_id is None: - return + return True closes_at = max( settles_at, _utc(obligation.period.end) + policy.official_close_lag, @@ -713,6 +729,14 @@ async def _acquire_with_settling_policy( target_finality="official", track_settling=True, ) + # An attempted close can still return not-ready. Retire only after + # the authoritative revision was actually committed. + revisions = await self._store.list_revisions( + account_id=obligation.account_id, + reporting_obligation_id=obligation.reporting_obligation_id, + ) + return any(item.finality == "official" for item in revisions) + return False async def acquire_obligation( self, diff --git a/src/adcp/reporting/ledger/schedule.py b/src/adcp/reporting/ledger/schedule.py index 13e6c8aac..9467a88b1 100644 --- a/src/adcp/reporting/ledger/schedule.py +++ b/src/adcp/reporting/ledger/schedule.py @@ -23,13 +23,17 @@ def committed_periods( - configuration: ReportingConfiguration, *, near: datetime | None = None + configuration: ReportingConfiguration, + *, + near: datetime | None = None, + near_start: datetime | None = None, ) -> Iterator[ReportingPeriodBoundary]: """Yield full committed periods, optionally seeking near an expected-at time. ``near`` is an optimization, not a filter: the caller still compares exact instants. Two predecessor civil slots retain the period containing the requested instant, including its DST fold. No wall clock is consulted. + ``near_start`` seeks around a period start independently of its SLA. """ activated = configuration.activated_at if activated is None: @@ -37,11 +41,14 @@ def committed_periods( schedule, timezone = configuration.schedule, configuration.account_timezone zone, duration, anchor = _schedule_clock(schedule, timezone) ordinal = first_ordinal_after(schedule, account_timezone=timezone, activated_at=activated) - if near is not None: + seek = near_start + if seek is None and near is not None: + seek = near - iso_duration_to_timedelta(schedule.delivery_sla) + if seek is not None: candidate = first_ordinal_after( schedule, account_timezone=timezone, - activated_at=near - iso_duration_to_timedelta(schedule.delivery_sla), + activated_at=seek, ) ordinal = max(ordinal, candidate - 2) while True: @@ -93,3 +100,22 @@ def next_reporting_expectation( future.append(period.expected_at) break return min(future) if future else None + + +def next_reporting_period_start( + configurations: Sequence[ReportingConfiguration], *, as_of: datetime +) -> datetime | None: + """rc.6 complete-summary forecast outside the closed evaluated horizon. + + Only captured committed generations supply this forecast. A completed + obligation's due time is not a period start, and a future forecast does + not create, count or lease an obligation. The producer still uses the + identical full-period activation/deactivation and civil-time boundaries. + """ + future = [] + for configuration in configurations: + for period in committed_periods(configuration, near_start=as_of): + if period.start > as_of: + future.append(period.start) + break + return min(future) if future else None diff --git a/src/adcp/reporting/ledger/status.py b/src/adcp/reporting/ledger/status.py index d4931463a..62e98850a 100644 --- a/src/adcp/reporting/ledger/status.py +++ b/src/adcp/reporting/ledger/status.py @@ -35,6 +35,11 @@ from datetime import datetime, timezone from typing import Any, Literal +from adcp._version import ( + is_adcp_version_at_least, + normalize_to_release_precision, + resolve_adcp_version, +) from adcp.reporting.canonical_json import canonical_json_utf8_v1 from adcp.reporting.ledger.consumer_status import condition_after_waiver from adcp.reporting.ledger.delivery_models import ReportingDeliveryRecord @@ -49,7 +54,7 @@ ReportingRevisionRecord, ) from adcp.reporting.ledger.notification_models import ReportingStatusScope -from adcp.reporting.ledger.schedule import next_reporting_expectation +from adcp.reporting.ledger.schedule import next_reporting_expectation, next_reporting_period_start from adcp.reporting.ledger.status_projection import ( ReportingStatusSnapshot, StatusProjectionInput, @@ -231,6 +236,13 @@ def render_snapshot( if snapshot.account_id != caller.account_id: raise LedgerConflictError("LOOKUP_UNAVAILABLE", "status is unavailable to this caller") filters = _filters(request) + version = normalize_to_release_precision( + request.get("adcp_version") or resolve_adcp_version(None) + ) + complete_start_forecast = is_adcp_version_at_least(version, "3.2-rc.6") + if complete_start_forecast: + # Bind the new wire contract without relabelling explicit rc.3 snapshots. + filters["adcp_version"] = version scope = ReportingStatusScope( caller.account_id, consumer_id=( @@ -384,36 +396,42 @@ def render_snapshot( if self._consumer_status_enabled: counts["consumer_status_pending"] = result.pending_count watermark = _scope_data_through(p.projection for p in result.obligations) - next_expected = next_reporting_expectation( - tuple( - c - for c in result.configurations - if (not request.get("finality") or c.required_finality in request["finality"]) - and ( - not request.get("health") - or project_status_scope( - replace( - value, - scope=ReportingStatusScope( - c.account_id, c.generation_key, consumer_id=scope.consumer_id - ), - ) - ).health - in request["health"] - ) - ), - tuple( - p.obligation - for p in result.obligations - if ( - not request.get("finality") - or p.obligation.required_finality in request["finality"] - ) - and (not request.get("health") or p.projection.health in request["health"]) - ), - as_of=snapshot.as_of, - period_start=value.period_start, - period_end=value.period_end, + configurations = tuple( + c + for c in result.configurations + if (not request.get("finality") or c.required_finality in request["finality"]) + and ( + not request.get("health") + or project_status_scope( + replace( + value, + scope=ReportingStatusScope( + c.account_id, c.generation_key, consumer_id=scope.consumer_id + ), + ) + ).health + in request["health"] + ) + ) + selected_obligations = tuple( + p.obligation + for p in result.obligations + if ( + not request.get("finality") + or p.obligation.required_finality in request["finality"] + ) + and (not request.get("health") or p.projection.health in request["health"]) + ) + next_expected = ( + next_reporting_period_start(configurations, as_of=snapshot.as_of) + if complete_start_forecast and result.health == "complete" + else next_reporting_expectation( + configurations, + selected_obligations, + as_of=snapshot.as_of, + period_start=value.period_start, + period_end=value.period_end, + ) ) return { **common, diff --git a/src/adcp/reporting/ledger/status_server.py b/src/adcp/reporting/ledger/status_server.py index 23a95f378..2f073cd32 100644 --- a/src/adcp/reporting/ledger/status_server.py +++ b/src/adcp/reporting/ledger/status_server.py @@ -5,6 +5,8 @@ from collections.abc import Awaitable, Callable from typing import Any +from adcp._version import is_adcp_version_at_least, resolve_adcp_version +from adcp.exceptions import ConfigurationError from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler from adcp.server.base import ADCPHandler, ToolContext from adcp.types import GetReportingStatusRequest @@ -24,19 +26,38 @@ class ReportingStatusNotificationHandler(ADCPHandler[ToolContext]): """ def __init__( - self, status: ReportingStatusHandler, *, resolve_caller: ReportingStatusCallerResolver + self, + status: ReportingStatusHandler, + *, + resolve_caller: ReportingStatusCallerResolver, + adcp_version: str | None = None, ) -> None: super().__init__() self.reporting_status_handler = status self._resolve_status_caller = resolve_caller + self._adcp_version = resolve_adcp_version(adcp_version) + if not is_adcp_version_at_least(self._adcp_version, "3.2-rc.3"): + raise ConfigurationError( + "reporting mounts require a supported AdCP 3.2 reporting contract; " + "use 3.2-rc.3 for retained walks or omit the pin for the packaged default" + ) + + def get_adcp_version(self) -> str: + """Public per-mount protocol pin, shared by schema and rendering.""" + return self._adcp_version async def get_reporting_status( self, params: GetReportingStatusRequest | dict[str, Any], context: ToolContext | None = None ) -> dict[str, Any]: request = ( - params + dict(params) if isinstance(params, dict) else params.model_dump(mode="json", exclude_unset=True) ) + request["adcp_version"] = ( + context.resolved_adcp_version + if context is not None and context.resolved_adcp_version is not None + else request.get("adcp_version") or self.get_adcp_version() + ) caller = await self._resolve_status_caller(request, context) return await self.reporting_status_handler.handle(request, caller=caller) diff --git a/src/adcp/reporting/production/handler.py b/src/adcp/reporting/production/handler.py index 75de9eabd..6a77df295 100644 --- a/src/adcp/reporting/production/handler.py +++ b/src/adcp/reporting/production/handler.py @@ -65,6 +65,7 @@ def __init__( *, resolve_account: ReceiptAccountResolver, buyer_agents: BuyerAgentRegistry | None = None, + adcp_version: str | None = None, ) -> None: self.production = production super().__init__( @@ -72,6 +73,7 @@ def __init__( resolve_account=resolve_account, buyer_agents=buyer_agents, consumer_status_enabled=production.projection.consumer_status_enabled, + adcp_version=adcp_version, ) def advertised_tools_for_instance(self) -> set[str]: @@ -127,7 +129,11 @@ async def get_adcp_capabilities( context: ToolContext | None = None, ) -> dict[str, Any]: response = capabilities_response( - ["media_buy"], sandbox=False, idempotency={"supported": False} + ["media_buy"], + sandbox=False, + idempotency={"supported": False}, + adcp_version=self.production._protocol_version, + supported_versions=[self.production._protocol_version], ) response["account"] = self.production.configuration_task.account_capabilities() reporting = await self.production.reporting_delivery() diff --git a/src/adcp/reporting/production/service.py b/src/adcp/reporting/production/service.py index 54848a145..1f486f9cf 100644 --- a/src/adcp/reporting/production/service.py +++ b/src/adcp/reporting/production/service.py @@ -165,6 +165,7 @@ def __init__( status_retention_days: int = 400, notification_workers: tuple[ReportingNotificationWorker, ...] = (), poll_seconds: float = 0.25, + adcp_version: str | None = None, ) -> None: from adcp.reporting.outbox.worker import ReportingNotificationWorker from adcp.reporting.production.handler import ReportingProductionHandler @@ -211,7 +212,10 @@ def __init__( self._notification_identity = tuple(worker_identity(w) for w in notification_workers) self.handler = ReportingProductionHandler( - self, resolve_account=resolve_account, buyer_agents=buyer_agents + self, + resolve_account=resolve_account, + buyer_agents=buyer_agents, + adcp_version=adcp_version, ) self._mounts: list[_MountedProduction] = [] self._task: asyncio.Task[None] | None = None @@ -227,6 +231,7 @@ def __init__( self._schema_task: asyncio.Task[bool] | None = None self._schema_epoch = 0 self._schema_positive: tuple[int, int] | None = None + self._protocol_version = self.handler.get_adcp_version() self._components = self._component_ids() self._methods = self._handler_methods() self.store._production_support = weakref.ref(self) @@ -269,6 +274,7 @@ def _handler_methods(self) -> tuple[object, ...]: "sync_reporting_status", "sync_accounts", "get_adcp_capabilities", + "get_adcp_version", ) ) @@ -306,6 +312,8 @@ def _assert_components(self, *, running: bool = True, mounted: bool = True) -> N ) or self._components != self._component_ids() or self._methods != self._handler_methods() + or type(self.handler._adcp_version) is not str + or self.handler._adcp_version != self._protocol_version or ( (running or mounted) and not isinstance(self.store, InMemoryReportingProductionStore) diff --git a/src/adcp/reporting/receipts/handler.py b/src/adcp/reporting/receipts/handler.py index f0d7fbbbf..62a5db83d 100644 --- a/src/adcp/reporting/receipts/handler.py +++ b/src/adcp/reporting/receipts/handler.py @@ -5,9 +5,10 @@ from collections.abc import Awaitable, Callable from typing import Any +from adcp._version import is_adcp_version_at_least, resolve_adcp_version from adcp.decisioning.context import AuthInfo, RequestContext from adcp.decisioning.registry import BuyerAgent, BuyerAgentRegistry, HttpSigCredential -from adcp.exceptions import ADCPTaskError +from adcp.exceptions import ADCPTaskError, ConfigurationError from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal from adcp.reporting.ledger.notification_models import ReportingNotificationError from adcp.reporting.outbox.identity import canonical_consumer, resolve_reporting_consumer @@ -109,6 +110,7 @@ def __init__( resolve_account: ReceiptAccountResolver, buyer_agents: BuyerAgentRegistry | None = None, consumer_status_enabled: bool = False, + adcp_version: str | None = None, ) -> None: super().__init__() if not isinstance(store, ReportingReceiptBatchStore): @@ -122,6 +124,16 @@ def __init__( store if isinstance(store, ReportingFeedStore) else None ) self._feed_consumer_status_enabled = consumer_status_enabled + self._adcp_version = resolve_adcp_version(adcp_version) + if not is_adcp_version_at_least(self._adcp_version, "3.2-rc.3"): + raise ConfigurationError( + "reporting mounts require a supported AdCP 3.2 reporting contract; " + "use 3.2-rc.3 for retained walks or omit the pin for the packaged default" + ) + + def get_adcp_version(self) -> str: + """Select rendering and advertised MCP/A2A schemas for this mount.""" + return self._adcp_version def advertised_tools_for_instance(self) -> set[str]: return {TASK, "get_reporting_status"} if self.reporting_feed_store is not None else {TASK} @@ -140,10 +152,14 @@ async def get_reporting_status( if self.reporting_feed_store is None: return self._not_supported("get_reporting_status") request = ( - params + dict(params) if isinstance(params, dict) else params.model_dump(mode="json", exclude_unset=True) ) + if context is not None and context.resolved_adcp_version is not None: + request["adcp_version"] = context.resolved_adcp_version + else: + request.setdefault("adcp_version", self.get_adcp_version()) try: if request.get("view") == "periods": FeedRequest.parse(request) diff --git a/tests/conformance/reporting/_feed_support.py b/tests/conformance/reporting/_feed_support.py index 3ef9ecfea..c976d0481 100644 --- a/tests/conformance/reporting/_feed_support.py +++ b/tests/conformance/reporting/_feed_support.py @@ -160,10 +160,9 @@ def __init__(self, h, *, feedback=False, **kwargs): resolve_account=self.resolve_account, buyer_agents=self.registry, consumer_status_enabled=feedback, + adcp_version=self.version, ) self.handler.get_reporting_status = self.idempotency.wrap(self.handler.get_reporting_status) - if kwargs.get("version") is not None: - self.handler.adcp_version = kwargs["version"] @asynccontextmanager async def sdk_clients(self, a2a_version, *, token="token-one"): @@ -215,7 +214,7 @@ def mcp_http(**kwargs): clients[protocol] = await stack.enter_async_context( ADCPClient( config, - adcp_version="3.2-rc.6", + adcp_version=self.version, force_a2a_version=a2a_version if protocol == "a2a" else None, httpx_client_factory=mcp_http if protocol == "mcp" else None, ) diff --git a/tests/conformance/reporting/_production_delivery_process.py b/tests/conformance/reporting/_production_delivery_process.py index 4e92e2b7b..655238e12 100644 --- a/tests/conformance/reporting/_production_delivery_process.py +++ b/tests/conformance/reporting/_production_delivery_process.py @@ -102,7 +102,8 @@ async def main(settings): await task raise AssertionError("paused child must be killed") before = await retained_windows(SimpleNamespace(pool=pool)) - assert await worker.deliver_one(account_id="acct_a") + production_operation_1 = await worker.deliver_one(account_id="acct_a") + assert production_operation_1 assert await retained_windows(SimpleNamespace(pool=pool)) == before row = before[0] states = await worker.outbox.list_deliveries(account_id="acct_a") diff --git a/tests/conformance/reporting/_production_installed.py b/tests/conformance/reporting/_production_installed.py index 128b489ae..7125abeb0 100644 --- a/tests/conformance/reporting/_production_installed.py +++ b/tests/conformance/reporting/_production_installed.py @@ -38,10 +38,11 @@ def main(settings): ) from adcp.validation import schema_loader - schema_root = schema_loader._resolve_schema_root("3.2.0-rc.3").root - assert schema_root.is_relative_to(Path(sys.prefix)) - for name, expected in settings["schemas"].items(): - assert hashlib.sha256((schema_root / name).read_bytes()).hexdigest() == expected + for version, schemas in settings["schemas"].items(): + schema_root = schema_loader._resolve_schema_root(version).root + assert schema_root.is_relative_to(Path(sys.prefix)) + for name, expected in schemas.items(): + assert hashlib.sha256((schema_root / name).read_bytes()).hexdigest() == expected if settings["driver_absent"]: assert importlib.util.find_spec("psycopg") is None assert importlib.util.find_spec("psycopg_pool") is None diff --git a/tests/conformance/reporting/_production_installed_process.py b/tests/conformance/reporting/_production_installed_process.py index 8db3c347e..47ebbea16 100644 --- a/tests/conformance/reporting/_production_installed_process.py +++ b/tests/conformance/reporting/_production_installed_process.py @@ -91,13 +91,15 @@ async def pending_identity(): if settings["pause"]: assert await pending_identity() == "pending" - assert await h.production.activate(account_id="acct_a") + production_operation_1 = await h.production.activate(account_id="acct_a") + assert production_operation_1 # Run the real producer's lease acquisition/release after # activation while the actual parent's attempt is pending. # The already-killed parent's lease deliberately spans setup. # Persist its expiry as a bounded crash-recovery fault, without # changing any original attempt, generation or external identity. - assert (await source_turn(h.production)).leased is not None + production_operation_2 = await source_turn(h.production) + assert (production_operation_2).leased is not None async with pool.connection() as c, c.transaction(): account = original_attempt.scope.principal.account_id await h.store._lock_account(c, account) @@ -142,7 +144,8 @@ async def pending_identity(): assert rows == (int(settings["notifications"]), int(settings["notifications"]), 0) await drain(h.projection, "acct_a") else: - assert not await h.production.activate(account_id="acct_a") + production_operation_3 = await h.production.activate(account_id="acct_a") + assert not production_operation_3 assert await pending_identity() == "acked" # A new eligible period was committed after both first pages. # Finish it through bounded real turns if startup first handled diff --git a/tests/conformance/reporting/_production_packaging.py b/tests/conformance/reporting/_production_packaging.py index b6ace99de..5eb9c20da 100644 --- a/tests/conformance/reporting/_production_packaging.py +++ b/tests/conformance/reporting/_production_packaging.py @@ -60,6 +60,9 @@ def production_modules(): "reporting/materializer/publication.py", "reporting/materializer/service.py", "reporting/materializer/verification.py", + "reporting/feed/request.py", + "reporting/feed/errors.py", + "reporting/ledger/status_server.py", "reporting/feed/snapshot.py", "reporting/feed/projection.py", "reporting/feed/memory.py", @@ -244,10 +247,13 @@ def installed_production(root, python, wheel, source, *, label, driver_absent): "modules": modules, "assets": assets, "schemas": { - name: hashlib.sha256( - (ROOT / "schemas/cache/3.2.0-rc.3" / name).read_bytes() - ).hexdigest() - for name in SCHEMAS + version: { + name: hashlib.sha256( + (ROOT / "schemas/cache" / version / name).read_bytes() + ).hexdigest() + for name in SCHEMAS + } + for version in ("3.2.0-rc.3", "3.2.0-rc.6") }, "tests": [str(p.relative_to(ROOT)) for p in tests], "driver_absent": driver_absent, diff --git a/tests/conformance/reporting/_production_support.py b/tests/conformance/reporting/_production_support.py index 192a6bfd4..c6e78668d 100644 --- a/tests/conformance/reporting/_production_support.py +++ b/tests/conformance/reporting/_production_support.py @@ -372,6 +372,8 @@ async def production_harness( feedback=False, identity_prefix="", poll_seconds=60, + source_factory=Source, + adcp_version=None, ): from contextlib import AsyncExitStack @@ -484,7 +486,7 @@ async def production_harness( io, ) source_clock = ManualClock(END) - source = Source( + source = source_factory( key, path.with_name("source"), rows if source_publication or periods is not None else None, @@ -498,7 +500,9 @@ async def production_harness( source=source, offerings=ProducerOfferings( snapshot_offering_id=None if reconciled else source.source_id, - official_offering_id=source.source_id if reconciled else None, + official_offering_id=( + source.source_id if reconciled else getattr(source, "official_source_id", None) + ), publication_namespace=source.capabilities.offerings[0].publication_namespace, source_scope=source.capabilities.source_scope, ), @@ -627,6 +631,7 @@ async def authorize(account, context, consumer): resolve_account=authorize, notification_workers=workers, poll_seconds=poll_seconds, + adcp_version=adcp_version, ) h.production, h.projection, h.item = support, projection, item h.source_clock = source_clock diff --git a/tests/conformance/reporting/_production_transport.py b/tests/conformance/reporting/_production_transport.py index b232ad999..f9789ad6f 100644 --- a/tests/conformance/reporting/_production_transport.py +++ b/tests/conformance/reporting/_production_transport.py @@ -9,6 +9,7 @@ class MountedProduction(MountedReceipts): def __init__(self, h): super().__init__(h) self.handler = h.production.handler + self.version = self.handler.get_adcp_version() def authorize(self, s, *, token="token-one"): super().authorize(s, token=token) @@ -18,7 +19,7 @@ async def middleware(self, name, params, context, call_next): return await call_next() async def call(self, client, task, request, *, transport="mcp", token="token-one"): - request = {"adcp_version": "3.2-rc.3", **request} + request = {"adcp_version": self.version, **request} def route(wire): value = json.loads(wire) diff --git a/tests/conformance/reporting/_receipt_transport.py b/tests/conformance/reporting/_receipt_transport.py index 1b25cb4f7..bf2e41376 100644 --- a/tests/conformance/reporting/_receipt_transport.py +++ b/tests/conformance/reporting/_receipt_transport.py @@ -54,6 +54,9 @@ def __init__(self, h, *, hydrated=False, registry_kind=None, version=None): self.registry = Registry() if registry_kind is not None else None self.sessions = {} self.counter = 0 + from adcp._version import normalize_to_release_precision, resolve_adcp_version + + self.version = normalize_to_release_precision(version or resolve_adcp_version(None)) self.idempotency = IdempotencyStore(backend=ForbiddenGenericCache()) self.handler = ReportingReceiptHandler( h.store, resolve_account=self.resolve_account, buyer_agents=self.registry @@ -63,8 +66,7 @@ def __init__(self, h, *, hydrated=False, registry_kind=None, version=None): self.handler.sync_reporting_receipts = self.idempotency.wrap( self.handler.sync_reporting_receipts ) - if version is not None: - self.handler.adcp_version = version + self.handler.get_adcp_version = lambda: self.version def authorize(self, s, *, token="token-one"): account, consumer = s.obligation.account_id, s.binding.consumer_id diff --git a/tests/conformance/reporting/test_reporting_production.py b/tests/conformance/reporting/test_reporting_production.py index 06fcf0cbd..b6310abcc 100644 --- a/tests/conformance/reporting/test_reporting_production.py +++ b/tests/conformance/reporting/test_reporting_production.py @@ -66,7 +66,8 @@ async def test_actual_admission_verified_finish_and_private_polling( assert validator is not None assert not list(validator.iter_errors(before)) assert "sync_reporting_receipts" not in support.handler.advertised_tools_for_instance() - assert await support.activate(account_id=item.config.account_id) + production_operation_1 = await support.activate(account_id=item.config.account_id) + assert production_operation_1 lease = await item.claim() assert isinstance(lease, ReportingMaterializerLease) assert lease.admission_epoch == 2 @@ -121,4 +122,5 @@ async def test_actual_admission_verified_finish_and_private_polling( assert repeat.state == "verified" assert item.writer.writes == 1 await support.aclose() - assert await support.reporting_delivery() == {} + production_operation_2 = await support.reporting_delivery() + assert production_operation_2 == {} diff --git a/tests/conformance/reporting/test_reporting_production_bindings.py b/tests/conformance/reporting/test_reporting_production_bindings.py index ceae59381..6073fc05e 100644 --- a/tests/conformance/reporting/test_reporting_production_bindings.py +++ b/tests/conformance/reporting/test_reporting_production_bindings.py @@ -82,7 +82,8 @@ async def test_full_method_controls_admission_acquisition_and_raw_discovery( finally: support._producer_turn.reset(token) assert producer._source.requests == [] - assert (await source_turn(support)).leased is None + production_operation_3 = await source_turn(support) + assert (production_operation_3).leased is None for transport in ("mcp", "a2a-0.3", "a2a-1.0"): _, caps = await mounted.call( client, "get_adcp_capabilities", {}, transport=transport @@ -144,7 +145,8 @@ async def test_products_are_explicit_for_shared_definition_and_colliding_account assert outside.delivery_config_id == item.config.delivery_config_id # Supported discovery does not need a first trusted account binding. assert not first.producer._source.bindings and not second.producer._source.bindings - assert bool(await support.reporting_delivery()) == (backend == "postgres") + production_condition_1 = bool(await support.reporting_delivery()) == (backend == "postgres") + assert production_condition_1 for config, obligation, binding, offering, product_id in entries: source = offering.producer._source source.rows = item.rows @@ -158,7 +160,8 @@ async def test_products_are_explicit_for_shared_definition_and_colliding_account frozen = await source_documents(h) assert len(frozen) == 3 # Admission fixes bindings even before activation can claim work. - assert (await source_turn(support)).leased is None + production_operation_2 = await source_turn(support) + assert (production_operation_2).leased is None for account in ("acct_a", "acct_b"): await support.activate(account_id=account) for config, obligation, binding, offering, product_id in entries: diff --git a/tests/conformance/reporting/test_reporting_production_lifecycle.py b/tests/conformance/reporting/test_reporting_production_lifecycle.py index a7acd8af2..837332c43 100644 --- a/tests/conformance/reporting/test_reporting_production_lifecycle.py +++ b/tests/conformance/reporting/test_reporting_production_lifecycle.py @@ -137,9 +137,10 @@ async def test_production_official_receipts_adjustment_and_frozen_wire_cycle( assert isinstance(lease, ReportingMaterializerLease) assert lease.attempt.reporting_revision_id == official.reporting_revision_id prepared, verified = await item.verified(lease) - assert ( - await h.store.finish_materialization(lease, prepared=prepared, verified=verified) - ).state == "verified" + production_operation_1 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified + ) + assert (production_operation_1).state == "verified" before_receipts = await generation(h) receipt = ReportingRevisionReceiptRecord( item.scope, @@ -163,7 +164,8 @@ async def test_production_official_receipts_adjustment_and_frozen_wire_cycle( assert rejected["results"][0]["result"] == "recorded", rejected after_rejection = await generation(h) assert after_rejection > before_receipts - assert not isinstance(await item.claim(), ReportingMaterializerLease) + production_condition_2 = not isinstance(await item.claim(), ReportingMaterializerLease) + assert production_condition_2 assert item.writer.writes == 1 # consumer rejection cannot schedule a retry feed_request = { "account": request["account"], diff --git a/tests/conformance/reporting/test_reporting_production_lock_order.py b/tests/conformance/reporting/test_reporting_production_lock_order.py index 276ef6f59..a3b46b4e6 100644 --- a/tests/conformance/reporting/test_reporting_production_lock_order.py +++ b/tests/conformance/reporting/test_reporting_production_lock_order.py @@ -203,12 +203,15 @@ async def acquire(offering, worker, now=END): finally: support._producer_turn.reset(token) - assert await acquire(selected, "preactivation") is None - assert await acquire(other, "preactivation") is None + production_operation_1 = await acquire(selected, "preactivation") + assert production_operation_1 is None + production_operation_2 = await acquire(other, "preactivation") + assert production_operation_2 is None await support.activate(account_id="acct_a") # Two actual source instances intentionally have identical report # contracts. Only explicit, verified offering admission chooses one. - assert await acquire(selected, "unbound") is None + production_operation_3 = await acquire(selected, "unbound") + assert production_operation_3 is None for configuration, offering in ( (item.config, selected), (peer, selected), diff --git a/tests/conformance/reporting/test_reporting_production_migration.py b/tests/conformance/reporting/test_reporting_production_migration.py index 4bf1673e1..0fe0c0bdf 100644 --- a/tests/conformance/reporting/test_reporting_production_migration.py +++ b/tests/conformance/reporting/test_reporting_production_migration.py @@ -102,7 +102,10 @@ async def test_populated_repeat_concurrent_migration_keeps_history_fairness_and_ assert all(current.get(key) == value for key, value in objects.items()) assert original_rows(await h.image(), before) == before assert await fairness(pool) == old_turns - assert await parent.ingest_receipt_batch(request, caller=case.binding.principal) == response + production_operation_1 = await parent.ingest_receipt_batch( + request, caller=case.binding.principal + ) + assert production_operation_1 == response assert ( await walk(child, feed_request(case), case.binding.principal, first=first) == expected ) @@ -188,7 +191,10 @@ async def interrupt(connection, query, *args, **kwargs): assert await schema_objects(c) == original await child.create_schema() assert original_rows(await h.image(), before) == before - assert await parent.ingest_receipt_batch(request, caller=case.binding.principal) == response + production_operation_2 = await parent.ingest_receipt_batch( + request, caller=case.binding.principal + ) + assert production_operation_2 == response @pytest.mark.parametrize( diff --git a/tests/conformance/reporting/test_reporting_production_notifications.py b/tests/conformance/reporting/test_reporting_production_notifications.py index 769c997ba..a8e98f910 100644 --- a/tests/conformance/reporting/test_reporting_production_notifications.py +++ b/tests/conformance/reporting/test_reporting_production_notifications.py @@ -78,7 +78,8 @@ async def queued_production(backend, tmp_path, monkeypatch): revision, rows = revision_for(obligation, suffix="ordinary-core") await h.store.commit_revision(revision, rows) await support.activate(account_id=item.config.account_id) - assert (await support.materializer.run_once()).state == "verified" + production_operation_1 = await support.materializer.run_once() + assert (production_operation_1).state == "verified" assert item.writer.writes == 1 for readable in (False, True): await h.store.set_revision_readable( @@ -192,7 +193,8 @@ async def test_retry_window_and_http_attempt_share_the_reservation_boundary( async with queued_production(backend, tmp_path, monkeypatch) as h: worker = h.workers[2] await pin_current_clock(h) - assert await worker.expand_one(account_id="acct_a") + production_operation_2 = await worker.expand_one(account_id="acct_a") + assert production_operation_2 calls = 0 with monkeypatch.context() as patch: if h.pool is None: @@ -255,7 +257,8 @@ async def test_window_insert_failure_rolls_back_attempt_head_and_all_state( async with queued_production(backend, tmp_path, monkeypatch) as h: await pin_current_clock(h) worker = h.workers[queue] - assert await worker.expand_one(account_id="acct_a") + production_operation_3 = await worker.expand_one(account_id="acct_a") + assert production_operation_3 lease = await worker.outbox.claim_delivery( account_id="acct_a", now=h.clock(), lease_seconds=60 ) @@ -327,36 +330,42 @@ async def test_timeout_backoff_duplicate_workers_and_configuration_change_keep_f if key[0] == "acct_a" and key[1] != "buyer": del h.subscriptions.values[key] worker = h.workers[queue] - assert await worker.expand_one(account_id="acct_a") + production_operation_4 = await worker.expand_one(account_id="acct_a") + assert production_operation_4 h.receiver.responses["buyer"].extend([429, httpx.ReadTimeout("controlled timeout")]) - assert await worker.deliver_one(account_id="acct_a") + production_operation_5 = await worker.deliver_one(account_id="acct_a") + assert production_operation_5 original = await retained_windows(h) assert len(original) == 1 first = (await worker.outbox.list_deliveries(account_id="acct_a"))[0] assert first.state == "pending" duplicate = fresh_workers(h)[queue] - assert await asyncio.gather( + production_operation_6 = await asyncio.gather( worker.deliver_one(account_id="acct_a"), duplicate.deliver_one(account_id="acct_a") - ) == [False, False] + ) + assert production_operation_6 == [False, False] h.clock.advance(timedelta(seconds=5)) # A reconstructed worker with a longer retry interval cannot postpone # expiration or turn it into another day of delivery eligibility. restarted = fresh_workers(h)[queue] restarted.retry_seconds = 2 * 86400 - assert await restarted.deliver_one(account_id="acct_a") + production_operation_7 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_7 assert await retained_windows(h) == original await h.store.put_configuration(replace(h.item.config, deactivated_at=h.clock())) h.signing.generation = 2 use_clock(h, original[0][5] - timedelta(microseconds=1)) - assert not await fresh_workers(h)[queue].deliver_one(account_id="acct_a") + production_operation_8 = await fresh_workers(h)[queue].deliver_one(account_id="acct_a") + assert not production_operation_8 use_clock(h, original[0][5]) current, duplicate = fresh_workers(h)[queue], fresh_workers(h)[queue] - assert sorted( + production_condition_9 = sorted( await asyncio.gather( current.deliver_one(account_id="acct_a"), duplicate.deliver_one(account_id="acct_a"), ) ) == [False, True] + assert production_condition_9 final = (await current.outbox.list_deliveries(account_id="acct_a"))[0] assert final.delivery == first.delivery assert (final.state, final.error_code) == ("suppressed", "lease_expired") @@ -367,7 +376,8 @@ async def test_timeout_backoff_duplicate_workers_and_configuration_change_keep_f assert [r.outcome.status for r in activity] == ["timeout", "failed"] assert [r.attempt for r in activity] == [2, 1] assert len(h.receiver.received) == 1 - assert not await current.deliver_one(account_id="acct_b") + production_operation_10 = await current.deliver_one(account_id="acct_b") + assert not production_operation_10 assert not caplog.records @@ -385,7 +395,8 @@ async def test_worker_timeout_after_reservation_preserves_original_window_and_pe del h.subscriptions.values[key] worker = h.workers[queue] worker.lease_seconds = 1 - assert await worker.expand_one(account_id="acct_a") + production_operation_11 = await worker.expand_one(account_id="acct_a") + assert production_operation_11 barrier = Barrier() h.notification_failures.at("http.before", barrier) task = asyncio.create_task(worker.deliver_one(account_id="acct_a")) @@ -394,14 +405,16 @@ async def test_worker_timeout_after_reservation_preserves_original_window_and_pe original = await retained_windows(h) assert len(original) == 1 use_clock(h, original[0][5]) - assert await asyncio.wait_for(task, 3) + production_operation_19 = await asyncio.wait_for(task, 3) + assert production_operation_19 finally: barrier.release() if not task.done(): task.cancel() await asyncio.gather(task, return_exceptions=True) restarted = fresh_workers(h)[queue] - assert await restarted.deliver_one(account_id="acct_a") + production_operation_12 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_12 assert await retained_windows(h) == original activity = await restarted.outbox.list_activity( account_id="acct_a", consumer_id=h.item.binding.consumer_id @@ -416,7 +429,8 @@ async def test_worker_timeout_after_reservation_preserves_original_window_and_pe async def test_pg_deadline_crossed_while_reserving_an_ordinal_rolls_it_back(tmp_path, monkeypatch): async with queued_production("postgres", tmp_path, monkeypatch) as h: worker = h.workers[2] - assert await worker.expand_one(account_id="acct_a") + production_operation_13 = await worker.expand_one(account_id="acct_a") + assert production_operation_13 h.notification_failures.at("http.accepted", SimulatedCrash()) with pytest.raises(SimulatedCrash): await worker.deliver_one(account_id="acct_a") @@ -434,7 +448,8 @@ async def cross_deadline(*args): return ordinal monkeypatch.setattr(restarted.outbox, "_next_attempt_on", cross_deadline) - assert await restarted.deliver_one(account_id="acct_a") + production_operation_14 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_14 assert len(h.receiver.received) == 1 assert await retained_windows(h) == original assert ( @@ -453,7 +468,8 @@ async def test_retry_horizon_is_immutable_across_crash_rotation_and_restart( ): async with queued_production(backend, tmp_path, monkeypatch) as h: worker = h.workers[queue] - assert await worker.expand_one(account_id="acct_a") + production_operation_15 = await worker.expand_one(account_id="acct_a") + assert production_operation_15 h.notification_failures.at("http.accepted", SimulatedCrash()) with pytest.raises(SimulatedCrash): await worker.deliver_one(account_id="acct_a") @@ -468,7 +484,8 @@ async def test_retry_horizon_is_immutable_across_crash_rotation_and_restart( use_clock(h, row[5] + timedelta(microseconds=offset)) h.signing.generation = 2 restarted = fresh_workers(h)[queue] - assert await restarted.deliver_one(account_id="acct_a") + production_operation_16 = await restarted.deliver_one(account_id="acct_a") + assert production_operation_16 received_count = len(h.receiver.received) assert received_count == (2 if offset < 0 else 1) assert await retained_windows(h) == original @@ -623,7 +640,8 @@ async def fail(*args, **kwargs): assert not await worker.outbox.list_deliveries(account_id="acct_a") await expire_queue_lease(h, worker, expansion=True) restarted = fresh_workers(h)[queue] - assert await restarted.expand_one(account_id="acct_a") + production_operation_17 = await restarted.expand_one(account_id="acct_a") + assert production_operation_17 assert len(await restarted.outbox.list_deliveries(account_id="acct_a")) == len(deliveries) @@ -643,7 +661,8 @@ async def test_current_signing_failure_never_sends_or_exposes_private_data( async with queued_production(backend, tmp_path, monkeypatch) as h: worker = h.workers[queue] - assert await worker.expand_one(account_id="acct_a") + production_operation_18 = await worker.expand_one(account_id="acct_a") + assert production_operation_18 quarantine = await h.queue() async def incompatible(**kwargs): @@ -674,7 +693,8 @@ async def incompatible(**kwargs): with pytest.raises(asyncio.CancelledError): await worker.deliver_one(account_id="acct_a") else: - assert await worker.deliver_one(account_id="acct_a") + production_operation_20 = await worker.deliver_one(account_id="acct_a") + assert production_operation_20 assert not h.receiver.received assert not caplog.records deliveries = await worker.outbox.list_deliveries(account_id="acct_a") diff --git a/tests/conformance/reporting/test_reporting_production_packaging.py b/tests/conformance/reporting/test_reporting_production_packaging.py index b529628d7..40119eb53 100644 --- a/tests/conformance/reporting/test_reporting_production_packaging.py +++ b/tests/conformance/reporting/test_reporting_production_packaging.py @@ -6,7 +6,7 @@ import pytest -from ._generation_support import assert_c_collated_rolling_database +from ._generation_support import require_rolling_database from ._production_packaging import installed_production from .test_reporting_materializer_packaging import b1_wheels, built_distribution from .test_reporting_notification_packaging import run_step @@ -21,7 +21,7 @@ async def test_floor_installed_production_contract(request, kind, drivers): if interpreter is None: pytest.skip("ADCP_PYTHON310 supplies the installed floor runtime") if drivers: - assert_c_collated_rolling_database() + require_rolling_database() root, wheels, _ = request.getfixturevalue("b1_wheels") _, _, source = request.getfixturevalue("built_distribution") label = kind + ("-pg" if drivers else "-base") diff --git a/tests/conformance/reporting/test_reporting_production_progress.py b/tests/conformance/reporting/test_reporting_production_progress.py index 8fbab7ca4..741076493 100644 --- a/tests/conformance/reporting/test_reporting_production_progress.py +++ b/tests/conformance/reporting/test_reporting_production_progress.py @@ -108,9 +108,10 @@ async def test_busy_account_window_advances_wraps_and_revisits_after_unlock( assert first.leased is None and samples[-1] == 32 # Another producer has its own hint; its blocked turn cannot erase # the first producer's progress beyond the full 32-row window. - assert ( - await bounded_turn(lambda: lease_source(support, "other", index=1), samples) is None + production_operation_1 = await bounded_turn( + lambda: lease_source(support, "other", index=1), samples ) + assert production_operation_1 is None assert await h.image() == before second = await bounded_turn(lambda: source_turn(support), samples) assert second.leased is not None, "locked prefix hid the eligible second account" @@ -122,13 +123,16 @@ async def test_busy_account_window_advances_wraps_and_revisits_after_unlock( assert source.requests[0].identity.account_id == "acct_b" assert not support.offerings[1].producer._source.requests # Successful acquisition resets discovery to the durable rank. - assert (await bounded_turn(lambda: source_turn(support), samples)).leased is None + production_operation_2 = await bounded_turn(lambda: source_turn(support), samples) + assert (production_operation_2).leased is None held = await bounded_turn(lambda: lease_source(support, "tail-held"), samples) assert held is not None and held.generation_key == outside.generation_key held_image = await h.image() - assert (await bounded_turn(lambda: source_turn(support), samples)).leased is None + production_operation_3 = await bounded_turn(lambda: source_turn(support), samples) + assert (production_operation_3).leased is None start = len(samples) - assert (await bounded_turn(lambda: source_turn(support), samples)).leased is None + production_operation_4 = await bounded_turn(lambda: source_turn(support), samples) + assert (production_operation_4).leased is None assert samples[start:] == [0, 32] # empty tail wraps once, never an unbounded scan assert await h.image() == held_image try: @@ -199,7 +203,8 @@ async def test_fresh_stores_and_concurrent_workers_preserve_bounded_sampling_and blocked, outside = await sample_configurations(h, count=65) async with h.pool.connection() as holder, holder.transaction(): await h.store._lock_account(holder, "acct_a") - assert (await source_turn(h.production)).leased is None + production_operation_5 = await source_turn(h.production) + assert (production_operation_5).leased is None await h.production.aclose() bindings = [ (c, h.production.offerings[0].offering_id, "catalog-7391") for c in [*blocked, outside] @@ -543,7 +548,8 @@ async def test_bounded_producer_advances_past_processed_first_window(backend, tm assert child.process.returncode == -9 async with restarted_process(h, path, pause=False) as child: third = await child.event("done") - assert await asyncio.wait_for(child.process.wait(), 10) == 0 + production_operation_6 = await asyncio.wait_for(child.process.wait(), 10) + assert production_operation_6 == 0 assert len(second["obligations"]) == len(second["revisions"]) == 64 assert len(third["obligations"]) == len(third["revisions"]) == 3 executions = first_document["executions"] + second["executions"] + third["executions"] diff --git a/tests/conformance/reporting/test_reporting_production_readiness.py b/tests/conformance/reporting/test_reporting_production_readiness.py index b0419c1bd..831327c36 100644 --- a/tests/conformance/reporting/test_reporting_production_readiness.py +++ b/tests/conformance/reporting/test_reporting_production_readiness.py @@ -26,12 +26,14 @@ async def test_warm_proof_rechecks_the_actual_mount(transport, mutation, tmp_pat if mutation == "replace": mapping["get_reporting_status"] = mapping["get_adcp_capabilities"] try: - assert await support.reporting_delivery() == {} + production_operation_5 = await support.reporting_delivery() + assert production_operation_5 == {} with pytest.raises(ReportingNotificationError, match="component_unready"): await support.activate(account_id=h.item.config.account_id) finally: mapping["get_reporting_status"] = original - assert (await support.reporting_delivery())["managed_delivery"] is True + production_operation_1 = await support.reporting_delivery() + assert (production_operation_1)["managed_delivery"] is True async def test_shared_scan_cancellation_and_post_scan_dynamic_check(tmp_path, monkeypatch): @@ -62,11 +64,13 @@ async def paused(*args, **kwargs): removed = h.mount._tool_manager._tools.pop("get_reporting_status") try: release.set() - assert await asyncio.wait_for(asyncio.gather(*callers), 10) == [{}] * 4 + production_operation_8 = await asyncio.wait_for(asyncio.gather(*callers), 10) + assert production_operation_8 == [{}] * 4 finally: h.mount._tool_manager._tools["get_reporting_status"] = removed assert scans == 1 - assert (await support.reporting_delivery())["managed_delivery"] is True + production_operation_6 = await support.reporting_delivery() + assert (production_operation_6)["managed_delivery"] is True await h.store.materializer_ready() assert scans == 1 @@ -286,7 +290,8 @@ async def test_optional_delivery_is_owned_and_broken_enabled_chain_fails_closed( outbox = worker.outbox worker.outbox = support.notification_workers[0].outbox try: - assert await support.reporting_delivery() == {} + production_operation_7 = await support.reporting_delivery() + assert production_operation_7 == {} with pytest.raises(ReportingNotificationError, match="notification_chain_unready"): await support.activate(account_id=h.item.config.account_id) finally: @@ -300,7 +305,8 @@ async def test_optional_delivery_is_owned_and_broken_enabled_chain_fails_closed( } await support.aclose() assert support._notification_task is None - assert await support.reporting_delivery() == {} + production_operation_2 = await support.reporting_delivery() + assert production_operation_2 == {} async def test_warm_catalog_proof_does_not_cache_signing_wiring(tmp_path, monkeypatch): @@ -310,7 +316,8 @@ async def test_warm_catalog_proof_does_not_cache_signing_wiring(tmp_path, monkey "postgres", tmp_path / "destination.sqlite", notifications=True, notification_delivery=True ) as h: support = h.production - assert (await support.reporting_delivery())["managed_delivery"] + production_operation_3 = await support.reporting_delivery() + assert (production_operation_3)["managed_delivery"] mount = MountedProduction(h) mount.authorize(h.item) @@ -378,4 +385,5 @@ async def incompatible(**kwargs): await h.production.activate(account_id="acct_a") assert await h.image() == before assert not caplog.records - assert await h.production.activate(account_id="acct_a") + production_operation_4 = await h.production.activate(account_id="acct_a") + assert production_operation_4 diff --git a/tests/conformance/reporting/test_reporting_production_restart.py b/tests/conformance/reporting/test_reporting_production_restart.py index ec30f2ebb..b4c39f416 100644 --- a/tests/conformance/reporting/test_reporting_production_restart.py +++ b/tests/conformance/reporting/test_reporting_production_restart.py @@ -75,7 +75,8 @@ async def test_pg_sigkill_after_accepted_http_keeps_first_retry_window( for key in tuple(h.subscriptions.values): if key[0] == "acct_a" and key[1] != "buyer": del h.subscriptions.values[key] - assert await h.workers[queue].expand_one(account_id="acct_a") + production_operation_1 = await h.workers[queue].expand_one(account_id="acct_a") + assert production_operation_1 async with delivery_child(h, tmp_path, queue=queue, pause=True, at=h.clock()) as child: first = await child.event("accepted_before_ack") assert len(await retained_windows(h)) == 1 @@ -85,7 +86,8 @@ async def test_pg_sigkill_after_accepted_http_keeps_first_retry_window( at = datetime.fromisoformat(first["expires_at"]) + timedelta(microseconds=offset) async with delivery_child(h, tmp_path, queue=queue, pause=False, at=at) as child: restored = await child.event("done") - assert await asyncio.wait_for(child.process.wait(), 10) == 0 + production_operation_2 = await asyncio.wait_for(child.process.wait(), 10) + assert production_operation_2 == 0 assert restored["http_calls"] == int(offset < 0) assert restored["activity_count"] == (2 if offset < 0 else 1) assert (restored["state"], restored["error_code"]) == ( diff --git a/tests/conformance/reporting/test_reporting_production_rolling.py b/tests/conformance/reporting/test_reporting_production_rolling.py index afc99fbb9..dda49a44a 100644 --- a/tests/conformance/reporting/test_reporting_production_rolling.py +++ b/tests/conformance/reporting/test_reporting_production_rolling.py @@ -33,7 +33,7 @@ from .test_reporting_notification_packaging import ROOT, run_step __all__ = ["approved_b23", "b1_wheels", "built_distribution", "feed_wheels", "installed_feed"] -HARDENING = "a09878f67ab397a4b51b3314e3e8a5e87cf96da5" +HARDENING = "e16eb8cf3074cabd45aab42840950f05ad6d2b43" @pytest.fixture(scope="module", params=["b23", "hardening"]) @@ -86,7 +86,7 @@ def production_parent(request, tmp_path_factory): **identity, "modules": modules, "python": [3, 10], - "tree": "528cf5fddb61c72a284dc0a9b22b169752db7691", + "tree": "c043d1e14c5071859f566e07cc9980058fa6ee07", }, wheel, ) @@ -367,7 +367,8 @@ async def test_actual_parent_page_one_to_installed_activation_sigkill_and_comple ) async with installed_child(python, script, settings, tmp_path) as child: result = await child.event("done") - assert await asyncio.wait_for(child.process.wait(), 10) == 0 + production_operation_1 = await asyncio.wait_for(child.process.wait(), 10) + assert production_operation_1 == 0 assert result["legacy"] == { "pages": expected[0][1:], "binding": snapshot.binding, diff --git a/tests/conformance/reporting/test_reporting_production_settling.py b/tests/conformance/reporting/test_reporting_production_settling.py new file mode 100644 index 000000000..0daa23858 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_production_settling.py @@ -0,0 +1,196 @@ +"""Production progress must retain unfinished source restatement policy across turns.""" + +from datetime import timedelta +from functools import partial + +import pytest + +from adcp.reporting.inline_source import InlineReportingSource +from adcp.reporting.source import ( + ReportingSourceCapabilitiesV1, + reporting_source_capabilities_sha256_v1, +) + +from ._generation_support import END +from ._materializer_support import reference_rows +from ._production_support import Source, SQLiteSeals, production_harness +from .test_reporting_production_lock_order import source_turn + + +class SettlingSource(Source): + """Declare the complete policy before the production service binds its hash.""" + + def __init__(self, key, path, rows=None, *, close_officially=True, **kwargs): + super().__init__(key, path, rows, **kwargs) + raw = self.capabilities.model_dump(mode="json") + raw["offerings"][0].update( + restatement_window="PT3H", + restatement_cadence="PT1H", + official_close_lag="PT4H" if close_officially else None, + ) + self.official_source_id = None + if close_officially: + official = Source(key, path.with_name("official-contract"), rows, official=True) + self.official_source_id = official.source_id + raw["offerings"].append(official.capabilities.offerings[0].model_dump(mode="json")) + raw["capabilities_sha256"] = reporting_source_capabilities_sha256_v1(raw) + self.capabilities = ReportingSourceCapabilitiesV1.model_validate(raw) + self.official_ready = False + self.inline = InlineReportingSource( + capabilities=self.capabilities, + fetch=self.fetch, + staging=self.inline.staging, + seals=SQLiteSeals(path.with_suffix(".seals")), + constituent_of=lambda row, req: req.coverage.constituents[0].constituent_id, + clock=kwargs.get("clock") or (lambda: END), + ) + + async def fetch(self, request): + if request.publication_class == "AUTHORITATIVE" and not self.official_ready: + self.requests.append(request) + return None + return await super().fetch(request) + + +async def revisions(h): + return await h.store.list_revisions( + account_id=h.item.config.account_id, + reporting_obligation_id=h.item.obligation.reporting_obligation_id, + ) + + +async def pending(h): + producer = h.production.offerings[0].producer + token = h.production._producer_turn.set(producer) + try: + return await h.store.next_producer_obligations( + h.item.config, now=h.source_clock(), limit=64 + ) + finally: + h.production._producer_turn.reset(token) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("close_officially", [False, True]) +async def test_progress_retains_policy_until_terminal_publication( + backend, close_officially, tmp_path +): + async with production_harness( + backend, + tmp_path / "destination.sqlite", + count=1, + source_publication=True, + periods=1, + source_factory=partial(SettlingSource, close_officially=close_officially), + ) as h: + await h.production.activate(account_id=h.item.config.account_id) + source = h.production.offerings[0].producer._source + first = await source_turn(h.production) + assert len(first.revisions_committed) == 1 + assert [r.finality for r in await revisions(h)] == ["snapshot"] + production_operation_1 = await pending(h) + assert production_operation_1 == (h.item.obligation.reporting_obligation_id,) + + h.source_clock.now = END + timedelta(minutes=59) + await source_turn(h.production) + assert len(source.requests) == 1 + h.source_clock.now = END + timedelta(hours=1) + unchanged = await source_turn(h.production) + assert not unchanged.revisions_committed + assert len(source.requests) == 2 + checkpoint = await h.store.get_restatement_checkpoint( + account_id=h.item.config.account_id, + reporting_obligation_id=h.item.obligation.reporting_obligation_id, + ) + assert checkpoint is not None and checkpoint.next_observation == 2 + await source_turn(h.production) + assert len(source.requests) == 2 + + source.rows = reference_rows(2) + h.source_clock.now = END + timedelta(hours=2) + changed = await source_turn(h.production) + assert len(changed.revisions_committed) == 1 + history = await revisions(h) + assert len(history) == 2 + assert history[1].supersedes_reporting_revision_id == history[0].reporting_revision_id + + h.source_clock.now = END + timedelta(hours=3) + await source_turn(h.production) + assert len(source.requests) == 3 + if not close_officially: + production_operation_5 = await pending(h) + assert production_operation_5 == () + return + production_operation_2 = await pending(h) + assert production_operation_2 == (h.item.obligation.reporting_obligation_id,) + h.source_clock.now = END + timedelta(hours=4) + not_ready = await source_turn(h.production) + assert not not_ready.revisions_committed + production_operation_3 = await pending(h) + assert production_operation_3 == (h.item.obligation.reporting_obligation_id,) + source.official_ready = True + completed = await source_turn(h.production) + assert len(completed.revisions_committed) == 1 + assert [r.finality for r in await revisions(h)] == ["snapshot", "snapshot", "official"] + production_operation_4 = await pending(h) + assert production_operation_4 == () + count = len(source.requests) + await source_turn(h.production) + assert len(source.requests) == count + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_settling_checkpoint_and_pending_work_survive_fresh_service(backend, tmp_path): + path = tmp_path / "restart.sqlite" + async with production_harness( + backend, + path, + count=1, + source_publication=True, + periods=1, + source_factory=SettlingSource, + ) as h: + await h.production.activate(account_id=h.item.config.account_id) + initial = await source_turn(h.production) + assert len(initial.revisions_committed) == 1 + h.source_clock.now = END + timedelta(hours=1) + noop = await source_turn(h.production) + assert not noop.revisions_committed + original_source = h.production.offerings[0].producer._source + assert len(original_source.requests) == 2 + prior_keys = {r.identity.source_execution_key for r in original_source.requests} + await h.production.aclose() + existing = {"existing_store": h.store} if h.pool is None else {"existing_pool": h.pool} + async with production_harness( + backend, + path, + count=1, + source_publication=True, + periods=1, + source_factory=SettlingSource, + **existing, + ) as fresh: + source = fresh.production.offerings[0].producer._source + assert source is not original_source + if h.pool is not None: + assert fresh.store is not h.store + checkpoint = await fresh.store.get_restatement_checkpoint( + account_id=fresh.item.config.account_id, + reporting_obligation_id=fresh.item.obligation.reporting_obligation_id, + ) + assert checkpoint is not None and checkpoint.next_observation == 2 + assert not source.requests + source.rows = reference_rows(2) + fresh.source_clock.now = END + timedelta(hours=2) + changed = await source_turn(fresh.production) + assert len(changed.revisions_committed) == 1 + assert len(source.requests) == 1 + assert source.requests[0].identity.source_execution_key not in prior_keys + checkpoint = await fresh.store.get_restatement_checkpoint( + account_id=fresh.item.config.account_id, + reporting_obligation_id=fresh.item.obligation.reporting_obligation_id, + ) + assert checkpoint is not None and checkpoint.next_observation == 3 + assert len(await revisions(fresh)) == 2 + unfinished = await pending(fresh) + assert unfinished == (fresh.item.obligation.reporting_obligation_id,) diff --git a/tests/conformance/reporting/test_reporting_production_transactions.py b/tests/conformance/reporting/test_reporting_production_transactions.py index cfd739cb3..ab07902e8 100644 --- a/tests/conformance/reporting/test_reporting_production_transactions.py +++ b/tests/conformance/reporting/test_reporting_production_transactions.py @@ -111,14 +111,16 @@ def fail(self, *args, **kwargs): assert ( await walk(h.store, feed_request(item), item.binding.principal, first=first) == frozen ) - assert (await finish()).state == "verified" + production_operation_1 = await finish() + assert (production_operation_1).state == "verified" assert len(await item.outcomes()) == 1 assert len(await h.store.read_production_boundaries(caller=item.binding.principal)) == 1 assert await h.store.read_materializer_boundaries(caller=item.binding.principal) == () assert len((await production_queue(h))[0]) == int(notifications) assert await h.queue() == old_queue and await h.ordinary_events() == ordinary committed = await h.image() - assert (await finish()).state == "verified" + production_operation_2 = await finish() + assert (production_operation_2).state == "verified" assert await h.image() == committed assert item.writer.writes == 1 @@ -179,7 +181,8 @@ def enqueue(self, event): await finish() assert await h.image() == before assert await production_queue(h) == ((), ()) - assert (await finish()).state == "verified" + production_operation_3 = await finish() + assert (production_operation_3).state == "verified" events, work = await production_queue(h) assert len(events) == 1 and work == ("pending",) @@ -257,9 +260,8 @@ async def observe_finish(self, lease, **kwargs): == 1 ) frozen = await fresh.image() - assert ( - await fresh.store.finish_materialization(lease, **arguments) - ).state == "verified" + production_operation_5 = await fresh.store.finish_materialization(lease, **arguments) + assert (production_operation_5).state == "verified" assert await fresh.image() == frozen await drain(fresh.projection, item.config.account_id) assert await production_queue(fresh) == ((), ()) @@ -299,11 +301,13 @@ async def acquire(worker, seconds): crashed = await acquire("crashed-producer", 0) assert crashed is not None - assert await acquire("duplicate-producer", 0) is None + production_operation_6 = await acquire("duplicate-producer", 0) + assert production_operation_6 is None recovered = await acquire("recovered-producer", 2) assert recovered is not None await h.store.release_period_close(crashed, worker_id="crashed-producer") - assert await acquire("duplicate-producer", 2) is None + production_operation_7 = await acquire("duplicate-producer", 2) + assert production_operation_7 is None await h.store.release_period_close(recovered, worker_id="recovered-producer") for seconds in (3, 4): repeated = await acquire("scheduled-producer", seconds) @@ -335,10 +339,10 @@ async def acquire(worker, seconds): assert len((await production_queue(h))[0]) == int(notifications and not source_changes) assert await h.queue() == ((), ()) committed = await h.image() - assert ( - await h.store.finish_materialization(lease, prepared=prepared, verified=verified) - == result + production_operation_4 = await h.store.finish_materialization( + lease, prepared=prepared, verified=verified ) + assert production_operation_4 == result assert await h.image() == committed @@ -399,8 +403,9 @@ async def acquire(): await h.store.release_period_close(lease, worker_id="bookkeeping-fault") assert await h.image() == image prepared, verified = await h.item.verified(pending) - assert ( - await h.store.finish_materialization(pending, prepared=prepared, verified=verified) - ).state == "verified" + production_operation_8 = await h.store.finish_materialization( + pending, prepared=prepared, verified=verified + ) + assert (production_operation_8).state == "verified" finally: h.production._producer_turn.reset(token) diff --git a/tests/conformance/reporting/test_reporting_production_worker_failure.py b/tests/conformance/reporting/test_reporting_production_worker_failure.py index 337aa7203..60ebef1d0 100644 --- a/tests/conformance/reporting/test_reporting_production_worker_failure.py +++ b/tests/conformance/reporting/test_reporting_production_worker_failure.py @@ -163,7 +163,8 @@ async def fail(*args, **kwargs): await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), 5) assert all(t.done() for t in tasks) assert not [r for r in caplog.records if r.name == "adcp.reporting.production"] - assert await support.reporting_delivery() == {} + production_operation_1 = await support.reporting_delivery() + assert production_operation_1 == {} finally: release.set() support._stop.set() @@ -206,7 +207,8 @@ def broken_sink(record): assert support._failed and support._stop.is_set() assert len(observed) == 1 assert "canary" not in json.dumps(observed[0].__dict__, default=str) - assert await support.reporting_delivery() == {} + production_operation_2 = await support.reporting_delivery() + assert production_operation_2 == {} with pytest.raises(ReportingNotificationError, match="component_unready"): await support.activate(account_id=h.item.config.account_id) finally: @@ -287,7 +289,8 @@ async def notification_wait(*args, **kwargs): if closing is not None: await asyncio.wait_for(closing, 5) assert not [r for r in caplog.records if r.name == "adcp.reporting.production"] - assert await support.reporting_delivery() == {} + production_operation_3 = await support.reporting_delivery() + assert production_operation_3 == {} finally: producer_release.set() notification_release.set() diff --git a/tests/conformance/reporting/test_reporting_projection_capture.py b/tests/conformance/reporting/test_reporting_projection_capture.py index 58fd5428b..5ac5be4f3 100644 --- a/tests/conformance/reporting/test_reporting_projection_capture.py +++ b/tests/conformance/reporting/test_reporting_projection_capture.py @@ -21,7 +21,8 @@ async def test_captured_receipt_adjustment_and_readability_cycles_are_not_collap h = projections s = await receipt_case(h) account = s.obligation.account_id - assert await h.projection.activate(account_id=account) + production_operation_1 = await h.projection.activate(account_id=account) + assert production_operation_1 assert await h.projection.baseline_ready(account_id=account) starting = next( c.generation @@ -81,7 +82,8 @@ async def test_captured_receipt_adjustment_and_readability_cycles_are_not_collap before = await h.image() await h.store.ingest_receipt_batch(request_for(s), caller=s.binding.principal) assert await h.image() == before - assert not await h.projection.activate(account_id=account) + production_operation_2 = await h.projection.activate(account_id=account) + assert not production_operation_2 async def test_activation_preserves_legacy_snapshot_and_new_pages_have_exact_local_ownership( @@ -92,7 +94,8 @@ async def test_activation_preserves_legacy_snapshot_and_new_pages_have_exact_loc req = feed_request(s) first = await h.store.read_reporting_feed(req, caller=s.binding.principal) legacy = await walk(h.store, req, s.binding.principal, first=first) - assert await h.projection.activate(account_id=s.obligation.account_id) + production_operation_3 = await h.projection.activate(account_id=s.obligation.account_id) + assert production_operation_3 resumed = await walk(h.store, req, s.binding.principal, first=first) assert resumed == legacy new = await walk(h.store, req, s.binding.principal) diff --git a/tests/conformance/reporting/test_reporting_projection_history.py b/tests/conformance/reporting/test_reporting_projection_history.py index 0a8b0949f..6a69890ad 100644 --- a/tests/conformance/reporting/test_reporting_projection_history.py +++ b/tests/conformance/reporting/test_reporting_projection_history.py @@ -142,13 +142,15 @@ async def history_replay(h, monkeypatch, *, legacy_baseline): from adcp.reporting.outbox.status_pg import PgStatusNotificationStore old = PgStatusNotificationStore(h.store) - assert await old.baseline(account_id=account) + production_operation_5 = await old.baseline(account_id=account) + assert production_operation_5 baselines = await old.checkpoints(account_id=account) assert baselines original_queue = await h.queue() originals = await original_inputs(h, (first, second)) assert len(originals) == 5 - assert await h.projection._begin_activation(account_id=account) + production_operation_1 = await h.projection._begin_activation(account_id=account) + assert production_operation_1 assert not await h.projection.baseline_ready(account_id=account) if h.pool is None: archived = h.store._projection_accounts[account].baselines @@ -165,7 +167,8 @@ async def history_replay(h, monkeypatch, *, legacy_baseline): ).fetchall() ) assert archived == {checkpoint_key(c): checkpoint_document(c) for c in baselines} - assert (await h.projection.project_one(account_id=account)).did_work + production_operation_2 = await h.projection.project_one(account_id=account) + assert (production_operation_2).did_work before = await history_image(h, account) assert before # An interrupted invocation has a committed first input; the next complete @@ -206,7 +209,8 @@ def failure(*args, **kwargs): consumer_status_enabled=False, revision_ownership=True, ) - assert not await projection.activate(account_id=account) + production_operation_3 = await projection.activate(account_id=account) + assert not production_operation_3 assert await projection.baseline_ready(account_id=account) history = await history_image(h, account) personal = [ @@ -240,7 +244,8 @@ def failure(*args, **kwargs): >= 4 ) assert await original_inputs(h, (first, second)) == originals - assert not await projection.activate(account_id=account) + production_operation_4 = await projection.activate(account_id=account) + assert not production_operation_4 async def test_missing_retained_capture_refuses_activation(projections): diff --git a/tests/conformance/reporting/test_reporting_projection_notifications.py b/tests/conformance/reporting/test_reporting_projection_notifications.py index 9cb55f5f1..83b2604f4 100644 --- a/tests/conformance/reporting/test_reporting_projection_notifications.py +++ b/tests/conformance/reporting/test_reporting_projection_notifications.py @@ -54,9 +54,10 @@ async def fail(connection, event): assert len(events) == turn.events assert all(e.notification_type == "reporting.status_changed" for e in events) assert await old.list_events(account_id=account) == () - assert ( - await old.claim_expansion(account_id=account, now=h.clock(), lease_seconds=30) is None + production_operation_1 = await old.claim_expansion( + account_id=account, now=h.clock(), lease_seconds=30 ) + assert production_operation_1 is None assert await h.queue() == original_readiness # Persist a real expansion lease, then verify a competing incarnation # cannot claim it and a separately constructed new outbox resumes it. @@ -68,5 +69,6 @@ async def fail(connection, event): assert await current.list_events(account_id=account) == events assert await h.queue() == original_readiness await h.store.ingest_receipt_batch(request_for(case), caller=case.binding.principal) - assert not (await h.projection.project_one(account_id=account)).did_work + production_operation_2 = await h.projection.project_one(account_id=account) + assert not (production_operation_2).did_work assert await current.list_events(account_id=account) == events diff --git a/tests/conformance/reporting/test_reporting_projection_rc6.py b/tests/conformance/reporting/test_reporting_projection_rc6.py new file mode 100644 index 000000000..6f74f43a7 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_rc6.py @@ -0,0 +1,581 @@ +"""The rc.6 forecast contract at producer, frozen store and public boundaries.""" + +from __future__ import annotations + +from copy import deepcopy +from dataclasses import replace +from datetime import datetime, timedelta, timezone +from types import SimpleNamespace + +import pytest +from jsonschema.validators import validator_for + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed.errors import ReportingFeedError +from adcp.reporting.ledger import ProducerOfferings, ReportingProducer, ReportingScheduleSpec +from adcp.reporting.ledger.delivery_models import ReportingDeliveryPrincipal +from adcp.reporting.ledger.status import ReportingStatusCaller, ReportingStatusHandler +from adcp.types import GetReportingStatusRequest +from adcp.validation.schema_loader import get_named_validator, get_validator + +from ._feed_support import MountedFeed, feed_harness, walk +from ._generation_support import START, UncalledSource, configuration, revision_for +from ._projection_support import projection_harness +from ._receipt_transport import error_code +from .test_reporting_schedule_schema import formats + +RC3 = "3.2-rc.3" +RC6 = "3.2-rc.6" +CONSUMER = "https://buyer.example.test/agent" + + +@pytest.fixture(autouse=True) +def _a2a_compat_send_and_aggregate(): + # Use the real public async-generator transport instead of the unit mock shim. + pass + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC6]) +async def test_production_discovery_advertises_its_usable_reporting_pin(backend, version, tmp_path): + from ._production_support import production_harness + from ._production_transport import MountedProduction + + async with production_harness( + backend, tmp_path / "discovery.sqlite", adcp_version=version + ) as h: + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client: + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, caps = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert caps["adcp"]["supported_versions"] == [version] + assert caps["adcp_version"] == version + + +@pytest.mark.parametrize("version", ["3.0", "3.1", RC3]) +async def test_production_mount_rejects_releases_without_reporting_schemas(version, tmp_path): + from adcp.exceptions import ConfigurationError + + from ._production_support import production_harness + + with pytest.raises(ConfigurationError): + async with production_harness( + "memory", tmp_path / "unsupported.sqlite", adcp_version=version + ): + pass + + +@pytest.mark.parametrize("mutation", ["value", "method"]) +async def test_warm_production_proof_cannot_hide_a_changed_protocol_pin( + mutation, tmp_path, monkeypatch +): + from ._production_support import production_harness + from ._production_transport import MountedProduction + + async with production_harness("postgres", tmp_path / "pin.sqlite", adcp_version=RC6) as h: + mounted = MountedProduction(h) + mounted.authorize(h.item) + async with mounted.client() as client: + _, before = await mounted.call(client, "get_adcp_capabilities", {}) + assert before["media_buy"]["reporting_delivery"]["managed_delivery"] + if mutation == "value": + monkeypatch.setattr(h.production.handler, "_adcp_version", RC3) + else: + monkeypatch.setattr(h.production.handler, "get_adcp_version", lambda: RC3) + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + _, after = await mounted.call( + client, "get_adcp_capabilities", {}, transport=transport + ) + assert after["adcp_version"] == RC6 + assert after["adcp"]["supported_versions"] == [RC6] + assert not after.get("media_buy", {}).get("reporting_delivery") + + +async def scheduled(h, *, now=0.5, complete=True): + h.clock.now = START + timedelta(hours=now) + h.store._clock = h.clock + config = replace(configuration(), deactivated_at=None) + await h.store.put_configuration(config) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + obligations = await producer.close_elapsed_periods(config, now=h.clock()) + if complete: + for obligation in obligations: + revision, rows = revision_for(obligation, suffix=obligation.reporting_obligation_id) + await h.store.commit_revision(revision, rows) + if hasattr(h, "projection"): + await h.projection.activate(account_id=config.account_id) + return config, obligations + + +def mount(h, config, version): + mounted = MountedFeed(h, version=version, hydrated=True, registry_kind="oauth") + mounted.authorize( + SimpleNamespace(obligation=config, binding=SimpleNamespace(consumer_id=CONSUMER)) + ) + return mounted + + +def request(version, view="summary"): + return {"adcp_version": version, "account": {"account_id": "acct_a"}, "view": view} + + +def capture_clock(h, monkeypatch): + if h.pool is not None: + from adcp.reporting.feed import pg + + async def now(connection): + # Same account-locked SQL read with an explicit conformance instant. + # No projection, persisted snapshot or producer is substituted. + return ( + await (await connection.execute("SELECT %s::timestamptz", (h.clock(),))).fetchone() + )[0] + + monkeypatch.setattr(pg, "_now", now) + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_complete_forecast_uses_period_start_without_creating_future_work( + backend, notifications +): + async with feed_harness(backend, notifications=notifications) as h: + config, obligations = await scheduled(h) + assert obligations == [] + before = await h.image() + captured = await h.store.read_status_snapshot(account_id=config.account_id) + handler = ReportingStatusHandler(h.store) + caller = ReportingStatusCaller(config.account_id, CONSUMER) + old = handler.render_snapshot(request(RC3), caller=caller, snapshot=captured) + new = handler.render_snapshot(request(RC6), caller=caller, snapshot=captured) + default = handler.render_snapshot({}, caller=caller, snapshot=captured) + assert old["next_expected_at"] == "2026-09-01T02:00:00Z" + assert new["next_expected_at"] == default["next_expected_at"] == "2026-09-01T01:00:00Z" + for field in ("health", "scope", "coverage", "obligation_counts", "issues", "ledger_as_of"): + assert old[field] == new[field] == default[field] + assert new["health"] == "complete" + assert new["obligation_counts"]["total"] == 0 + assert new["coverage"]["media_buy_ids"] == [] + assert new["ledger_snapshot_id"] != old["ledger_snapshot_id"] + for relative in ( + "media-buy/get-reporting-status-response.json", + "bundled/media-buy/get-reporting-status-response.json", + ): + raw = get_named_validator(relative, version=RC6) + assert raw is not None + raw.validate(new) + assert await h.image() == before + await h.store.put_configuration(replace(config, deactivated_at=START)) + h.clock.now += timedelta(hours=8) + assert handler.render_snapshot(request(RC6), caller=caller, snapshot=captured) == new + assert handler.render_snapshot(request(RC3), caller=caller, snapshot=captured) == old + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("mode", ["core", "projection"]) +@pytest.mark.parametrize("version", [RC6]) +@pytest.mark.parametrize("complete", [False, True]) +async def test_mounted_summary_periods_schema_and_client_use_the_same_pin( + backend, mode, version, complete, monkeypatch +): + factory = feed_harness if mode == "core" else projection_harness + async with factory(backend) as h: + config, obligations = await scheduled(h, now=0.5 if complete else 1.5, complete=complete) + capture_clock(h, monkeypatch) + mounted = mount(h, config, version) + before_work = await h.works() + async with mounted.client() as client: + _, inventory = await mounted.mcp(client, inventory=True) + schema = next( + t["outputSchema"] for t in inventory["tools"] if t["name"] == "get_reporting_status" + ) + advertised = validator_for(schema)(schema, format_checker=formats()) + for view in ("summary", "periods"): + for protocol in ("mcp", "a2a-0.3", "a2a-1.0"): + call = mounted.mcp if protocol == "mcp" else mounted.a2a + kwargs = {} if protocol == "mcp" else {"v1": protocol == "a2a-1.0"} + _, raw = await call(client, request(version, view), **kwargs) + assert "health" in raw, raw + assert (raw["health"] == "complete") == complete + advertised.validate(raw) + get_validator("get_reporting_status", "sync", version=version).validate(raw) + if complete and view == "summary": + assert raw["next_expected_at"] == ( + "2026-09-01T01:00:00Z" if version == RC6 else "2026-09-01T02:00:00Z" + ) + assert raw["obligation_counts"]["total"] == 0 + elif complete and view == "periods": + assert raw["periods"] == [] + assert ("next_expected_at" in raw) == ( + version == RC3 and mode == "projection" + ) + elif view == "summary" or mode == "projection": + assert raw["next_expected_at"] == "2026-09-01T02:00:00Z" + if view == "periods": + assert len(raw["periods"]) == len(obligations) + # Actual SDK transports, generated response parsing and client validators. + for a2a_version in ("0.3", "1.0"): + async with mounted.sdk_clients(a2a_version) as (clients, observed): + for client in clients.values(): + result = await client.get_reporting_status( + GetReportingStatusRequest.model_validate(request(version)) + ) + assert result.success, result + data = result.data.model_dump(mode="json", exclude_none=True) + assert data["next_expected_at"] == ( + "2026-09-01T01:00:00Z" + if complete and version == RC6 + else "2026-09-01T02:00:00Z" + ) + assert observed and all(p[2]["adcp_version"] == version for p in observed) + assert await h.works() == before_work + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC3, RC6]) +async def test_frozen_continuations_retain_bytes_and_reject_cross_version_positions( + backend, version, monkeypatch +): + async with projection_harness(backend) as h: + config, _ = await scheduled(h, now=1.5) + capture_clock(h, monkeypatch) + caller = ReportingDeliveryPrincipal("acct_a", CONSUMER) + req = {**request(version, "periods"), "pagination": {"max_results": 1}} + first = await h.store.read_reporting_feed(req, caller=caller) + assert first["pagination"]["has_more"] + snapshot = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + document = canonical_json_utf8_v1(snapshot.to_storage()) + assert ("next_expected_at" in first) == (version == RC3) + other = RC6 if version == RC3 else RC3 + pagination = {"max_results": 1, "cursor": first["pagination"]["cursor"]} + for position in ( + {"pagination": pagination}, + {"changes_after": first["changes_checkpoint"]}, + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed( + {**req, **position, "adcp_version": other}, caller=caller + ) + assert error.value.code == "REPORTING_FEED_VERSION_MISMATCH" + original_pages, original_rows, checkpoint = await walk(h.store, req, caller, first=first) + await h.store.put_configuration(replace(config, deactivated_at=START)) + h.clock.now += timedelta(hours=8) + if h.pool is None: + from adcp.reporting.projection.memory import InMemoryReportingProjectionStore + + store = InMemoryReportingProjectionStore(clock=h.clock) + for key, value in vars(h.store).items(): + if key not in {"_clock", "_lock"}: + vars(store)[key] = deepcopy(value) + else: + from adcp.reporting.projection.pg import PgReportingProjectionStore + + store = PgReportingProjectionStore(pool=h.pool, clock=h.clock) + h.store = store + repeated, rows, repeated_checkpoint = await walk(store, req, caller, first=first) + assert ( + repeated == original_pages + and rows == original_rows + and repeated_checkpoint == checkpoint + ) + snapshot = await store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + assert canonical_json_utf8_v1(snapshot.to_storage()) == document + if version == RC3: + # Historical stored bytes remain replayable through the store API. + # The current SDK does not advertise rc.3 as a live server/client pin. + return + mounted = mount(h, config, version) + continuation = {**req, "pagination": pagination} + async with mounted.client() as client: + for call in (mounted.mcp, mounted.a2a): + _, raw = await call(client, continuation) + assert raw == original_pages[1] + _, crossed = await call(client, {**continuation, "adcp_version": other}) + assert error_code(crossed) == "VERSION_UNSUPPORTED", crossed + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC6]) +async def test_status_notification_mount_pin_controls_rendering_without_optional_validation( + backend, version +): + from adcp.reporting.ledger.status_server import ReportingStatusNotificationHandler + from adcp.reporting.receipts.handler import _consumer + + async with feed_harness(backend) as h: + config, _ = await scheduled(h) + mounted = mount(h, config, version) + + async def resolve_caller(params, context): + consumer = await _consumer(context, mounted.registry) + account = await mounted.resolve_account(params["account"], context, consumer) + return ReportingStatusCaller(account, consumer) + + mounted.handler = ReportingStatusNotificationHandler( + ReportingStatusHandler(h.store), + resolve_caller=resolve_caller, + adcp_version=version, + ) + async with mounted.client(validation=None) as client: + _, inventory = await mounted.mcp(client, inventory=True) + schema = next( + t["outputSchema"] for t in inventory["tools"] if t["name"] == "get_reporting_status" + ) + for call in (mounted.mcp, mounted.a2a): + _, result = await call(client, request(version)) + assert result["next_expected_at"] == ( + "2026-09-01T01:00:00Z" if version == RC6 else "2026-09-01T02:00:00Z" + ) + validator_for(schema)(schema, format_checker=formats()).validate(result) + _, rejected = await call(client, request(RC3 if version == RC6 else RC6)) + assert error_code(rejected) == "VERSION_UNSUPPORTED", rejected + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +async def test_rc6_nearest_captured_generation_and_historical_scope_filters(backend): + async with feed_harness(backend) as h: + h.clock.now = START + timedelta(minutes=30) + h.store._clock = h.clock + first = replace( + configuration(), + delivery_config_id="forecast-first", + deactivated_at=None, + schedule=ReportingScheduleSpec("PT2H", "PT1H", "utc", period_anchor=START), + ) + anchor = START + timedelta(minutes=45) + second = replace( + first, + delivery_config_id="forecast-second", + activated_at=anchor, + schedule=ReportingScheduleSpec("PT1H", "PT1H", "utc", period_anchor=anchor), + ) + foreign = replace( + second, + account_id="acct_b", + activated_at=START, + schedule=replace(second.schedule, period_anchor=START + timedelta(minutes=40)), + ) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + for config in (first, second, foreign): + await h.store.put_configuration(config) + production_operation_2 = await producer.close_elapsed_periods(config, now=h.clock()) + assert production_operation_2 == [] + captured = await h.store.read_status_snapshot(account_id="acct_a") + handler = ReportingStatusHandler(h.store) + caller = ReportingStatusCaller("acct_a", CONSUMER) + before = await h.image() + common = {"adcp_version": RC6, "view": "summary"} + expected = { + "next_expected_at": "2026-09-01T00:45:00Z", + "health": "complete", + } + original = handler.render_snapshot(common, caller=caller, snapshot=captured) + for filters, forecast in ( + ({}, expected["next_expected_at"]), + ({"delivery_config_ids": [first.delivery_config_id]}, "2026-09-01T02:00:00Z"), + ({"delivery_config_ids": [second.delivery_config_id]}, expected["next_expected_at"]), + ({"delivery_config_ids": ["absent"]}, None), + ({"feed_purposes": ["billing"]}, None), + ({"media_buy_ids": ["foreign-buy"]}, None), + ( + { + "delivery_config_ids": [first.delivery_config_id], + "period": {"start": START.isoformat(), "end": h.clock().isoformat()}, + }, + "2026-09-01T02:00:00Z", + ), + ): + result = handler.render_snapshot( + {**common, **filters}, caller=caller, snapshot=captured + ) + assert result["health"] == "complete" and result["obligation_counts"]["total"] == 0 + assert result.get("next_expected_at") == forecast + assert result["ledger_as_of"] == "2026-09-01T00:30:00Z" + get_named_validator( + "media-buy/get-reporting-status-response.json", version=RC6 + ).validate(result) + assert all(original[key] == value for key, value in expected.items()) + assert await h.image() == before + await h.store.put_configuration(replace(second, deactivated_at=anchor)) + h.clock.now += timedelta(days=1) + assert handler.render_snapshot(common, caller=caller, snapshot=captured) == original + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize( + "start,following", + [ + ("2026-03-08T05:00:00+00:00", "2026-03-09T04:00:00Z"), + ("2026-11-01T04:00:00+00:00", "2026-11-02T05:00:00Z"), + ], +) +async def test_rc6_period_start_forecast_retains_civil_dst_and_offset_instants( + backend, start, following +): + start = datetime.fromisoformat(start) + async with feed_harness(backend) as h: + h.clock.now = (start + timedelta(minutes=30)).astimezone( + timezone(timedelta(hours=5, minutes=30)) + ) + h.store._clock = h.clock + config = replace( + configuration(), + activated_at=start, + deactivated_at=None, + account_timezone="America/New_York", + schedule=ReportingScheduleSpec("P1D", "PT1H", "account_timezone", period_anchor=start), + ) + await h.store.put_configuration(config) + producer = ReportingProducer( + source=UncalledSource(), offerings=ProducerOfferings(), store=h.store + ) + production_operation_1 = await producer.close_elapsed_periods(config, now=h.clock()) + assert production_operation_1 == [] + caller = ReportingStatusCaller(config.account_id, CONSUMER) + handler = ReportingStatusHandler(h.store) + snapshot = await h.store.read_status_snapshot(account_id=config.account_id) + result = handler.render_snapshot(request(RC6), caller=caller, snapshot=snapshot) + assert result["next_expected_at"] == following + assert result["obligation_counts"]["total"] == 0 and result["health"] == "complete" + boundary = datetime.fromisoformat(following.replace("Z", "+00:00")) + h.clock.now = boundary + obligations = await producer.close_elapsed_periods(config, now=h.clock()) + assert len(obligations) == 1 and obligations[0].period.end == boundary + assert obligations[0].period.expected_at == boundary + timedelta(hours=1) + revision, rows = revision_for(obligations[0]) + await h.store.commit_revision(revision, rows) + next_day = await handler.handle(request(RC6), caller=caller) + assert next_day["next_expected_at"] == (boundary + timedelta(days=1)).isoformat().replace( + "+00:00", "Z" + ) + assert next_day["obligation_counts"]["total"] == 1 + assert handler.render_snapshot(request(RC6), caller=caller, snapshot=snapshot) == result + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("version", [RC6]) +@pytest.mark.parametrize("reconciled", [False, True]) +async def test_admitted_producer_verified_artifact_and_receipt_keep_the_future_summary( + backend, version, reconciled, tmp_path +): + from adcp.reporting.ledger import ReportingRevisionReceiptRecord + from adcp.reporting.ledger.delivery import receipt_to_wire + + from ._production_support import production_harness + from ._production_transport import MountedProduction + from ._projection_support import drain + from .test_reporting_production_lifecycle import observed_now + from .test_reporting_production_lock_order import source_turn + + async with production_harness( + backend, + tmp_path / "rc6-provider.sqlite", + count=3, + source_publication=True, + reconciled=reconciled, + adcp_version=version, + ) as h: + support, item = h.production, h.item + assert support.handler.get_adcp_version() == version + await support.activate(account_id=item.config.account_id) + turn = await source_turn(support) + assert not turn.slices_failed and len(turn.revisions_committed) == 1 + revisions = await h.store.list_revisions( + account_id=item.config.account_id, + reporting_obligation_id=item.obligation.reporting_obligation_id, + ) + assert len(revisions) == 1 + item.revision = revisions[0] + materialized = await support.materializer.run_once() + assert materialized.state == "verified" and item.writer.writes == 1 + mounted = MountedProduction(h) + mounted.authorize(item) + async with mounted.client() as client: + if reconciled: + receipt = ReportingRevisionReceiptRecord( + item.scope, + "rc6-production-accepted", + item.revision.reporting_revision_id, + materialized.reporting_materialization_id, + "accepted", + item.binding.verification_profile, + item.revision.row_count, + item.revision.managed_control_totals, + await observed_now(h), + observed_canonical_content_digest=item.revision.canonical_content_digest, + ) + _, accepted = await mounted.call( + client, + "sync_reporting_receipts", + { + "account": {"account_id": item.config.account_id}, + "idempotency_key": "rc6-production-receipt", + "receipts": [receipt_to_wire(receipt)], + }, + transport="a2a-1.0", + ) + assert accepted["results"][0]["result"] == "recorded", accepted + await drain(h.projection, item.config.account_id) + await h.store.put_configuration( + replace(item.config, deactivated_at=item.obligation.period.end) + ) + anchor = ((await observed_now(h)) + timedelta(days=1)).replace( + minute=0, second=0, microsecond=0 + ) + future = replace( + item.config, + delivery_config_version=2, + activated_at=anchor, + deactivated_at=None, + schedule=replace(item.config.schedule, period_anchor=anchor), + ) + producer = support.offerings[0].producer + producer._source.bind_generation(future) + destination = replace(item.binding, generation_key=future.generation_key) + item.writer.grant(destination) + await h.store.admit_production_configuration( + future, destination, offering_id=support.offerings[0].offering_id + ) + idle = await source_turn(support) + assert not idle.slices_failed and not idle.revisions_committed + assert len(producer._source.requests) == 1 + before_work = await h.works() + expectation = anchor if version == RC6 else anchor + timedelta(hours=2) + for transport in ("mcp", "a2a-0.3", "a2a-1.0"): + for view in ("summary", "periods"): + _, page = await mounted.call( + client, "get_reporting_status", request(version, view), transport=transport + ) + assert page["health"] == "complete", page + get_validator("get_reporting_status", "sync", version=version).validate(page) + if version == RC6: + for relative in ( + "media-buy/get-reporting-status-response.json", + "bundled/media-buy/get-reporting-status-response.json", + ): + get_named_validator(relative, version=version).validate(page) + if view == "summary": + assert page["next_expected_at"] == expectation.isoformat().replace( + "+00:00", "Z" + ) + assert page["obligation_counts"]["total"] == 1 + elif version == RC6: + assert "next_expected_at" not in page + assert await h.works() == before_work + async with MountedFeed.sdk_clients(mounted, "1.0") as (clients, observed): + for client in clients.values(): + result = await client.get_reporting_status( + GetReportingStatusRequest.model_validate(request(version)) + ) + assert result.success and result.data.health == "complete", result + assert observed and all(p[2]["adcp_version"] == version for p in observed) diff --git a/tests/conformance/reporting/test_reporting_projection_schedule.py b/tests/conformance/reporting/test_reporting_projection_schedule.py index a0b683ea6..591c46f1c 100644 --- a/tests/conformance/reporting/test_reporting_projection_schedule.py +++ b/tests/conformance/reporting/test_reporting_projection_schedule.py @@ -64,7 +64,8 @@ async def test_producer_and_public_summary_share_all_activation_and_due_boundari ) obligations = await producer.close_elapsed_periods(config, now=h.clock()) assert len(obligations) == closed - assert await producer.close_elapsed_periods(config, now=h.clock()) == [] + production_operation_1 = await producer.close_elapsed_periods(config, now=h.clock()) + assert production_operation_1 == [] # Completing all existing evidence must not erase tomorrow's commitment. for obligation in obligations: revision, rows = revision_for(obligation, suffix=obligation.reporting_obligation_id) diff --git a/tests/conformance/reporting/test_reporting_projection_waiver_rc6.py b/tests/conformance/reporting/test_reporting_projection_waiver_rc6.py new file mode 100644 index 000000000..a0b888647 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_waiver_rc6.py @@ -0,0 +1,72 @@ +"""Production projection inputs preserve the exact-scoped rc.6 waiver contract.""" + +from dataclasses import replace +from datetime import timedelta + +import pytest + +from adcp.reporting.ledger.status_projection import mismatch_key + +from ._projection_support import drain, inputs, projection_harness +from ._receipt_support import receipt_case +from .test_reporting_notification_outbox import statement + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_projection_captures_exact_waiver_and_rearms_later_statement(backend, notifications): + async with projection_harness(backend, notifications=notifications, feedback=True) as h: + scenario = await receipt_case( + h, finality="snapshot", billing=False, reconciliation_mode="delivery_only" + ) + original = replace( + statement(scenario.obligation, consumer=scenario.binding.consumer_id), + consumer_status="unreadable", + failure_code="access_denied", + ) + await h.store.record_consumer_status(original) + await h.projection.activate(account_id=scenario.obligation.account_id) + request = {"view": "summary", "adcp_version": "3.2-rc.6"} + before = await h.store.read_tier_status( + request, caller=scenario.binding.principal, consumer_status_enabled=True + ) + assert before["health"] == "action_required" + + waived = await h.store.set_issue_state( + issue_key=mismatch_key(original), + account_id=scenario.obligation.account_id, + state="waived", + at=h.clock(), + external_ref="private-bilateral-audit", + ) + await drain(h.projection, scenario.obligation.account_id) + recovered = await h.store.read_tier_status( + request, caller=scenario.binding.principal, consumer_status_enabled=True + ) + assert recovered["health"] == "complete" and recovered["issues"] == [] + captured = await inputs(h, scenario.obligation.account_id) + assert any( + row.issue_id == waived.issue_id and row.waived_conflict_sha256 + for row in captured[-1].core.lifecycles + ) + + h.clock.now += timedelta(seconds=1) + later = replace( + original, + reporting_status_id="next-mismatch", + supersedes_reporting_status_id=original.reporting_status_id, + recorded_at=h.clock(), + status_as_of=h.clock(), + ) + await h.store.record_consumer_status(later) + await drain(h.projection, scenario.obligation.account_id) + current = await h.store.read_tier_status( + request, caller=scenario.binding.principal, consumer_status_enabled=True + ) + assert current["health"] == "action_required" + assert current["issues"][0]["reporting_status_id"] == later.reporting_status_id + assert current["issues"][0]["issue_id"] != waived.issue_id + retained = await h.store.get_issue( + account_id=scenario.obligation.account_id, issue_key=waived.issue_key + ) + assert retained == waived diff --git a/tests/conformance/reporting/test_reporting_schedule_schema.py b/tests/conformance/reporting/test_reporting_schedule_schema.py index 849bc1602..fa039cd53 100644 --- a/tests/conformance/reporting/test_reporting_schedule_schema.py +++ b/tests/conformance/reporting/test_reporting_schedule_schema.py @@ -56,7 +56,8 @@ async def deterministic_summary(): store = InMemoryReportingLedgerStore(clock=lambda: START + timedelta(minutes=30)) await store.put_configuration(replace(configuration(), deactivated_at=None)) return await ReportingStatusHandler(store).handle( - {}, caller=ReportingStatusCaller("acct_a", "https://buyer.example.test/agent") + {"adcp_version": "3.2-rc.3"}, + caller=ReportingStatusCaller("acct_a", "https://buyer.example.test/agent"), ) @@ -198,7 +199,7 @@ def test_correction_is_limited_to_the_known_rule_and_version(mutation): @pytest.mark.parametrize("backend", ["memory", "postgres"]) @pytest.mark.parametrize("mode", ["core", "projection"]) @pytest.mark.parametrize("notifications", [False, True]) -@pytest.mark.parametrize("version", [None, PIN]) +@pytest.mark.parametrize("version", [None, "3.2.0-rc.6"]) async def test_complete_future_expectation_on_actual_summary_mounts( backend, mode, notifications, version ): @@ -217,7 +218,7 @@ async def test_complete_future_expectation_on_actual_summary_mounts( ) mounted.authorize(identity) request = { - "adcp_version": "3.2-rc.3", + "adcp_version": "3.2-rc.6", "account": {"account_id": config.account_id}, "view": "summary", } @@ -238,7 +239,7 @@ async def test_complete_future_expectation_on_actual_summary_mounts( else: _, raw = await mounted.a2a(client, request, v1=transport == "a2a-1.0") assert raw.get("health") == "complete", raw - assert raw["next_expected_at"] == "2026-09-01T02:00:00Z" + assert raw["next_expected_at"] == "2026-09-01T01:00:00Z" assert_original_rejection(raw) schema_loader.get_validator("get_reporting_status", "sync", version=PIN).validate( raw From bb2921619762175adcc101ca09c74bd52cc196a8 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 25 Sep 2026 00:04:01 +0000 Subject: [PATCH 4/6] fix(reporting): retain unversioned feed continuations --- docs/reporting-production.md | 8 ++ src/adcp/reporting/feed/request.py | 6 +- src/adcp/reporting/feed/snapshot.py | 21 ++++- .../test_reporting_projection_legacy_feed.py | 82 +++++++++++++++++++ 4 files changed, 111 insertions(+), 6 deletions(-) create mode 100644 tests/conformance/reporting/test_reporting_projection_legacy_feed.py diff --git a/docs/reporting-production.md b/docs/reporting-production.md index d89178e36..e4b7a1ab2 100644 --- a/docs/reporting-production.md +++ b/docs/reporting-production.md @@ -189,6 +189,14 @@ packaged version. Cursor/checkpoint version mismatches fail closed only after caller and signed-position verification. Changing a production mount's captured protocol pin invalidates readiness even after a successful schema proof. +Integrated parents created representation-one snapshots without a protocol +filter marker. Their original pages and checkpoints remain usable under rc.6, +before and after activation, without re-projecting or modifying their captured +bytes. Caller, signature and every original semantic filter must still match. +This exception is restricted to unversioned representation one without revision +ownership; newer version-bound representations retain strict pin matching. A +legacy checkpoint can start a fresh rc.6 walk, which records the new marker. + These are Python source and conformance boundaries, not TypeScript, release or cross-language acceptance. The tests in `test_reporting_projection_rc6.py` cover current mounted transports, clients and schema agreement, frozen historical diff --git a/src/adcp/reporting/feed/request.py b/src/adcp/reporting/feed/request.py index de12580b3..7a81c3529 100644 --- a/src/adcp/reporting/feed/request.py +++ b/src/adcp/reporting/feed/request.py @@ -160,9 +160,9 @@ def parse(cls, request: dict[str, Any]) -> FeedRequest: request.get("adcp_version") or resolve_adcp_version(None) ) if is_adcp_version_at_least(version, "3.2-rc.6"): - # Existing snapshots omit this marker and keep their rc.3 - # bytes. The persisted filter binding prevents cross-version - # replay under an incompatible advertised output schema. + # New snapshots bind their rendering contract. Integrated + # parents' unversioned v1 walks retain their captured bytes; + # StoredFeedSnapshot checks that narrow compatibility case. filters["adcp_version"] = version for name in ( "delivery_config_ids", diff --git a/src/adcp/reporting/feed/snapshot.py b/src/adcp/reporting/feed/snapshot.py index ef89a406b..91f4a3e3a 100644 --- a/src/adcp/reporting/feed/snapshot.py +++ b/src/adcp/reporting/feed/snapshot.py @@ -248,12 +248,27 @@ def check( ): raise ReportingFeedError("INVALID_CHECKPOINT") if snapshot.filters_json != request.filters_json: + captured_filters = json.loads(snapshot.filters_json) + proposed_filters = request.filters + if ( + snapshot.representation_version == 1 + and snapshot.ownership_mode == "absent" + and "adcp_version" not in captured_filters + and proposed_filters.get("adcp_version") == "3.2-rc.6" + ): + # Integrated parents persisted unversioned v1 filters. Their + # immutable representation has no complete-summary forecast; + # the rc.6 marker must not invalidate those original walks. + # Caller, position and signature were verified above. Every + # previously bound semantic filter must still match exactly. + proposed_filters.pop("adcp_version") + if captured_filters != proposed_filters: + raise ReportingFeedError("INVALID_CHECKPOINT") + return position # Only an authenticated, correctly signed position can disclose # this actionable version boundary. Other callers/tokens retain # the indistinguishable INVALID_CHECKPOINT error above. - if json.loads(snapshot.filters_json).get("adcp_version") != request.filters.get( - "adcp_version" - ): + if captured_filters.get("adcp_version") != proposed_filters.get("adcp_version"): raise ReportingFeedError("REPORTING_FEED_VERSION_MISMATCH") raise ReportingFeedError("INVALID_CHECKPOINT") return position diff --git a/tests/conformance/reporting/test_reporting_projection_legacy_feed.py b/tests/conformance/reporting/test_reporting_projection_legacy_feed.py new file mode 100644 index 000000000..d33424105 --- /dev/null +++ b/tests/conformance/reporting/test_reporting_projection_legacy_feed.py @@ -0,0 +1,82 @@ +"""Unversioned representation-one walks survive the protocol-filter addition. + +These are legacy-format store controls. The separate rolling lane supplies +actual installed-parent evidence; this fixture does not claim that provenance. +""" + +import json +from dataclasses import replace + +import pytest + +from adcp.reporting.canonical_json import canonical_json_utf8_v1 +from adcp.reporting.feed import ReportingFeedError + +from ._feed_support import feed_request, mixed_case, walk +from ._projection_support import projection_harness + + +@pytest.mark.parametrize("backend", ["memory", "postgres"]) +@pytest.mark.parametrize("notifications", [False, True]) +async def test_legacy_unversioned_walk_survives_activation_and_current_protocol( + backend, notifications +): + async with projection_harness(backend, notifications=notifications) as h: + scenario, _, _ = await mixed_case(h) + caller = scenario.binding.principal + # The historical direct pin produces the pre-marker filter format. + # Before activation its representation is the parent's original v1. + old_request = feed_request(scenario, adcp_version="3.2-rc.3") + first = await h.store.read_reporting_feed(old_request, caller=caller) + snapshot = await h.store.read_reporting_feed_snapshot( + first["ledger_snapshot_id"], caller=caller + ) + assert snapshot.representation_version == 1 + assert snapshot.ownership_mode == "absent" + assert "adcp_version" not in json.loads(snapshot.filters_json) + stored_bytes = canonical_json_utf8_v1(snapshot.to_storage()) + expected = await walk(h.store, old_request, caller, first=first) + current_request = feed_request(scenario) + + for activated in (False, True): + if activated: + await h.projection.activate(account_id=caller.account_id) + continued = await walk(h.store, current_request, caller, first=first) + assert continued == expected + retained = await h.store.read_reporting_feed_snapshot( + snapshot.snapshot_id, caller=caller + ) + assert canonical_json_utf8_v1(retained.to_storage()) == stored_bytes + + # Caller, filter and signature failures remain indistinguishable. + continuation = feed_request( + scenario, + pagination={"cursor": first["pagination"]["cursor"], "max_results": 1}, + ) + for changed, principal in ( + ({**continuation, "media_buy_ids": ["different"]}, caller), + (continuation, replace(caller, consumer_id="different")), + ( + { + **continuation, + "pagination": {"cursor": first["pagination"]["cursor"] + "x"}, + }, + caller, + ), + ): + with pytest.raises(ReportingFeedError) as error: + await h.store.read_reporting_feed(changed, caller=principal) + assert error.value.code == "INVALID_CHECKPOINT" + + # A legacy checkpoint can start a new current-version walk without + # modifying its original boundary or any persisted old page. + repaired = await h.store.read_reporting_feed( + feed_request(scenario, changes_after=expected[2]), caller=caller + ) + newer = await h.store.read_reporting_feed_snapshot( + repaired["ledger_snapshot_id"], caller=caller + ) + assert newer.snapshot_id != snapshot.snapshot_id + assert newer.after == snapshot.through + assert json.loads(newer.filters_json)["adcp_version"] == "3.2-rc.6" + assert newer.representation_version == 2 From e9a1c8fcb653870bddef486b56e76820f467c231 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 25 Sep 2026 00:42:01 +0000 Subject: [PATCH 5/6] test(reporting): align installed production contracts with rc6 --- .../reporting/_production_installed.py | 55 +++++++++++++++++- .../reporting/_production_packaging.py | 30 +++++++++- .../test_reporting_production_migration.py | 21 ++++--- .../test_reporting_projection_schedule.py | 57 +++++++++++-------- 4 files changed, 127 insertions(+), 36 deletions(-) diff --git a/tests/conformance/reporting/_production_installed.py b/tests/conformance/reporting/_production_installed.py index 7125abeb0..c01c3f7b2 100644 --- a/tests/conformance/reporting/_production_installed.py +++ b/tests/conformance/reporting/_production_installed.py @@ -19,18 +19,33 @@ def main(settings): root = Path(settings["fixtures"]) workspace = Path(settings["workspace"]) + evidence = Path(settings["evidence"]) + evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + phases = [] + + def enter_phase(name): + # Closed phase names identify a failed preflight without publishing + # arbitrary child stderr, provider detail or runtime values. + phases.append(name) + (evidence / (settings["label"] + "-phases.json")).write_text( + json.dumps({"entered_phases": phases}) + "\n" + ) + + enter_phase("runtime") assert sys.version_info[:2] == tuple(settings["python"]) assert not any(Path(p).resolve().is_relative_to(workspace) for p in sys.path) assert not (root / "adcp").exists() and not (root / "src").exists() sys.path.insert(0, str(root)) from tests.conformance.reporting._hardening_installed import Results + enter_phase("installed_modules") origins = {} for name, expected in settings["modules"].items(): path = Path(importlib.import_module(name).__file__).resolve() assert path.is_relative_to(Path(sys.prefix)) and not path.is_relative_to(workspace) assert hashlib.sha256(path.read_bytes()).hexdigest() == expected origins[name] = str(path) + enter_phase("installed_assets") for name, expected in settings["assets"].items(): assert ( hashlib.sha256(files("adcp.reporting").joinpath(name).read_bytes()).hexdigest() @@ -38,19 +53,40 @@ def main(settings): ) from adcp.validation import schema_loader + enter_phase("installed_current_schemas") + assert set(settings["schemas"]) == {schema_loader._sdk_pinned_bundle_key()} for version, schemas in settings["schemas"].items(): - schema_root = schema_loader._resolve_schema_root(version).root + resolved = schema_loader._resolve_schema_root(version) + assert resolved is not None + schema_root = resolved.root assert schema_root.is_relative_to(Path(sys.prefix)) for name, expected in schemas.items(): assert hashlib.sha256((schema_root / name).read_bytes()).hexdigest() == expected + enter_phase("historical_reference_schemas") + reference = settings["historical_reference_schema"] + reference_root = Path(reference["root"]) + assert not reference_root.is_relative_to(Path(sys.prefix)) + assert not reference_root.is_relative_to(workspace) + assert reference["version"] not in settings["schemas"] + assert schema_loader._resolve_schema_root(reference["version"]).root == reference_root + + def reference_manifest(): + return { + str(p.relative_to(reference_root)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(reference_root.rglob("*.json")) + } + + assert reference_manifest() == reference["files"] + enter_phase("optional_driver_boundary") if settings["driver_absent"]: assert importlib.util.find_spec("psycopg") is None assert importlib.util.find_spec("psycopg_pool") is None os.environ.pop("ADCP_PG_TEST_URL", None) else: assert os.environ.get("ADCP_PG_TEST_URL") - evidence = Path(settings["evidence"]) - evidence.mkdir(parents=True, exist_ok=True, mode=0o700) + reference_inputs = evidence / (settings["label"] + "-historical-schema-inputs.json") + reference_bytes = (json.dumps(reference, indent=2, sort_keys=True) + "\n").encode() + reference_inputs.write_bytes(reference_bytes) log = evidence / (settings["label"] + ".log") recorder = Results() command = [ @@ -66,6 +102,7 @@ def main(settings): str(root / "temp"), "--deselect=tests/test_reporting_capability_models.py::test_post_generation_repair_is_idempotent_for_both_actual_model_layouts", ] + enter_phase("conformance") started = time.monotonic() with ( log.open("x") as stream, @@ -107,13 +144,16 @@ def main(settings): "--no-incremental", str(root / "adopter.py"), ] + enter_phase("strict_adopter") typed = subprocess.run(typing_command, cwd=root, capture_output=True, timeout=120) typing_log = evidence / (settings["label"] + "-adopter.log") typing_log.write_bytes(typed.stdout + typed.stderr) result["valid"] &= typed.returncode == 0 + enter_phase("final_origins_and_reference_preservation") for name, module in tuple(sys.modules.items()): if (name == "adcp" or name.startswith("adcp.")) and getattr(module, "__file__", None): assert Path(module.__file__).resolve().is_relative_to(Path(sys.prefix)) + assert reference_manifest() == reference["files"] record = { "python": sys.version, "source_basis": settings["source_basis"], @@ -125,6 +165,14 @@ def main(settings): "wheel_sha256": settings["wheel_sha256"], "assets": settings["assets"], "schemas": settings["schemas"], + "historical_reference_schema": { + "version": reference["version"], + "root": str(reference_root), + "origin": "copied immutable test reference; not a shipped SDK bundle", + "files": len(reference["files"]), + "inputs": str(reference_inputs), + "inputs_sha256": hashlib.sha256(reference_bytes).hexdigest(), + }, "driver_absent": settings["driver_absent"], "result": result, "adopter": { @@ -136,6 +184,7 @@ def main(settings): }, } (evidence / (settings["label"] + ".json")).write_text(json.dumps(record, indent=2) + "\n") + enter_phase("record_complete") print(json.dumps(record), flush=True) diff --git a/tests/conformance/reporting/_production_packaging.py b/tests/conformance/reporting/_production_packaging.py index 5eb9c20da..7cfd900f9 100644 --- a/tests/conformance/reporting/_production_packaging.py +++ b/tests/conformance/reporting/_production_packaging.py @@ -193,6 +193,33 @@ def git(*args): return basis +def historical_schema_fixture(root): + """Copy immutable rc.3 reference inputs, never claim them as wheel contents. + + The installed suite retains historical rejection and correction controls. + rc.3 is no longer a shipped bundle. The existing source-layout fallback + can read these explicit test inputs without changing installed SDK code, + its current bundle, or the public protocol-version allowlist. + """ + version = "3.2.0-rc.3" + source = ROOT / "schemas/cache" / version + destination = root / "schemas/cache" / version + assert not root.resolve().is_relative_to(ROOT.resolve()) + expected = { + str(p.relative_to(source)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(source.rglob("*.json")) + } + assert expected + if not destination.exists(): + shutil.copytree(source, destination) + actual = { + str(p.relative_to(destination)): hashlib.sha256(p.read_bytes()).hexdigest() + for p in sorted(destination.rglob("*.json")) + } + assert actual == expected + return {"version": version, "root": str(destination), "files": expected} + + def installed_production(root, python, wheel, source, *, label, driver_absent): fixture_root = copied_fixtures(root, label) script = fixture_root / "run_installed.py" @@ -253,8 +280,9 @@ def installed_production(root, python, wheel, source, *, label, driver_absent): ).hexdigest() for name in SCHEMAS } - for version in ("3.2.0-rc.3", "3.2.0-rc.6") + for version in ("3.2.0-rc.6",) }, + "historical_reference_schema": historical_schema_fixture(root), "tests": [str(p.relative_to(ROOT)) for p in tests], "driver_absent": driver_absent, "python": [3, 10], diff --git a/tests/conformance/reporting/test_reporting_production_migration.py b/tests/conformance/reporting/test_reporting_production_migration.py index 0fe0c0bdf..363aefc75 100644 --- a/tests/conformance/reporting/test_reporting_production_migration.py +++ b/tests/conformance/reporting/test_reporting_production_migration.py @@ -38,7 +38,8 @@ def original_rows(image, before): # Its default leaves old C checkpoints compatible until explicit activation. result = deepcopy({key: image[key] for key in before}) for (row,) in result.get("reporting_status_scope_checkpoints", []): - assert row.pop("projection_writer_floor", 1) == 1 + writer_floor = row.pop("projection_writer_floor", 1) + assert writer_floor == 1 return result @@ -200,15 +201,21 @@ async def interrupt(connection, query, *args, **kwargs): @pytest.mark.parametrize( "damage", [ - "ALTER TABLE reporting_production_delivery_windows" - " DISABLE TRIGGER reporting_production_delivery_window_immutable", + ( + "ALTER TABLE reporting_production_delivery_windows" + " DISABLE TRIGGER reporting_production_delivery_window_immutable" + ), "ALTER TABLE reporting_production_delivery_windows ALTER COLUMN expires_at DROP NOT NULL", "DROP TABLE reporting_production_delivery_windows", "DROP INDEX reporting_production_source_pending", - "ALTER TABLE reporting_projection_inputs" - " DISABLE TRIGGER reporting_projection_input_immutable", - "ALTER TABLE reporting_status_scope_checkpoints" - " DISABLE TRIGGER reporting_projection_checkpoint_guard", + ( + "ALTER TABLE reporting_projection_inputs" + " DISABLE TRIGGER reporting_projection_input_immutable" + ), + ( + "ALTER TABLE reporting_status_scope_checkpoints" + " DISABLE TRIGGER reporting_projection_checkpoint_guard" + ), ], ) @pytest.mark.parametrize("notifications", [False, True]) diff --git a/tests/conformance/reporting/test_reporting_projection_schedule.py b/tests/conformance/reporting/test_reporting_projection_schedule.py index 591c46f1c..f6b005452 100644 --- a/tests/conformance/reporting/test_reporting_projection_schedule.py +++ b/tests/conformance/reporting/test_reporting_projection_schedule.py @@ -1,4 +1,4 @@ -"""Captured generation forecasts agree with actual period creation (#1179).""" +"""Versioned complete forecasts preserve activation, due times and captured state.""" from dataclasses import replace from datetime import datetime, timedelta, timezone @@ -30,26 +30,27 @@ def hour(value): @pytest.mark.parametrize( - "activation,deactivation,now,expected,closed", + "activation,deactivation,now,legacy_due,rc6_start,closed", [ - (0, None, 0.5, 2, 0), # mid-period, no obligation or coverage yet - (0, None, 1, 2, 1), # closes at 01:00, due at 02:00 - (0, None, 1.5, 2, 1), - (0, None, 2, 3, 2), # strictly future expectation at the exact SLA - (0, None, 2.5, 3, 2), - (3, None, 0.5, 5, 0), # already committed future activation - (0.5, None, 0.5, 3, 0), # first full period only - (1, None, 1, 3, 0), - (0, 3, 0.5, 2, 0), # future deactivation - (0, 1, 2, None, 1), # stop exactly at the next start - (0, 0.5, 0.5, 2, 0), # begun full period remains owed - (0, 0.5, 2, None, 1), - (1, 1, 0.5, None, 0), # no committed active interval - (None, None, 0.5, None, 0), + (0, None, 0.5, 2, 1, 0), # mid-period, no obligation or coverage yet + (0, None, 1, 2, 2, 1), # closes at 01:00, due at 02:00 + (0, None, 1.5, 2, 2, 1), + (0, None, 2, 3, 3, 2), # strictly future expectation at the exact SLA + (0, None, 2.5, 3, 3, 2), + (3, None, 0.5, 5, 3, 0), # already committed future activation + (0.5, None, 0.5, 3, 1, 0), # first full period only + (1, None, 1, 3, 2, 0), + (0, 3, 0.5, 2, 1, 0), # future deactivation + (0, 1, 2, None, None, 1), # stop exactly at the next start + (0, 0.5, 0.5, 2, None, 0), # begun full period remains owed + (0, 0.5, 2, None, None, 1), + (1, 1, 0.5, None, None, 0), # no committed active interval + (None, None, 0.5, None, None, 0), ], ) -async def test_producer_and_public_summary_share_all_activation_and_due_boundaries( - schedules, activation, deactivation, now, expected, closed +@pytest.mark.parametrize("adcp_version", ["3.2-rc.3", "3.2-rc.6"]) +async def test_complete_forecast_retains_versioned_activation_and_due_boundaries( + schedules, activation, deactivation, now, legacy_due, rc6_start, closed, adcp_version ): h = schedules h.clock.now = hour(now) @@ -72,13 +73,17 @@ async def test_producer_and_public_summary_share_all_activation_and_due_boundari await h.store.commit_revision(revision, rows) handler = ReportingStatusHandler(h.store) for consumer in ("buyer-one", "buyer-two"): - raw = await handler.handle({}, caller=ReportingStatusCaller("acct_a", consumer)) + raw = await handler.handle( + {"adcp_version": adcp_version}, caller=ReportingStatusCaller("acct_a", consumer) + ) GetReportingStatusResponse.model_validate(raw) - validator = get_validator("get_reporting_status", "sync") + validator = get_validator("get_reporting_status", "sync", version=adcp_version) assert validator is not None validator.validate(raw) assert raw["health"] == "complete" assert raw["obligation_counts"]["total"] == closed + # Complete rc.6 forecasts name the next start, never the obligation due time. + expected = rc6_start if adcp_version == "3.2-rc.6" else legacy_due assert raw.get("next_expected_at") == ( hour(expected).isoformat().replace("+00:00", "Z") if expected is not None else None ) @@ -87,8 +92,9 @@ async def test_producer_and_public_summary_share_all_activation_and_due_boundari assert raw["issues"] == [] +@pytest.mark.parametrize("adcp_version,expected_minute", [("3.2-rc.3", 10), ("3.2-rc.6", 0)]) async def test_nearest_generation_and_account_filters_use_captured_not_current_configuration( - schedules, + schedules, adcp_version, expected_minute ): h = schedules h.clock.now = hour(0.5) @@ -105,12 +111,13 @@ async def test_nearest_generation_and_account_filters_use_captured_not_current_c handler = ReportingStatusHandler(h.store) caller = ReportingStatusCaller("acct_a", "buyer") captured = await h.store.read_status_snapshot(account_id="acct_a") - expected = hour(1) + timedelta(minutes=10) - original = handler.render_snapshot({}, caller=caller, snapshot=captured) + expected = hour(1) + timedelta(minutes=expected_minute) + request = {"adcp_version": adcp_version} + original = handler.render_snapshot(request, caller=caller, snapshot=captured) assert original["next_expected_at"] == expected.isoformat().replace("+00:00", "Z") await h.store.put_configuration(replace(second, deactivated_at=hour(0))) h.clock.now = hour(5) - assert handler.render_snapshot({}, caller=caller, snapshot=captured) == original + assert handler.render_snapshot(request, caller=caller, snapshot=captured) == original for filters in ( {"delivery_config_ids": ["absent"]}, {"feed_purposes": ["billing"]}, @@ -118,7 +125,7 @@ async def test_nearest_generation_and_account_filters_use_captured_not_current_c {"period": {"start": hour(-2).isoformat(), "end": hour(0).isoformat()}}, ): assert "next_expected_at" not in handler.render_snapshot( - filters, caller=caller, snapshot=captured + {**request, **filters}, caller=caller, snapshot=captured ) From decfb4eb53845c22b019466b4167293f6c6181c7 Mon Sep 17 00:00:00 2001 From: Brian O'Kelley Date: Fri, 25 Sep 2026 01:51:27 +0000 Subject: [PATCH 6/6] perf(schema): reuse immutable MCP materializations --- src/adcp/server/mcp_tools.py | 10 +- src/adcp/validation/schema_loader.py | 82 ++++--- tests/test_mcp_schema_materialization.py | 295 +++++++++++++++++++++++ 3 files changed, 357 insertions(+), 30 deletions(-) create mode 100644 tests/test_mcp_schema_materialization.py diff --git a/src/adcp/server/mcp_tools.py b/src/adcp/server/mcp_tools.py index 5859b5d22..aba71b5b4 100644 --- a/src/adcp/server/mcp_tools.py +++ b/src/adcp/server/mcp_tools.py @@ -2447,7 +2447,15 @@ def get_tools_for_handler( input_schema = get_mcp_schema(name, "request", version=resolved_version) if input_schema is None: continue - definition = copy.deepcopy(tool) + # The current-model schemas can be large and are replaced below by + # the exact versioned wire schemas. Copy only retained metadata. + definition = copy.deepcopy( + { + key: value + for key, value in tool.items() + if key not in {"inputSchema", "outputSchema"} + } + ) if name == "sync_reporting_receipts": from adcp.reporting.receipts.wire import receipt_schema diff --git a/src/adcp/validation/schema_loader.py b/src/adcp/validation/schema_loader.py index 028f4b90f..14267f0ac 100644 --- a/src/adcp/validation/schema_loader.py +++ b/src/adcp/validation/schema_loader.py @@ -34,7 +34,7 @@ from datetime import date from importlib.resources import as_file, files from pathlib import Path -from typing import Any, Literal +from typing import Any, Literal, cast from urllib.parse import unquote, urlparse from adcp.validation.version import resolve_bundle_key @@ -147,6 +147,10 @@ def __init__(self, root: _SchemaRoot, bundle_key: str) -> None: self.compiled: dict[tuple[str, Direction], Any] = {} self.named_compiled: dict[str, Any] = {} self.portable: dict[tuple[str, Direction], dict[str, Any]] = {} + # Serialized JSON keeps the immutable cached value private and makes + # every returned tree independent, including any repeated branches. + self.mcp_schemas: dict[tuple[str, Direction], str] = {} + self.mcp_schema_lock = threading.Lock() self.registry: dict[str, dict[str, Any]] = {} self._registry_loaded = False @@ -922,39 +926,59 @@ def get_mcp_schema( Newer bundles provide self-contained production-profile schemas that remove duplicated descriptions and definitions. Releases without those artifacts fall back to their canonical versioned schema. + + Successful materializations belong to the immutable versioned loader + state. Each caller receives an independent, alias-free JSON tree; missing + or invalid schemas are not added to the materialization cache. """ state = _ensure_state(version) if state is None: return None key = (tool_name, direction) - file = state.mcp_index.get(key) or state.source_index.get(key) or state.file_index.get(key) - if file is None: - return None - try: - schema = json.loads(file.read_text()) - except (OSError, json.JSONDecodeError) as exc: - logger.warning( - "Failed to load MCP schema %s for %s::%s: %s", - file, - tool_name, - direction, - exc, - ) - return None - if not isinstance(schema, dict): - logger.warning("MCP schema %s is not a JSON object", file) - return None - try: - portable = _self_contained_schema( - state, - file, - _effective_task_schema(schema, tool_name, direction, bundle_key=state.bundle_key), - ) - except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: - logger.warning("Failed to make MCP schema %s portable: %s", file, exc) - return None - compact = _strip_schema_annotations(portable) - return compact if isinstance(compact, dict) else None + cached = state.mcp_schemas.get(key) + if cached is None: + with state.mcp_schema_lock: + # Only one concurrent first caller traverses the reference graph. + cached = state.mcp_schemas.get(key) + if cached is None: + file = ( + state.mcp_index.get(key) + or state.source_index.get(key) + or state.file_index.get(key) + ) + if file is None: + return None + try: + schema = json.loads(file.read_text()) + except (OSError, json.JSONDecodeError) as exc: + logger.warning( + "Failed to load MCP schema %s for %s::%s: %s", + file, + tool_name, + direction, + exc, + ) + return None + if not isinstance(schema, dict): + logger.warning("MCP schema %s is not a JSON object", file) + return None + try: + portable = _self_contained_schema( + state, + file, + _effective_task_schema( + schema, tool_name, direction, bundle_key=state.bundle_key + ), + ) + except (OSError, json.JSONDecodeError, KeyError, ValueError) as exc: + logger.warning("Failed to make MCP schema %s portable: %s", file, exc) + return None + compact = _strip_schema_annotations(portable) + if not isinstance(compact, dict): + return None + cached = json.dumps(compact, separators=(",", ":")) + state.mcp_schemas[key] = cached + return cast(dict[str, Any], json.loads(cached)) def list_validator_keys(*, version: str | None = None) -> list[str]: diff --git a/tests/test_mcp_schema_materialization.py b/tests/test_mcp_schema_materialization.py new file mode 100644 index 000000000..338e5675d --- /dev/null +++ b/tests/test_mcp_schema_materialization.py @@ -0,0 +1,295 @@ +"""Versioned public schema construction is cached without sharing mutable state.""" + +from __future__ import annotations + +import hashlib +import json +import threading +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +from copy import deepcopy +from typing import Any + +import httpx +import pytest +from asgi_lifespan import LifespanManager + +from adcp.server import ADCPHandler, create_mcp_server, mcp_tools +from adcp.server.a2a_server import create_a2a_server +from adcp.validation import schema_loader as loader +from adcp.validation.schema_validator import validate_request + +PINS = ("3.2.0-rc.3", "3.2.0-rc.6", "3.2.0-beta.6") + +# Canonical JSON digests captured from unchanged e9a1c8fc before adding +# the materialization cache. These are transformed public schemas, +# not a claim of byte identity with the signed upstream schema files. +EXPECTED_PUBLIC_SHA256 = { + "3.2.0-rc.3": "6656874ca37ea0732e65a5f0313c8ead7b56ed12460bdbae297c4c648fa26f14", + "3.2.0-rc.6": "d712168e85932dfabad8a76b49a24aa6411dc11748832d18dd0ae00ad7106b3e", + "3.2.0-beta.6": "e9a10b9f1ee5654a51219c91329089f591c953b972e57a913450a1b9acdb810c", +} + + +class SchemaHandler(ADCPHandler): + def __init__(self) -> None: + self.calls = 0 + + async def get_products(self, params: Any, context: Any = None) -> dict[str, Any]: + self.calls += 1 + return {"products": []} + + async def get_reporting_status(self, params: Any, context: Any = None) -> dict[str, Any]: + raise AssertionError("schema discovery must not execute a reporting task") + + +class PinnedSchemaHandler(SchemaHandler): + def __init__(self, version: str) -> None: + super().__init__() + self.version = version + + def get_adcp_version(self) -> str: + return self.version + + +@pytest.fixture(autouse=True) +def isolated_loader(): + loader._reset_for_tests() + yield + loader._reset_for_tests() + + +def canonical(value: Any) -> bytes: + return json.dumps(value, sort_keys=True, separators=(",", ":")).encode() + + +def mutable_ids(value: Any) -> set[int]: + """Reject aliases within a result as well as between independent results.""" + seen: set[int] = set() + pending = [value] + while pending: + node = pending.pop() + if isinstance(node, (dict, list)): + assert id(node) not in seen + seen.add(id(node)) + pending.extend(node.values() if isinstance(node, dict) else node) + return seen + + +def public_definitions(version: str) -> list[dict[str, Any]]: + return mcp_tools.get_tools_for_handler(PinnedSchemaHandler(version)) + + +def count_materializations(monkeypatch): + original = loader._self_contained_schema + calls: Counter[tuple[str, str]] = Counter() + lock = threading.Lock() + + def counted(state, path, schema): + with lock: + calls[state.bundle_key, str(path)] += 1 + return original(state, path, schema) + + monkeypatch.setattr(loader, "_self_contained_schema", counted) + return calls + + +@pytest.mark.parametrize("version", PINS) +def test_repeated_public_schema_construction_materializes_once(monkeypatch, version): + calls = count_materializations(monkeypatch) + first = loader.get_mcp_schema("get_products", "request", version=version) + assert first is not None + original = canonical(first) + second = loader.get_mcp_schema("get_products", "request", version=version) + assert canonical(second) == original + assert not mutable_ids(first).intersection(mutable_ids(second)) + first["properties"]["brief"] = {"type": "array"} + third = loader.get_mcp_schema("get_products", "request", version=version) + assert canonical(third) == original + assert not mutable_ids(second).intersection(mutable_ids(third)) + assert list(calls.values()) == [1] + + +def test_public_handler_pins_have_isolated_materializations(monkeypatch): + calls = count_materializations(monkeypatch) + saved = {} + for version in PINS: + definitions = public_definitions(version) + saved[version] = canonical(definitions) + assert hashlib.sha256(saved[version]).hexdigest() == EXPECTED_PUBLIC_SHA256[version] + for definition in definitions: + definition["inputSchema"].clear() + if "outputSchema" in definition: + definition["outputSchema"].clear() + for version in reversed(PINS): + definitions = public_definitions(version) + assert canonical(definitions) == saved[version] + assert {key[0] for key in calls} == set(PINS) + assert calls and set(calls.values()) == {1} + assert len(set(saved.values())) == len(PINS) + + +def test_loader_reset_discards_materializations(monkeypatch): + calls = count_materializations(monkeypatch) + first = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + loader._reset_for_tests() + second = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert first == second + assert not mutable_ids(first).intersection(mutable_ids(second)) + assert list(calls.values()) == [2] + + +def test_failed_materialization_is_not_cached(monkeypatch): + original = loader._self_contained_schema + attempts = 0 + + def transient_failure(*args, **kwargs): + nonlocal attempts + attempts += 1 + if attempts == 1: + raise ValueError("missing fixture reference") + return original(*args, **kwargs) + + monkeypatch.setattr(loader, "_self_contained_schema", transient_failure) + missing = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert missing is None + restored = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert restored is not None + cached = loader.get_mcp_schema("get_products", "request", version=PINS[1]) + assert cached == restored + assert attempts == 2 + + +def test_concurrent_first_callers_share_one_materialization(monkeypatch): + workers = 8 + ready = threading.Barrier(workers + 1) + entered = threading.Event() + release = threading.Event() + count_lock = threading.Lock() + original = loader._self_contained_schema + attempts = 0 + + def blocked(*args, **kwargs): + nonlocal attempts + with count_lock: + attempts += 1 + entered.set() + released = release.wait(timeout=20) + assert released + return original(*args, **kwargs) + + def load(): + ready.wait(timeout=20) + return loader.get_mcp_schema("get_products", "request", version=PINS[1]) + + monkeypatch.setattr(loader, "_self_contained_schema", blocked) + with ThreadPoolExecutor(max_workers=workers) as pool: + pending = [pool.submit(load) for _ in range(workers)] + try: + ready.wait(timeout=20) + started = entered.wait(timeout=20) + assert started + finally: + release.set() + results = [future.result(timeout=30) for future in pending] + assert results[0] is not None and all(value == results[0] for value in results) + seen: set[int] = set() + for result in results: + identities = mutable_ids(result) + assert not seen.intersection(identities) + seen.update(identities) + assert attempts == 1 + + +def test_pinned_schemas_do_not_copy_superseded_current_models(monkeypatch): + class SupersededSchema(dict): + def __deepcopy__(self, memo): + raise AssertionError("copied a current-model schema that the pin replaces") + + def unexpected_generation(*args, **kwargs): + raise AssertionError("pinned discovery generated current-model schemas") + + definition = next(t for t in mcp_tools.ADCP_TOOL_DEFINITIONS if t["name"] == "get_products") + monkeypatch.setitem(definition, "inputSchema", SupersededSchema()) + monkeypatch.setitem(definition, "outputSchema", SupersededSchema()) + monkeypatch.setattr(mcp_tools, "_ensure_pydantic_schemas_applied", unexpected_generation) + definitions = public_definitions(PINS[1]) + assert hashlib.sha256(canonical(definitions)).hexdigest() == (EXPECTED_PUBLIC_SHA256[PINS[1]]) + + +def test_current_model_fallback_retains_exact_definitions_and_mutation_isolation(): + first = mcp_tools.get_tools_for_handler(SchemaHandler()) + names = {definition["name"] for definition in first} + expected = [t for t in mcp_tools.ADCP_TOOL_DEFINITIONS if t["name"] in names] + assert canonical(first) == canonical(expected) + snapshot = canonical(first) + for definition in first: + mutable_ids(definition) + definition["inputSchema"].clear() + repeated = mcp_tools.get_tools_for_handler(SchemaHandler()) + assert canonical(repeated) == snapshot + + +def test_unsupported_public_pin_never_reuses_a_warm_supported_schema(): + public_definitions(PINS[1]) + with pytest.raises(ValueError, match="no bundled AdCP schemas"): + public_definitions("3.2.0-rc.999") + unsupported = loader.get_mcp_schema("get_products", "request", version="3.2.0-rc.999") + assert unsupported is None + + +@pytest.mark.parametrize("version", PINS) +@pytest.mark.asyncio +async def test_mutation_cannot_change_mounted_discovery_registration_or_validation(version): + initial = public_definitions(version) + expected = next(t for t in initial if t["name"] == "get_products") + expected_input = deepcopy(expected["inputSchema"]) + expected_output = deepcopy(expected["outputSchema"]) + expected["inputSchema"]["properties"]["brief"] = {"type": "array"} + expected["outputSchema"].clear() + handler = PinnedSchemaHandler(version) + mcp = create_mcp_server(handler, stateless_http=True, allowed_hosts=["test"]) + mcp.settings.json_response = True + app = mcp.streamable_http_app() + headers = {"accept": "application/json, text/event-stream"} + async with LifespanManager(app): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://test", + follow_redirects=True, + ) as client: + response = await client.post( + "/mcp/", + json={"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": {}}, + headers=headers, + ) + assert response.status_code == 200 + tools = response.json()["result"]["tools"] + advertised = next(t for t in tools if t["name"] == "get_products") + assert advertised["inputSchema"] == expected_input + assert advertised["outputSchema"] == expected_output + response = await client.post( + "/mcp/", + json={ + "jsonrpc": "2.0", + "id": 2, + "method": "tools/call", + "params": {"name": "get_products", "arguments": {"brief": []}}, + }, + headers=headers, + ) + assert response.status_code == 200 + result = response.json() + assert "error" in result or result.get("result", {}).get("isError") is True + assert handler.calls == 0 + assert not validate_request("get_products", {"brief": []}, version=version).valid + a2a = create_a2a_server(PinnedSchemaHandler(version), name="schema-materialization") + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=a2a), base_url="http://test" + ) as client: + for path in ("/.well-known/agent.json", "/.well-known/agent-card.json"): + response = await client.get(path) + assert response.status_code == 200 + assert "get_products" in {skill["id"] for skill in response.json()["skills"]} + repeated = public_definitions(version) + assert hashlib.sha256(canonical(repeated)).hexdigest() == (EXPECTED_PUBLIC_SHA256[version])