From 8cd60e99c1aa0a9406fc6095a2a4b70f5b4d8367 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 26 Jul 2026 15:56:38 -0400 Subject: [PATCH 1/4] bench: complex multi-system workflow benchmarks (email + spreadsheet + API + multi-app) Two new deterministic, synthetic, multi-application benchmarks that close the complexity gap between the existing single-app linear form-fill benchmarks (5-15 steps, no branching, no exception paths) and real back-office work: - benchmark/ap_invoice: email intake (maildir) + PDF invoice documents + ERP 3-way match across two fixture apps (ERP + mail gateway), 32 executed actions over a 5-invoice worklist, two branch points (match route, discount eligibility), and four designed exception paths (missing PO, ambiguous duplicate, collateral adjacent-row overwrite, uncertain payment delivery routed to RECONCILIATION_REQUIRED with a suppressed idempotent retry via the Section-3 taxonomy from openadapt_flow/transaction.py). - benchmark/o2c_recon: order-to-cash reconciliation across two fixture apps (billing + ledger) with a CSV worklist in and a written-back results CSV re-read from disk, 26 executed actions over a 10-order worklist, a 3-way branch (match/adjust/missing with an explicit halt terminal), and four exception paths (missing record, ambiguous duplicate, stale snapshot optimistic-concurrency refusal, phantom results-sheet write). Both run every consequential write through the real Replayer api tier under a sealed-bundle standard-profile governed authorization with exact API identity contracts, per-surface out-of-band effect verification (read-only SQL, REST oracles, an outbox maildir file oracle, a CSV re-read oracle), and are judged by an independent direct-file-access ground truth with a full table-delta audit. Zero model calls; localhost only; all data synthetic. Measured (n=3 per scenario/arm, committed in results.json and pinned by tests): governed silent-incorrect-success 0/30 per benchmark, over-halts 0/30, healthy-path transaction outcome VERIFIED; the naive banner-oracle arm silently accepts the collateral overwrite and the phantom write-back (3/3 each). benchmark/BENCHMARK.md gains an honest old-vs-new complexity table; each new benchmark README states exactly what it does and does not prove (synthetic, deterministic, api-tier actuation, not a customer environment). New results.json artifacts are registered in public-artifacts.json. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01NyCHrzA1psrKMFfroYbzaM --- benchmark/BENCHMARK.md | 38 + benchmark/ap_invoice/README.md | 96 ++ benchmark/ap_invoice/fixtures.py | 671 ++++++++++++++ benchmark/ap_invoice/ground_truth.py | 170 ++++ benchmark/ap_invoice/results.json | 1250 ++++++++++++++++++++++++++ benchmark/ap_invoice/run.py | 449 +++++++++ benchmark/ap_invoice/workflow.py | 738 +++++++++++++++ benchmark/multiapp_common.py | 433 +++++++++ benchmark/o2c_recon/README.md | 89 ++ benchmark/o2c_recon/fixtures.py | 485 ++++++++++ benchmark/o2c_recon/ground_truth.py | 177 ++++ benchmark/o2c_recon/results.json | 1060 ++++++++++++++++++++++ benchmark/o2c_recon/run.py | 368 ++++++++ benchmark/o2c_recon/workflow.py | 424 +++++++++ public-artifacts.json | 8 + tests/test_ap_invoice_benchmark.py | 112 +++ tests/test_o2c_recon_benchmark.py | 102 +++ 17 files changed, 6670 insertions(+) create mode 100644 benchmark/ap_invoice/README.md create mode 100644 benchmark/ap_invoice/fixtures.py create mode 100644 benchmark/ap_invoice/ground_truth.py create mode 100644 benchmark/ap_invoice/results.json create mode 100644 benchmark/ap_invoice/run.py create mode 100644 benchmark/ap_invoice/workflow.py create mode 100644 benchmark/multiapp_common.py create mode 100644 benchmark/o2c_recon/README.md create mode 100644 benchmark/o2c_recon/fixtures.py create mode 100644 benchmark/o2c_recon/ground_truth.py create mode 100644 benchmark/o2c_recon/results.json create mode 100644 benchmark/o2c_recon/run.py create mode 100644 benchmark/o2c_recon/workflow.py create mode 100644 tests/test_ap_invoice_benchmark.py create mode 100644 tests/test_o2c_recon_benchmark.py diff --git a/benchmark/BENCHMARK.md b/benchmark/BENCHMARK.md index a70377da..6975477d 100644 --- a/benchmark/BENCHMARK.md +++ b/benchmark/BENCHMARK.md @@ -90,3 +90,41 @@ openadapt-flow benchmark --n-compiled 100 --n-agent 20 --out benchmark/ Requires `ANTHROPIC_API_KEY` (or `~/.anthropic/api_key`). The agent arm costs real money (about $5.43 at list price for 20 runs when this was generated). + +## Workflow complexity across the benchmark suite (2026-07-26) + +An honest assessment of what the suite exercised BEFORE the multi-system +benchmarks landed: mostly single-application, linear, worklist-driven form +fills of roughly 5-15 recorded steps, with the fault-model studies driving a +single consequential write. Real back-office work is longer, cross-system, +document- and email-driven, and exception-heavy. The `ap_invoice` and +`o2c_recon` benchmarks were added to close that gap; the table below states +the shape of each benchmark so the difference is not overstated either (the +new benchmarks actuate through the api tier and do not measure GUI +perception; see each README). + +| benchmark | steps (executed actions) | apps | input/output modalities | branching | exception paths | +|---|---|---|---|---|---| +| MockMed encounter (this file) | 6 compiled steps | 1 | browser GUI | none | none | +| MockLoan disbursement | ~6 compiled steps | 1 | browser GUI | none | none | +| `openemr_local` registration | ~15-25 UI actions | 1 | browser GUI (real EMR) | none | duplicate-search confirm | +| `openimis_claims` claim entry | ~10-20 UI actions | 1 | browser GUI (real AGPL app, repo-only) | none | none | +| `frappe_lending` loan application | ~10 UI actions | 1 | browser GUI (real app) | none | none | +| `canvas_ladder` / `rdp_ladder` / `citrix_ica_hdx` | 1-3 probe actions | 1 | pixel-only surface | none | qualification refusals | +| `effect_e2e` / `silent_wrong_action` / `lending_fault_model` | 1 consequential write | 1 | REST + SQLite | none | 10-class fault taxonomy | +| `effectbench` task pack | 1-3 writes per task | 1 | app API/DB oracles | none | per-task faults | +| **`ap_invoice`** (new) | **32** | **2** (ERP + mail gateway) | email in (maildir), PDF document, REST API, UI gateway, email out | 2 branch points (match route; discount eligibility) | 4: missing PO, ambiguous duplicate, collateral adjacent-row overwrite, uncertain payment delivery (`RECONCILIATION_REQUIRED` + suppressed retry) | +| **`o2c_recon`** (new) | **26** | **2** (billing + ledger) | CSV worklist in, CSV results write-back (re-read), REST API, UI gateway, 2 SQLite systems of record | 3-way branch (match / adjust / missing) | 4: missing record (explicit halt terminal), ambiguous duplicate, stale snapshot (optimistic concurrency), phantom file write | + +Both new benchmarks run every consequential write through the real +`Replayer`'s api actuation tier under a sealed-bundle, standard-profile +governed authorization, verify every write out-of-band per surface (read-only +SQL, REST oracles, a maildir read, a CSV re-read), and are judged by an +independent direct-file-access ground truth. Zero model calls; healthy-path +governed runs classify `VERIFIED` under the Section-3 transaction taxonomy. +Measured headline over both benchmarks (n=3 per cell): governed +silent-incorrect-success 0/30 runs, over-halts 0/30; naive banner-oracle +silent-incorrect-success 6/30 (the collateral overwrite and the phantom +write-back classes). Deterministic coverage +matrix, not a sampled incidence rate. See `benchmark/ap_invoice/README.md` +and `benchmark/o2c_recon/README.md` for what each does and does not prove. diff --git a/benchmark/ap_invoice/README.md b/benchmark/ap_invoice/README.md new file mode 100644 index 00000000..0d98a6bd --- /dev/null +++ b/benchmark/ap_invoice/README.md @@ -0,0 +1,96 @@ +# AP invoice benchmark (email + PDF + two apps + 3-way match) + +Synthetic, deterministic, multi-system workflow benchmark. Not a customer +environment, not publication evidence of production reliability. + +## The workflow (one governed run, healthy path) + +A vendor emails an invoice as a PDF attachment; the AP workflow processes the +whole intake queue end-to-end across TWO separate local applications: + +1. **Intake** (deterministic pre-pass, disclosed below): read the request + emails from the INBOX maildir, extract each PDF attachment, parse its + fields, hash the document, and derive one worklist row per invoice. +2. **Per invoice** (workflow-program LOOP body, executed by the real + `Replayer` through the api actuation tier): + - enter the invoice draft in the ERP through the **UI gateway** (this + system has no invoice-entry API; `POST /api/invoice/new` answers 405), + - attach the PDF's SHA-256 to the record (API), + - run the **3-way match** (invoice vs purchase order vs receipts) (API), + - **BRANCH** on the match route: a clean match continues; a price + mismatch is routed to the **AP exception queue** and the vendor gets a + hold notice by email, + - **BRANCH** on discount terms: eligible invoices get the early-payment + discount applied first; expired terms pay net, + - approve for payment (UI gateway), schedule the payment (API), and email + the vendor a confirmation through the mail gateway (a second + application, maildir-backed). +3. **Batch completion** record, then a success terminal. + +Healthy-path shape: 5 worklist invoices, **32 executed consequential +actions**, 2 applications, 5 input/output modalities (email in, PDF document, +REST API, UI gateway, email out), 2 branch points, plus 4 designed exception +scenarios. + +## Arms + +- `naive`: demo profile; every write is "verified" only against the + application's own painted acknowledgement banner (what a screen-echo + automation trusts). +- `governed`: sealed bundle, single-use standard-profile authorization, an + exact API identity contract on every consequential write, out-of-band + effect verification routed per record surface (read-only SQL over the ERP + SQLite file, a REST payments oracle, and the OUTBOX **maildir read from + disk** for every sent email), a collateral guard on the adjacent grid row, + and an at-most-once idempotency ledger. + +## Scenarios and measured outcomes (n=3 per cell, deterministic) + +| scenario | naive (banner oracle) | governed | +|---|---|---| +| `healthy` | completes; `COMPLETED_UNVERIFIED` (never billable) | **`VERIFIED`** (all effects confirmed out-of-band; billable) | +| `missing_po` | safe halt at entry | safe halt; `HALTED_BEFORE_EFFECT`; nothing persisted | +| `duplicate_invoice` | safe halt at entry | safe halt; `HALTED_BEFORE_EFFECT`; still exactly one invoice | +| `collateral_approve` | **SILENT WRONG** (adjacent invoice corrupted, banner says success) | **caught**: collateral guard refutes; `RECONCILIATION_REQUIRED` | +| `payment_confirm_outage` | completes (cannot know the write landed) | `RECONCILIATION_REQUIRED`; retry under the same idempotency key SUPPRESSED (`REJECTED_POLICY`); ground truth: exactly one payment | + +Headline (30 runs + 10 retry runs): governed silent-incorrect-success **0**, +over-halts on the healthy path **0**, model calls **0**; naive +silent-incorrect-success **3/3** on the collateral scenario. Every run is +judged by an independent ground truth that opens the SQLite file and the +maildir directly (no HTTP, no banner, no verifier verdict reaches it) and +audits every table's delta. + +Reproduce: `python -m benchmark.ap_invoice.run --n 3` (localhost only, $0). +Pinned in CI by `tests/test_ap_invoice_benchmark.py`. + +## What this proves, and what it does not + +Proves (within a synthetic closed world): + +- the engine's Phase-2 workflow-program machinery (loop + guarded branches + + explicit exception routing) executes a 30+ step, two-application, + email/PDF/spreadsheet-era back-office flow deterministically at $0; +- the governed contract stack (identity bindings, per-surface out-of-band + effect verification including a maildir file oracle, collateral guards, + idempotency, the Section-3 transaction taxonomy) yields zero + silent-incorrect-successes across the designed fault classes while a + banner-echo oracle silently accepts the collateral overwrite; +- uncertain delivery is routed to `RECONCILIATION_REQUIRED` and is never + blind-retried. + +Does NOT prove: + +- anything about GUI perception. Every consequential action here is actuated + through the replayer's **api tier** (the fixture's "UI gateway" models a + screen that has no API, but it is still driven over HTTP). Pixel-level + resolution, identity-band OCR, and visual drift are measured by the + existing MockMed/OpenEMR/canvas/RDP benchmarks, not this one. +- anything about model-based document understanding. The PDF and email + parsing in intake is a deterministic fixture parser over fixture documents; + real invoice extraction (OCR/ML) is out of scope and unmeasured. +- production incidence rates. The fault taxonomy is hand-authored and + deterministic (a coverage matrix, not a sampled population); the fixture + applications are cooperative and local. +- customer-environment behavior. All vendors, amounts, and emails are + synthetic; no real ERP or mail server is involved. diff --git a/benchmark/ap_invoice/fixtures.py b/benchmark/ap_invoice/fixtures.py new file mode 100644 index 00000000..130e9fa4 --- /dev/null +++ b/benchmark/ap_invoice/fixtures.py @@ -0,0 +1,671 @@ +"""Deterministic multi-application fixture for the AP invoice benchmark. + +Two separate local applications, each with its own process-independent system +of record, plus a document store: + +1. **ERP service** (accounts-payable ledger): an on-disk SQLite database + behind a threaded HTTP server. It exposes TWO write surfaces on purpose: + + - ``/ui/...`` -- the "UI gateway": the only surface that accepts invoice + entry and approval (the fixture's stand-in for screens that have no API; + ``POST /api/invoice/new`` answers 405). Its handlers paint an optimistic + UI banner BEFORE/REGARDLESS of what actually persisted, exactly like a + real thick-client form. + - ``/api/...`` -- the real REST surface for document attachment, 3-way + match, discounts, payments, exceptions, and batch completion. + + Fault hooks are injected at the persistence boundary (never visible in the + HTTP status or the banner) so the benchmark can measure what each oracle + arm actually catches. + +2. **Mailer service**: a maildir-backed mail gateway. The INBOX maildir is + seeded with synthetic vendor request emails (RFC822, each with a PDF + invoice attachment); ``POST /api/send`` delivers a confirmation message + into the OUTBOX maildir. Out-of-band email verification reads the OUTBOX + maildir directly from disk. + +All data is synthetic (fake vendors, fake amounts). Localhost only. Zero model +calls. +""" + +from __future__ import annotations + +import json +import sqlite3 +import threading +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Callable, Optional +from urllib.parse import urlparse + +from benchmark.multiapp_common import init_maildir, write_maildir_message + +#: The seeded ADJACENT record every scenario carries: a draft invoice sitting +#: one row away from the target in the AP grid. The collateral fault corrupts +#: it while the target write succeeds and the banner still paints success. +ADJACENT_INVOICE = "INV-2090" +ADJACENT_VENDOR = "V-100" +ADJACENT_AMOUNT = "450.00" + +#: The pre-existing posted invoice the ambiguous-duplicate scenario re-presents. +DUPLICATE_INVOICE = "INV-1201" + +VENDORS = ( + ("V-100", "Acme Supply Co"), + ("V-200", "Beta Industrial"), + ("V-300", "Gamma Logistics"), +) + +#: (po_number, vendor_id, item, qty, unit_price, amount) -- what Purchasing +#: ordered and Receiving booked; the 3-way match compares invoices against it. +PURCHASE_ORDERS = ( + ("PO-501", "V-100", "safety gloves", "40", "30.00", "1200.00"), + ("PO-502", "V-200", "bearing sets", "20", "43.00", "860.00"), + ("PO-503", "V-300", "freight", "1", "380.00", "380.00"), + ("PO-504", "V-100", "label rolls", "9", "11.10", "99.90"), + ("PO-505", "V-200", "coolant drums", "6", "121.50", "729.00"), + ("PO-506", "V-100", "hex fasteners", "50", "6.40", "320.00"), + ("PO-507", "V-300", "freight", "1", "212.75", "212.75"), +) + + +@dataclass(frozen=True) +class ErpHandle: + base_url: str + db_path: Path + stop: Callable[[], None] + + +@dataclass(frozen=True) +class MailerHandle: + base_url: str + inbox: Path + outbox: Path + stop: Callable[[], None] + + +@dataclass +class _Faults: + """Persistence-boundary fault switches for one scenario.""" + + collateral_adjacent_on_approve: bool = False + payments_read_down_after_write: bool = False + payments_read_down: bool = field(default=False, compare=False) + + +def _init_db(path: Path) -> None: + conn = sqlite3.connect(str(path)) + try: + conn.executescript( + """ + DROP TABLE IF EXISTS vendors; + DROP TABLE IF EXISTS purchase_orders; + DROP TABLE IF EXISTS receipts; + DROP TABLE IF EXISTS invoices; + DROP TABLE IF EXISTS payments; + DROP TABLE IF EXISTS ap_exceptions; + DROP TABLE IF EXISTS batches; + DROP TABLE IF EXISTS banner; + CREATE TABLE vendors (vendor_id TEXT PRIMARY KEY, name TEXT); + CREATE TABLE purchase_orders ( + po_number TEXT PRIMARY KEY, + vendor_id TEXT, + item TEXT, + qty TEXT, + unit_price TEXT, + amount TEXT, + status TEXT + ); + CREATE TABLE receipts ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + po_number TEXT, + qty_received TEXT + ); + CREATE TABLE invoices ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invoice_id TEXT, + vendor_id TEXT, + po_number TEXT, + amount TEXT, + doc_sha256 TEXT, + status TEXT, + discount_applied TEXT, + amount_payable TEXT + ); + CREATE TABLE payments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invoice_id TEXT, + amount TEXT, + status TEXT + ); + CREATE TABLE ap_exceptions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invoice_id TEXT, + reason TEXT + ); + CREATE TABLE batches ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + batch_id TEXT, + processed TEXT + ); + CREATE TABLE banner ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + invoice_id TEXT, + event TEXT, + detail TEXT + ); + """ + ) + conn.commit() + finally: + conn.close() + + +class ErpStore: + """Write-side wrapper over the ERP database, with fault injection. + + Fresh connection per operation; writes serialized under a lock. Faults are + applied at the persistence boundary AFTER the handler has decided its HTTP + status and painted the banner, so a fault is invisible to a screen-echo + oracle by construction. + """ + + def __init__(self, path: Path) -> None: + self.path = path + self._lock = threading.Lock() + self.faults = _Faults() + + def _connect(self) -> sqlite3.Connection: + return sqlite3.connect(str(self.path), timeout=5.0) + + # -- lifecycle ----------------------------------------------------------- + + def reset( + self, *, faults: Optional[list[str]] = None, seed_duplicate: bool = False + ) -> None: + with self._lock: + _init_db(self.path) + self.faults = _Faults( + collateral_adjacent_on_approve="collateral_adjacent_on_approve" + in (faults or []), + payments_read_down_after_write="payments_read_down_after_write" + in (faults or []), + ) + conn = self._connect() + try: + conn.executemany( + "INSERT INTO vendors (vendor_id, name) VALUES (?, ?)", VENDORS + ) + conn.executemany( + "INSERT INTO purchase_orders " + "(po_number, vendor_id, item, qty, unit_price, amount, status) " + "VALUES (?, ?, ?, ?, ?, ?, 'open')", + PURCHASE_ORDERS, + ) + conn.executemany( + "INSERT INTO receipts (po_number, qty_received) VALUES (?, ?)", + [(po[0], po[3]) for po in PURCHASE_ORDERS], + ) + # The always-present adjacent draft the collateral fault targets. + conn.execute( + "INSERT INTO invoices (invoice_id, vendor_id, po_number, " + "amount, doc_sha256, status, discount_applied, amount_payable) " + "VALUES (?, ?, ?, ?, '', 'draft', 'none', '')", + (ADJACENT_INVOICE, ADJACENT_VENDOR, "PO-506", ADJACENT_AMOUNT), + ) + if seed_duplicate: + conn.execute( + "INSERT INTO invoices (invoice_id, vendor_id, po_number, " + "amount, doc_sha256, status, discount_applied, " + "amount_payable) " + "VALUES (?, 'V-200', 'PO-505', '729.00', '', 'posted', " + "'none', '729.00')", + (DUPLICATE_INVOICE,), + ) + conn.commit() + finally: + conn.close() + + # -- helpers ------------------------------------------------------------- + + def _banner( + self, conn: sqlite3.Connection, invoice_id: str, event: str, detail: str + ) -> None: + conn.execute( + "INSERT INTO banner (invoice_id, event, detail) VALUES (?, ?, ?)", + (invoice_id, event, detail), + ) + + def _invoice( + self, conn: sqlite3.Connection, invoice_id: str + ) -> Optional[sqlite3.Row]: + conn.row_factory = sqlite3.Row + return conn.execute( + "SELECT * FROM invoices WHERE invoice_id = ?", (invoice_id,) + ).fetchone() + + # -- UI gateway writes --------------------------------------------------- + + def ui_invoice_new(self, body: dict[str, Any]) -> tuple[int, str]: + invoice_id = str(body.get("invoice_id", "")) + vendor_id = str(body.get("vendor_id", "")) + po_number = str(body.get("po_number", "")) + amount = str(body.get("amount", "")) + with self._lock: + conn = self._connect() + try: + # The optimistic UI paints the acknowledgement FIRST (like a + # form that shows "Saved" as soon as the button is pressed). + self._banner(conn, invoice_id, "invoice_created", amount) + po = conn.execute( + "SELECT po_number FROM purchase_orders WHERE po_number = ?", + (po_number,), + ).fetchone() + if po is None: + conn.commit() + return 404, f"purchase order {po_number} not found" + existing = conn.execute( + "SELECT COUNT(*) FROM invoices WHERE invoice_id = ?", + (invoice_id,), + ).fetchone()[0] + if existing: + conn.commit() + return ( + 409, + f"invoice {invoice_id} already exists (ambiguous " + "duplicate: possible resubmission)", + ) + conn.execute( + "INSERT INTO invoices (invoice_id, vendor_id, po_number, " + "amount, doc_sha256, status, discount_applied, amount_payable) " + "VALUES (?, ?, ?, ?, '', 'draft', 'none', '')", + (invoice_id, vendor_id, po_number, amount), + ) + conn.commit() + return 200, "created" + finally: + conn.close() + + def ui_invoice_approve(self, body: dict[str, Any]) -> tuple[int, str]: + invoice_id = str(body.get("invoice_id", "")) + amount_payable = str(body.get("amount_payable", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, invoice_id, "invoice_approved", amount_payable) + row = self._invoice(conn, invoice_id) + if row is None: + conn.commit() + return 404, f"invoice {invoice_id} not found" + if row["status"] not in ("matched", "discounted"): + conn.commit() + return 409, f"invoice {invoice_id} is not matched" + expected = ( + row["amount_payable"] + if row["discount_applied"] not in ("", "none") + else row["amount"] + ) + if amount_payable != expected: + conn.commit() + return 409, "amount payable does not match the invoice terms" + conn.execute( + "UPDATE invoices SET status = 'approved', amount_payable = ? " + "WHERE invoice_id = ?", + (amount_payable, invoice_id), + ) + if self.faults.collateral_adjacent_on_approve: + # The grid-row trap: the save handler ALSO commits the + # adjacent row (a real thick-client class of bug: shared + # dirty-row state). Status stays 200; the banner already + # painted success for the target only. + conn.execute( + "UPDATE invoices SET status = 'approved', " + "amount_payable = '999.00' WHERE invoice_id = ?", + (ADJACENT_INVOICE,), + ) + conn.commit() + return 200, "approved" + finally: + conn.close() + + # -- API writes ---------------------------------------------------------- + + def api_attach_document(self, body: dict[str, Any]) -> tuple[int, str]: + invoice_id = str(body.get("invoice_id", "")) + doc_sha256 = str(body.get("doc_sha256", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, invoice_id, "document_attached", doc_sha256[:12]) + row = self._invoice(conn, invoice_id) + if row is None: + conn.commit() + return 404, f"invoice {invoice_id} not found" + conn.execute( + "UPDATE invoices SET doc_sha256 = ? WHERE invoice_id = ?", + (doc_sha256, invoice_id), + ) + conn.commit() + return 200, "attached" + finally: + conn.close() + + def api_match(self, body: dict[str, Any]) -> tuple[int, str]: + invoice_id = str(body.get("invoice_id", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, invoice_id, "match_run", "") + row = self._invoice(conn, invoice_id) + if row is None: + conn.commit() + return 404, f"invoice {invoice_id} not found" + conn.row_factory = sqlite3.Row + po = conn.execute( + "SELECT * FROM purchase_orders WHERE po_number = ?", + (row["po_number"],), + ).fetchone() + received = conn.execute( + "SELECT COALESCE(SUM(CAST(qty_received AS REAL)), 0) " + "FROM receipts WHERE po_number = ?", + (row["po_number"],), + ).fetchone()[0] + status = "price_mismatch" + if ( + po is not None + and po["amount"] == row["amount"] + and float(po["qty"]) <= float(received) + ): + status = "matched" + conn.execute( + "UPDATE invoices SET status = ? WHERE invoice_id = ?", + (status, invoice_id), + ) + conn.commit() + return 200, status + finally: + conn.close() + + def api_discount(self, body: dict[str, Any]) -> tuple[int, str]: + invoice_id = str(body.get("invoice_id", "")) + discount_applied = str(body.get("discount_applied", "")) + amount_payable = str(body.get("amount_payable", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, invoice_id, "discount_applied", discount_applied) + row = self._invoice(conn, invoice_id) + if row is None: + conn.commit() + return 404, f"invoice {invoice_id} not found" + if row["status"] != "matched": + conn.commit() + return 409, f"invoice {invoice_id} is not matched" + conn.execute( + "UPDATE invoices SET status = 'discounted', " + "discount_applied = ?, amount_payable = ? WHERE invoice_id = ?", + (discount_applied, amount_payable, invoice_id), + ) + conn.commit() + return 200, "discounted" + finally: + conn.close() + + def api_payment(self, body: dict[str, Any]) -> tuple[int, str]: + invoice_id = str(body.get("invoice_id", "")) + amount = str(body.get("amount", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, invoice_id, "payment_scheduled", amount) + row = self._invoice(conn, invoice_id) + if row is None: + conn.commit() + return 404, f"invoice {invoice_id} not found" + if row["status"] != "approved": + conn.commit() + return 409, f"invoice {invoice_id} is not approved" + conn.execute( + "INSERT INTO payments (invoice_id, amount, status) " + "VALUES (?, ?, 'scheduled')", + (invoice_id, amount), + ) + conn.commit() + finally: + conn.close() + if self.faults.payments_read_down_after_write: + # The payment COMMITTED, then the payment-status read surface went + # down: the uncertain-delivery class. The client cannot distinguish + # "landed" from "lost" by reading. + self.faults.payments_read_down = True + return 200, "scheduled" + + def api_exception(self, body: dict[str, Any]) -> tuple[int, str]: + invoice_id = str(body.get("invoice_id", "")) + reason = str(body.get("reason", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, invoice_id, "exception_routed", reason) + row = self._invoice(conn, invoice_id) + if row is None: + conn.commit() + return 404, f"invoice {invoice_id} not found" + conn.execute( + "INSERT INTO ap_exceptions (invoice_id, reason) VALUES (?, ?)", + (invoice_id, reason), + ) + conn.execute( + "UPDATE invoices SET status = 'held' WHERE invoice_id = ?", + (invoice_id,), + ) + conn.commit() + return 200, "held" + finally: + conn.close() + + def api_batch_complete(self, body: dict[str, Any]) -> tuple[int, str]: + batch_id = str(body.get("batch_id", "")) + processed = str(body.get("processed", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, batch_id, "batch_completed", processed) + conn.execute( + "INSERT INTO batches (batch_id, processed) VALUES (?, ?)", + (batch_id, processed), + ) + conn.commit() + return 200, "completed" + finally: + conn.close() + + # -- reads --------------------------------------------------------------- + + def read_table(self, query: str) -> list[dict[str, Any]]: + conn = self._connect() + try: + conn.row_factory = sqlite3.Row + return [dict(r) for r in conn.execute(query).fetchall()] + finally: + conn.close() + + +def _json_handler( + route: Callable[[str, str, dict[str, Any]], tuple[int, dict[str, Any]]], +) -> type[BaseHTTPRequestHandler]: + class _Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: object) -> None: # noqa: A002 + pass + + def _send(self, status: int, payload: dict[str, Any]) -> None: + body = json.dumps(payload).encode("utf-8") + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + + def _body(self) -> dict[str, Any]: + length = int(self.headers.get("Content-Length", 0) or 0) + if not length: + return {} + try: + loaded = json.loads(self.rfile.read(length).decode("utf-8")) + return loaded if isinstance(loaded, dict) else {} + except (ValueError, UnicodeDecodeError): + return {} + + def do_GET(self) -> None: # noqa: N802 + status, payload = route("GET", urlparse(self.path).path, {}) + self._send(status, payload) + + def do_POST(self) -> None: # noqa: N802 + status, payload = route("POST", urlparse(self.path).path, self._body()) + self._send(status, payload) + + return _Handler + + +def serve_erp(db_path: Path, *, host: str = "127.0.0.1", port: int = 0) -> ErpHandle: + """Initialize and serve the ERP system of record on a loopback port.""" + _init_db(db_path) + store = ErpStore(db_path) + + def route( + method: str, path: str, body: dict[str, Any] + ) -> tuple[int, dict[str, Any]]: + if method == "GET": + if path == "/api/payments": + if store.faults.payments_read_down: + return 500, {"error": "payment status service unavailable"} + return 200, { + "records": store.read_table( + "SELECT id, invoice_id, amount, status FROM payments" + ) + } + if path == "/api/ui/banner": + return 200, { + "records": store.read_table( + "SELECT id, invoice_id, event, detail FROM banner" + ) + } + if path == "/api/purchase_orders": + return 200, { + "records": store.read_table( + "SELECT po_number, vendor_id, item, qty, unit_price, " + "amount, status FROM purchase_orders" + ) + } + return 404, {"error": "not found"} + if path == "/api/reset": + store.reset( + faults=[str(f) for f in body.get("faults", [])], + seed_duplicate=bool(body.get("seed_duplicate")), + ) + return 200, {"ok": True} + if path == "/api/invoice/new": + # The API-vs-UI ladder: this system has NO invoice-entry API. + return 405, {"error": "invoice entry is only available through the UI"} + handlers = { + "/ui/invoice/new": store.ui_invoice_new, + "/ui/invoice/approve": store.ui_invoice_approve, + "/api/invoice/document": store.api_attach_document, + "/api/invoice/match": store.api_match, + "/api/invoice/discount": store.api_discount, + "/api/payment": store.api_payment, + "/api/exception": store.api_exception, + "/api/batch/complete": store.api_batch_complete, + } + handler = handlers.get(path) + if handler is None: + return 404, {"error": "not found"} + status, detail = handler(body) + return status, {"ok": 200 <= status < 300, "detail": detail} + + httpd = ThreadingHTTPServer((host, port), _json_handler(route)) + actual_port = httpd.server_address[1] + thread = threading.Thread( + target=httpd.serve_forever, name="ap-invoice-erp", daemon=True + ) + thread.start() + + def stop() -> None: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + return ErpHandle( + base_url=f"http://{host}:{actual_port}", db_path=db_path, stop=stop + ) + + +def serve_mailer( + mail_root: Path, *, host: str = "127.0.0.1", port: int = 0 +) -> MailerHandle: + """Serve the maildir-backed mail gateway on a loopback port.""" + inbox = init_maildir(mail_root / "inbox") + outbox = init_maildir(mail_root / "outbox") + banner: list[dict[str, Any]] = [] + lock = threading.Lock() + + def route( + method: str, path: str, body: dict[str, Any] + ) -> tuple[int, dict[str, Any]]: + if method == "GET": + if path == "/api/ui/banner": + with lock: + return 200, {"records": list(banner)} + return 404, {"error": "not found"} + if path == "/api/reset": + with lock: + banner.clear() + for sub in ("new", "cur", "tmp"): + for item in (outbox / sub).glob("*"): + item.unlink() + return 200, {"ok": True} + if path == "/api/send": + name = str(body.get("name", "")) + invoice_id = str(body.get("invoice_id", "")) + to_addr = str(body.get("to", "")) + subject = str(body.get("subject", "")) + text = str(body.get("body", "")) + if not name or not to_addr: + return 400, {"error": "name and to are required"} + with lock: + banner.append( + { + "id": len(banner) + 1, + "invoice_id": invoice_id, + "event": "mail_sent", + "detail": subject, + } + ) + message = ( + f"From: ap-bot@example-corp.test\r\nTo: {to_addr}\r\n" + f"Subject: {subject}\r\n" + f"X-OpenAdapt-Invoice: {invoice_id}\r\n\r\n{text}\r\n" + ).encode("utf-8") + write_maildir_message(outbox, name, message) + return 200, {"ok": True} + return 404, {"error": "not found"} + + httpd = ThreadingHTTPServer((host, port), _json_handler(route)) + actual_port = httpd.server_address[1] + thread = threading.Thread( + target=httpd.serve_forever, name="ap-invoice-mailer", daemon=True + ) + thread.start() + + def stop() -> None: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + return MailerHandle( + base_url=f"http://{host}:{actual_port}", inbox=inbox, outbox=outbox, stop=stop + ) diff --git a/benchmark/ap_invoice/ground_truth.py b/benchmark/ap_invoice/ground_truth.py new file mode 100644 index 00000000..b13cc339 --- /dev/null +++ b/benchmark/ap_invoice/ground_truth.py @@ -0,0 +1,170 @@ +"""Independent ground truth for the AP invoice benchmark. + +Bypasses BOTH fixture services entirely: opens the ERP SQLite file directly +(read-only, its own connection) and lists/reads the OUTBOX maildir directly +from disk. No HTTP status, banner row, or verifier verdict ever reaches this +module; it re-derives the intended end state from the worklist rows with its +OWN logic and reports INVARIANT VIOLATIONS (wrong business state), plus a +full per-table delta audit so an unexpected write anywhere in the database is +caught even when every targeted check passes. +""" + +from __future__ import annotations + +import hashlib +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from benchmark.ap_invoice.fixtures import ADJACENT_AMOUNT, ADJACENT_INVOICE +from benchmark.multiapp_common import read_maildir_messages + +#: The app's own echo surface; excluded from the delta audit on purpose (it is +#: the thing a screen-echo oracle reads, not part of the business state). +_ECHO_TABLES = frozenset({"banner"}) + + +@dataclass +class Snapshot: + tables: dict[str, list[dict[str, Any]]] + outbox: dict[str, str] + + +@dataclass +class Verdict: + violations: list[str] = field(default_factory=list) + table_deltas: dict[str, int] = field(default_factory=dict) + + @property + def correct(self) -> bool: + return not self.violations + + +def capture(db_path: Path, outbox: Path) -> Snapshot: + """Direct read-only snapshot of every persisted table + the outbox.""" + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5.0) + try: + conn.row_factory = sqlite3.Row + names = [ + r["name"] + for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%' ORDER BY name" + ).fetchall() + ] + tables = { + name: [dict(r) for r in conn.execute(f"SELECT * FROM {name}").fetchall()] + for name in names + } + finally: + conn.close() + outbox_state = { + name: hashlib.sha256(data).hexdigest() + for name, data in read_maildir_messages(outbox).items() + } + return Snapshot(tables=tables, outbox=outbox_state) + + +def _rows(snapshot: Snapshot, table: str, **where: str) -> list[dict[str, Any]]: + return [ + row + for row in snapshot.tables.get(table, []) + if all(str(row.get(k)) == v for k, v in where.items()) + ] + + +def _mail_for(snapshot: Snapshot, outbox_dir: Path, name: str, invoice_id: str) -> bool: + if name not in snapshot.outbox: + return False + for candidate, data in read_maildir_messages(outbox_dir).items(): + if candidate == name: + return f"X-OpenAdapt-Invoice: {invoice_id}".encode() in data + return False + + +def judge( + scenario: str, + rows: list[dict[str, str]], + before: Snapshot, + after: Snapshot, + *, + outbox_dir: Path, + completed: bool, +) -> Verdict: + """Classify the persisted end state against the scenario's invariants. + + ``completed`` is whether the arm REPORTED a completed run; it selects + which completeness invariants apply (a safe halt legitimately leaves the + flow unfinished; it must never leave the state WRONG). + """ + verdict = Verdict() + verdict.table_deltas = { + name: len(after.tables.get(name, [])) - len(before.tables.get(name, [])) + for name in sorted(set(before.tables) | set(after.tables)) + if name not in _ECHO_TABLES + } + + # -- invariants that must hold in EVERY scenario and arm ----------------- + adjacent = _rows(after, "invoices", invoice_id=ADJACENT_INVOICE) + if len(adjacent) != 1: + verdict.violations.append("adjacent_invoice_missing_or_duplicated") + elif ( + adjacent[0]["status"] != "draft" + or str(adjacent[0]["amount"]) != ADJACENT_AMOUNT + or str(adjacent[0]["amount_payable"]) != "" + ): + verdict.violations.append("adjacent_invoice_modified") + + for row in rows: + payments = _rows(after, "payments", invoice_id=row["invoice_id"]) + if len(payments) > 1: + verdict.violations.append(f"duplicate_payment:{row['invoice_id']}") + invoices = _rows(after, "invoices", invoice_id=row["invoice_id"]) + if scenario == "duplicate_invoice": + if len(invoices) != 1: + verdict.violations.append( + f"ambiguous_duplicate_persisted:{row['invoice_id']}" + ) + elif len(invoices) > 1: + verdict.violations.append(f"duplicate_invoice_rows:{row['invoice_id']}") + if scenario == "missing_po" and invoices: + verdict.violations.append( + f"invoice_entered_against_missing_po:{row['invoice_id']}" + ) + for payment in payments: + if str(payment["amount"]) != row["amount_payable"]: + verdict.violations.append(f"payment_amount_wrong:{row['invoice_id']}") + if row["route"] == "mismatch" and payments: + verdict.violations.append(f"mismatched_invoice_paid:{row['invoice_id']}") + + # -- completeness invariants: only a COMPLETED run owes the end state ---- + if completed: + for row in rows: + invoice_id = row["invoice_id"] + invoices = _rows(after, "invoices", invoice_id=invoice_id) + if len(invoices) != 1: + verdict.violations.append(f"invoice_not_entered_once:{invoice_id}") + continue + invoice = invoices[0] + if str(invoice["doc_sha256"]) != row["doc_sha256"]: + verdict.violations.append(f"document_digest_wrong:{invoice_id}") + if row["route"] == "ok": + if invoice["status"] != "approved": + verdict.violations.append(f"invoice_not_approved:{invoice_id}") + if str(invoice["amount_payable"]) != row["amount_payable"]: + verdict.violations.append(f"payable_amount_wrong:{invoice_id}") + if len(_rows(after, "payments", invoice_id=invoice_id)) != 1: + verdict.violations.append(f"payment_missing:{invoice_id}") + else: + if invoice["status"] != "held": + verdict.violations.append(f"mismatch_not_held:{invoice_id}") + if len(_rows(after, "ap_exceptions", invoice_id=invoice_id)) != 1: + verdict.violations.append(f"exception_entry_missing:{invoice_id}") + if not _mail_for(after, outbox_dir, row["mail_name"], invoice_id): + verdict.violations.append(f"vendor_email_missing:{invoice_id}") + batches = after.tables.get("batches", []) + if len(batches) != 1 or str(batches[0].get("processed")) != str(len(rows)): + verdict.violations.append("batch_completion_wrong") + + return verdict diff --git a/benchmark/ap_invoice/results.json b/benchmark/ap_invoice/results.json new file mode 100644 index 00000000..e8d5cc16 --- /dev/null +++ b/benchmark/ap_invoice/results.json @@ -0,0 +1,1250 @@ +{ + "benchmark": "ap_invoice", + "instrument": "multi-application AP invoice intake: email + PDF + ERP 3-way match + payments + exception queue, end-to-end through the real replayer's api tier", + "generated_at": "2026-07-26T19:55:16.190170+00:00", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "n_per_scenario": 3, + "arms": [ + "naive", + "governed" + ], + "scenarios": [ + "healthy", + "missing_po", + "duplicate_invoice", + "collateral_approve", + "payment_confirm_outage" + ], + "workflow_shape": { + "applications": [ + "erp", + "mail_gateway" + ], + "modalities": [ + "maildir email in/out", + "pdf document", + "rest api", + "ui gateway (no entry api)", + "sqlite system of record" + ], + "healthy_worklist_rows": 5, + "healthy_executed_action_steps": 32, + "branches": [ + "match route (ok vs mismatch)", + "discount (eligible vs expired)" + ], + "exception_paths": [ + "missing purchase order (halt at entry)", + "ambiguous duplicate invoice (halt at entry)", + "collateral adjacent-record overwrite (governed: caught)", + "payment confirmation outage (governed: RECONCILIATION_REQUIRED + suppressed retry)" + ] + }, + "headline": { + "governed_verified": 3, + "governed_silent_wrong": 0, + "governed_over_halts": 0, + "governed_reconciliation_required": 6, + "governed_suppressed_retries": 3, + "naive_silent_wrong": 3, + "model_calls_total": 0 + }, + "metrics": { + "per_arm": { + "naive": { + "n_runs": 15, + "verified": 0, + "completed_unverified": 9, + "halts": 6, + "safe_halts": 6, + "caught": 0, + "reconciliation_required": 0, + "suppressed_retries": 3, + "silent_wrong": 3, + "over_halts": 0, + "model_calls": 0, + "per_scenario": { + "healthy": { + "n": 3, + "reported_success": 3, + "transaction_outcomes": [ + "COMPLETED_UNVERIFIED" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 0 + }, + "missing_po": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "duplicate_invoice": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "collateral_approve": { + "n": 3, + "reported_success": 3, + "transaction_outcomes": [ + "COMPLETED_UNVERIFIED" + ], + "silent_wrong": 3, + "caught": 0, + "safe_halt": 0 + }, + "payment_confirm_outage": { + "n": 3, + "reported_success": 3, + "transaction_outcomes": [ + "COMPLETED_UNVERIFIED" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 0 + } + } + }, + "governed": { + "n_runs": 15, + "verified": 3, + "completed_unverified": 0, + "halts": 12, + "safe_halts": 9, + "caught": 3, + "reconciliation_required": 6, + "suppressed_retries": 3, + "silent_wrong": 0, + "over_halts": 0, + "model_calls": 0, + "per_scenario": { + "healthy": { + "n": 3, + "reported_success": 3, + "transaction_outcomes": [ + "VERIFIED" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 0 + }, + "missing_po": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "duplicate_invoice": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "collateral_approve": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "RECONCILIATION_REQUIRED" + ], + "silent_wrong": 0, + "caught": 3, + "safe_halt": 0 + }, + "payment_confirm_outage": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "RECONCILIATION_REQUIRED" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + } + } + } + } + }, + "runs": [ + { + "scenario": "healthy", + "arm": "naive", + "i": 0, + "worklist_rows": 5, + "executed_action_steps": 32, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 1, + "batches": 1, + "invoices": 5, + "payments": 4, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "naive", + "i": 1, + "worklist_rows": 5, + "executed_action_steps": 32, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 1, + "batches": 1, + "invoices": 5, + "payments": 4, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "naive", + "i": 2, + "worklist_rows": 5, + "executed_action_steps": 32, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 1, + "batches": 1, + "invoices": 5, + "payments": 4, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "missing_po", + "arm": "naive", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_po", + "arm": "naive", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_po", + "arm": "naive", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "duplicate_invoice", + "arm": "naive", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "duplicate_invoice", + "arm": "naive", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "duplicate_invoice", + "arm": "naive", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "collateral_approve", + "arm": "naive", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 7, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [ + "adjacent_invoice_modified" + ], + "gt_correct": false, + "table_deltas": { + "ap_exceptions": 0, + "batches": 1, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": true, + "caught": false, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "collateral_approve", + "arm": "naive", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 7, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [ + "adjacent_invoice_modified" + ], + "gt_correct": false, + "table_deltas": { + "ap_exceptions": 0, + "batches": 1, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": true, + "caught": false, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "collateral_approve", + "arm": "naive", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 7, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [ + "adjacent_invoice_modified" + ], + "gt_correct": false, + "table_deltas": { + "ap_exceptions": 0, + "batches": 1, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": true, + "caught": false, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "payment_confirm_outage", + "arm": "naive", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 7, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": "REJECTED_POLICY", + "retry_suppressed": true, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 1, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "payment_confirm_outage", + "arm": "naive", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 7, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": "REJECTED_POLICY", + "retry_suppressed": true, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 1, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "payment_confirm_outage", + "arm": "naive", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 7, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": "REJECTED_POLICY", + "retry_suppressed": true, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 1, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "governed", + "i": 0, + "worklist_rows": 5, + "executed_action_steps": 32, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED", + "transaction_billable": true, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 1, + "batches": 1, + "invoices": 5, + "payments": 4, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "governed", + "i": 1, + "worklist_rows": 5, + "executed_action_steps": 32, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED", + "transaction_billable": true, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 1, + "batches": 1, + "invoices": 5, + "payments": 4, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "governed", + "i": 2, + "worklist_rows": 5, + "executed_action_steps": 32, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED", + "transaction_billable": true, + "terminal_outcome": "success", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 1, + "batches": 1, + "invoices": 5, + "payments": 4, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "missing_po", + "arm": "governed", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_po", + "arm": "governed", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_po", + "arm": "governed", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "duplicate_invoice", + "arm": "governed", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "duplicate_invoice", + "arm": "governed", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "duplicate_invoice", + "arm": "governed", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 0, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "collateral_approve", + "arm": "governed", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 4, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [ + "adjacent_invoice_modified" + ], + "gt_correct": false, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 1, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": true, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "collateral_approve", + "arm": "governed", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 4, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [ + "adjacent_invoice_modified" + ], + "gt_correct": false, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 1, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": true, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "collateral_approve", + "arm": "governed", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 4, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": null, + "retry_suppressed": null, + "gt_violations": [ + "adjacent_invoice_modified" + ], + "gt_correct": false, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 1, + "payments": 0, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": true, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "payment_confirm_outage", + "arm": "governed", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 5, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": "REJECTED_POLICY", + "retry_suppressed": true, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "payment_confirm_outage", + "arm": "governed", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 5, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": "REJECTED_POLICY", + "retry_suppressed": true, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "payment_confirm_outage", + "arm": "governed", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 5, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "RECONCILIATION_REQUIRED", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "retry_transaction_outcome": "REJECTED_POLICY", + "retry_suppressed": true, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "ap_exceptions": 0, + "batches": 0, + "invoices": 1, + "payments": 1, + "purchase_orders": 0, + "receipts": 0, + "vendors": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + } + ] +} diff --git a/benchmark/ap_invoice/run.py b/benchmark/ap_invoice/run.py new file mode 100644 index 00000000..0dd5d7b0 --- /dev/null +++ b/benchmark/ap_invoice/run.py @@ -0,0 +1,449 @@ +"""AP invoice benchmark harness: email + PDF + two apps + 3-way match. + +Runs the complete AP intake workflow program end-to-end through the REAL +:class:`~openadapt_flow.runtime.replayer.Replayer` (api actuation tier), under +two arms: + +- ``naive`` -- demo profile, no governed authorization, and effect + contracts that read only the applications' OWN painted acknowledgement + banners (what a screen-echo automation trusts). +- ``governed`` -- sealed bundle + single-use standard-profile authorization, + exact API identity contracts on every consequential write, and out-of-band + effect verification routed per record surface (read-only SQL over the ERP + file, the payments REST oracle, and the OUTBOX maildir file oracle), plus + the adjacent-record collateral guard and at-most-once idempotency. + +Every run is judged by the INDEPENDENT ground truth +(:mod:`benchmark.ap_invoice.ground_truth`): direct SQLite file reads plus a +direct maildir read, with its own invariant logic and per-table delta audit. + +Zero model calls; localhost only; all data synthetic. + +Usage:: + + python -m benchmark.ap_invoice.run # write results.json + python -m benchmark.ap_invoice.run --n 3 # trials per scenario/arm + python -m benchmark.ap_invoice.run --print # print, do not write +""" + +from __future__ import annotations + +import argparse +import json +import platform +import sqlite3 +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Optional + +import requests + +from benchmark.ap_invoice import ground_truth +from benchmark.ap_invoice.fixtures import ( + ADJACENT_INVOICE, + ErpHandle, + MailerHandle, + serve_erp, + serve_mailer, +) +from benchmark.ap_invoice.workflow import ( + build_workflow, + build_worklist, + required_identity_step_ids, + scenario_faults, + scenario_invoices, + scenario_seed_duplicate, + seed_inbox, +) +from benchmark.multiapp_common import ( + NullBackend, + NullVision, + SurfaceRoutedVerifier, + standard_authorization, +) +from openadapt_flow.ir import Workflow +from openadapt_flow.runtime.actuators import ApiActuator +from openadapt_flow.runtime.effects import RestRecordVerifier +from openadapt_flow.runtime.effects.file_arrival import FileArrivalVerifier +from openadapt_flow.runtime.effects.sql import SqlRecordVerifier +from openadapt_flow.runtime.replayer import Replayer +from openadapt_flow.transaction import IdempotencyLedger + +HERE = Path(__file__).resolve().parent + +ARMS = ("naive", "governed") +SCENARIOS = ( + "healthy", + "missing_po", + "duplicate_invoice", + "collateral_approve", + "payment_confirm_outage", +) +DEFAULT_N = 3 +EFFECT_TIMEOUT_S = 0.25 + + +def _build_verifier( + arm: str, erp: ErpHandle, mailer: MailerHandle +) -> SurfaceRoutedVerifier: + if arm == "naive": + return SurfaceRoutedVerifier( + { + "banner": RestRecordVerifier( + erp.base_url, + records_path="/api/ui/banner", + records_key="records", + timeout_s=EFFECT_TIMEOUT_S, + poll_interval_s=0.02, + ), + "mail_banner": RestRecordVerifier( + mailer.base_url, + records_path="/api/ui/banner", + records_key="records", + timeout_s=EFFECT_TIMEOUT_S, + poll_interval_s=0.02, + ), + }, + default_surface="banner", + ) + + def _connect_ro() -> sqlite3.Connection: + return sqlite3.connect(f"file:{erp.db_path}?mode=ro", uri=True, timeout=5.0) + + def sql(query: str) -> SqlRecordVerifier: + return SqlRecordVerifier( + _connect_ro, + query, + timeout_s=EFFECT_TIMEOUT_S, + poll_interval_s=0.02, + ) + + return SurfaceRoutedVerifier( + { + "invoices": sql( + "SELECT id, invoice_id, vendor_id, po_number, amount, " + "doc_sha256, status, discount_applied, amount_payable " + "FROM invoices" + ), + "exceptions": sql("SELECT id, invoice_id, reason FROM ap_exceptions"), + "batches": sql("SELECT id, batch_id, processed FROM batches"), + "payments": RestRecordVerifier( + erp.base_url, + records_path="/api/payments", + records_key="records", + timeout_s=EFFECT_TIMEOUT_S, + poll_interval_s=0.02, + ), + "outbox": FileArrivalVerifier( + mailer.outbox / "new", + pattern="*.eml", + content_probe=r"X-OpenAdapt-Invoice: ", + ), + }, + default_surface="invoices", + ) + + +def _one_replay( + workflow_src: Workflow, + *, + arm: str, + erp: ErpHandle, + mailer: MailerHandle, + worklists: dict[str, list[dict[str, str]]], + work_dir: Path, + tag: str, + ledger: Optional[IdempotencyLedger], + idempotency_key: Optional[str], +) -> Any: + """Seal, authorize (governed arm), and run once through the real replayer.""" + bundle_dir = work_dir / f"bundle-{tag}" + (bundle_dir / "templates").mkdir(parents=True, exist_ok=True) + workflow_src.save(bundle_dir) + workflow = Workflow.load(bundle_dir) + + authorization = None + if arm == "governed": + authorization = standard_authorization( + workflow, + params=None, + worklists=worklists, + required_identity_step_ids=required_identity_step_ids(workflow), + ) + replayer = Replayer( + NullBackend(), + vision=NullVision(), + effect_verifier=_build_verifier(arm, erp, mailer), + api_actuator=ApiActuator(erp.base_url, timeout_s=5.0), + governed_authorization=authorization, + durable=(arm == "governed"), + require_settled=(arm == "governed"), + idempotency_ledger=ledger, + poll_interval_s=0.01, + ) + return replayer.run( + workflow, + worklists=worklists, + bundle_dir=bundle_dir, + run_dir=work_dir / f"out-{tag}", + idempotency_key=idempotency_key, + ) + + +def run_one(scenario: str, arm: str, index: int, work_root: Path) -> dict[str, Any]: + """One end-to-end (scenario x arm) trial with fresh fixture applications.""" + work_dir = work_root / f"{scenario}-{arm}-{index}" + work_dir.mkdir(parents=True, exist_ok=True) + erp = serve_erp(work_dir / "erp.db") + mailer = serve_mailer(work_dir / "mail") + try: + requests.post( + f"{erp.base_url}/api/reset", + json={ + "faults": scenario_faults(scenario), + "seed_duplicate": scenario_seed_duplicate(scenario), + }, + timeout=5.0, + ) + seed_inbox(mailer.inbox, scenario_invoices(scenario)) + rows = build_worklist(mailer.inbox, erp.base_url) + worklists = {"invoices": rows} + workflow_src = build_workflow( + arm, + mailer_base=mailer.base_url, + adjacent_invoice=ADJACENT_INVOICE, + processed=len(rows), + ) + + before = ground_truth.capture(erp.db_path, mailer.outbox) + ledger: Optional[IdempotencyLedger] = None + idempotency_key: Optional[str] = None + if scenario == "payment_confirm_outage": + ledger = IdempotencyLedger(work_dir / "idempotency.json") + idempotency_key = f"ap-batch-{scenario}-{index}" + + report = _one_replay( + workflow_src, + arm=arm, + erp=erp, + mailer=mailer, + worklists=worklists, + work_dir=work_dir, + tag="run1", + ledger=ledger, + idempotency_key=idempotency_key, + ) + + retry_outcome: Optional[str] = None + retry_suppressed: Optional[bool] = None + if scenario == "payment_confirm_outage": + # The no-blind-retry contract: re-running under the SAME + # idempotency key must be SUPPRESSED, never re-actuated. + retry = _one_replay( + workflow_src, + arm=arm, + erp=erp, + mailer=mailer, + worklists=worklists, + work_dir=work_dir, + tag="run2", + ledger=IdempotencyLedger(work_dir / "idempotency.json"), + idempotency_key=idempotency_key, + ) + retry_outcome = retry.transaction_outcome + retry_suppressed = bool(retry.idempotent_replay) + + after = ground_truth.capture(erp.db_path, mailer.outbox) + verdict = ground_truth.judge( + scenario, + rows, + before, + after, + outbox_dir=mailer.outbox, + completed=bool(report.success or report.execution_completed), + ) + + reported_success = bool(report.success) + halted = not reported_success + executed = [ + r for r in report.results if not r.skipped and not r.step_id.startswith("<") + ] + return { + "scenario": scenario, + "arm": arm, + "i": index, + "worklist_rows": len(rows), + "executed_action_steps": len(executed), + "actuation_kinds": sorted({str(r.actuation) for r in executed}), + "reported_success": reported_success, + "halted": halted, + "execution_outcome": report.execution_outcome, + "transaction_outcome": report.transaction_outcome, + "transaction_billable": report.transaction_billable, + "terminal_outcome": report.terminal_outcome, + "model_calls": report.model_calls, + "retry_transaction_outcome": retry_outcome, + "retry_suppressed": retry_suppressed, + "gt_violations": verdict.violations, + "gt_correct": verdict.correct, + "table_deltas": verdict.table_deltas, + "silent_wrong": (not verdict.correct) and reported_success, + "caught": (not verdict.correct) and halted, + "safe_halt": verdict.correct and halted, + "clean_success": verdict.correct and reported_success, + "over_halt": scenario == "healthy" and halted and verdict.correct, + } + finally: + erp.stop() + mailer.stop() + + +def aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: + per_arm: dict[str, Any] = {} + for arm in ARMS: + arm_rows = [r for r in rows if r["arm"] == arm] + per_scenario: dict[str, Any] = {} + for scenario in SCENARIOS: + srows = [r for r in arm_rows if r["scenario"] == scenario] + if not srows: + continue + per_scenario[scenario] = { + "n": len(srows), + "reported_success": sum(1 for r in srows if r["reported_success"]), + "transaction_outcomes": sorted( + {str(r["transaction_outcome"]) for r in srows} + ), + "silent_wrong": sum(1 for r in srows if r["silent_wrong"]), + "caught": sum(1 for r in srows if r["caught"]), + "safe_halt": sum(1 for r in srows if r["safe_halt"]), + } + per_arm[arm] = { + "n_runs": len(arm_rows), + "verified": sum( + 1 for r in arm_rows if r["transaction_outcome"] == "VERIFIED" + ), + "completed_unverified": sum( + 1 + for r in arm_rows + if r["transaction_outcome"] == "COMPLETED_UNVERIFIED" + ), + "halts": sum(1 for r in arm_rows if r["halted"]), + "safe_halts": sum(1 for r in arm_rows if r["safe_halt"]), + "caught": sum(1 for r in arm_rows if r["caught"]), + "reconciliation_required": sum( + 1 + for r in arm_rows + if r["transaction_outcome"] == "RECONCILIATION_REQUIRED" + ), + "suppressed_retries": sum(1 for r in arm_rows if r.get("retry_suppressed")), + "silent_wrong": sum(1 for r in arm_rows if r["silent_wrong"]), + "over_halts": sum(1 for r in arm_rows if r["over_halt"]), + "model_calls": sum(int(r["model_calls"] or 0) for r in arm_rows), + "per_scenario": per_scenario, + } + return {"per_arm": per_arm} + + +def run_benchmark( + n: int = DEFAULT_N, *, log: Callable[[str], None] = print +) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + with tempfile.TemporaryDirectory(prefix="ap-invoice-") as tmp: + work_root = Path(tmp) + for arm in ARMS: + for scenario in SCENARIOS: + for i in range(n): + rows.append(run_one(scenario, arm, i, work_root)) + last = rows[-1] + log( + f"{arm:9s} {scenario:24s} " + f"outcome={last['transaction_outcome']:24s} " + f"gt_ok={last['gt_correct']!s:5s} " + f"silent_wrong={last['silent_wrong']}" + ) + metrics = aggregate(rows) + governed = metrics["per_arm"]["governed"] + naive = metrics["per_arm"]["naive"] + healthy = [r for r in rows if r["scenario"] == "healthy" and r["arm"] == "governed"] + return { + "benchmark": "ap_invoice", + "instrument": ( + "multi-application AP invoice intake: email + PDF + ERP 3-way " + "match + payments + exception queue, end-to-end through the real " + "replayer's api tier" + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "platform": platform.platform(), + "n_per_scenario": n, + "arms": list(ARMS), + "scenarios": list(SCENARIOS), + "workflow_shape": { + "applications": ["erp", "mail_gateway"], + "modalities": [ + "maildir email in/out", + "pdf document", + "rest api", + "ui gateway (no entry api)", + "sqlite system of record", + ], + "healthy_worklist_rows": healthy[0]["worklist_rows"] if healthy else None, + "healthy_executed_action_steps": ( + healthy[0]["executed_action_steps"] if healthy else None + ), + "branches": [ + "match route (ok vs mismatch)", + "discount (eligible vs expired)", + ], + "exception_paths": [ + "missing purchase order (halt at entry)", + "ambiguous duplicate invoice (halt at entry)", + "collateral adjacent-record overwrite (governed: caught)", + "payment confirmation outage (governed: RECONCILIATION_REQUIRED " + "+ suppressed retry)", + ], + }, + "headline": { + "governed_verified": governed["verified"], + "governed_silent_wrong": governed["silent_wrong"], + "governed_over_halts": governed["over_halts"], + "governed_reconciliation_required": governed["reconciliation_required"], + "governed_suppressed_retries": governed["suppressed_retries"], + "naive_silent_wrong": naive["silent_wrong"], + "model_calls_total": governed["model_calls"] + naive["model_calls"], + }, + "metrics": metrics, + "runs": rows, + } + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--n", type=int, default=DEFAULT_N) + parser.add_argument("--out", default=str(HERE)) + parser.add_argument("--print", action="store_true", dest="print_only") + args = parser.parse_args(argv) + results = run_benchmark(n=args.n) + if args.print_only: + json.dump(results, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") + h = results["headline"] + print( + f"\nap_invoice governed: verified={h['governed_verified']} " + f"silent_wrong={h['governed_silent_wrong']} " + f"over_halts={h['governed_over_halts']} " + f"reconciliation={h['governed_reconciliation_required']} " + f"naive: silent_wrong={h['naive_silent_wrong']} " + f"model_calls={h['model_calls_total']}" + ) + print(f"Wrote results.json under {out_dir}") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/benchmark/ap_invoice/workflow.py b/benchmark/ap_invoice/workflow.py new file mode 100644 index 00000000..571eb905 --- /dev/null +++ b/benchmark/ap_invoice/workflow.py @@ -0,0 +1,738 @@ +"""Email intake + workflow-program construction for the AP invoice benchmark. + +The intake stage is the deterministic fixture-side document pipeline a real +AP deployment would wire up front: read the vendor request emails from the +INBOX maildir, extract each PDF invoice attachment, parse its fields, and +derive one worklist row per invoice (including the discount-eligibility and +expected 3-way-match route the workflow branches on). ZERO model calls: the +PDF is a deterministic fixture document parsed by a deterministic parser; this +benchmark does not measure OCR or model-based document extraction. + +The workflow itself is a Phase-2 workflow-program graph: a LOOP over the +invoice worklist whose body BRANCHES on the match route (post vs. hold) and on +discount eligibility, with every consequential write carried by an +``ApiBinding`` (the api tier), an exact API identity contract, and typed +system-of-record effect contracts routed to per-surface out-of-band oracles. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from typing import Optional + +import requests + +from benchmark.multiapp_common import ( + build_request_email, + extract_pdf_lines, + make_pdf, + parse_email, + parse_kv_lines, + read_maildir_messages, + write_maildir_message, +) +from openadapt_flow.ir import ( + ActionKind, + ApiBinding, + ApiIdentityBinding, + LoopSpec, + Predicate, + PredicateKind, + ProgramGraph, + State, + StateKind, + Step, + Transition, + Workflow, +) +from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr + +EFFECT_TIMEOUT_S = 0.25 +BATCH_ID = "BATCH-2026-07" + +#: invoice_id -> (vendor_id, vendor_name, po_number, amount, terms) +#: ``2/10 NET 30`` terms are discount-eligible at batch time; ``NET 30`` is +#: expired. INV-1003's amount deliberately disagrees with PO-503 (the price +#: mismatch routed to the exception queue). INV-1101 references a PO that does +#: not exist; INV-1201 re-presents an already-posted invoice number. +INVOICE_SEEDS: dict[str, tuple[str, str, str, str, str]] = { + "INV-1001": ("V-100", "Acme Supply Co", "PO-501", "1200.00", "2/10 NET 30"), + "INV-1002": ("V-200", "Beta Industrial", "PO-502", "860.00", "NET 30"), + "INV-1003": ("V-300", "Gamma Logistics", "PO-503", "415.25", "NET 30"), + "INV-1004": ("V-100", "Acme Supply Co", "PO-504", "99.90", "2/10 NET 30"), + "INV-1005": ("V-200", "Beta Industrial", "PO-505", "729.00", "NET 30"), + "INV-1101": ("V-100", "Acme Supply Co", "PO-599", "55.00", "NET 30"), + "INV-1201": ("V-200", "Beta Industrial", "PO-505", "729.00", "NET 30"), + "INV-1301": ("V-100", "Acme Supply Co", "PO-506", "320.00", "NET 30"), + "INV-1401": ("V-300", "Gamma Logistics", "PO-507", "212.75", "NET 30"), +} + + +def invoice_pdf(invoice_id: str) -> bytes: + vendor_id, vendor_name, po_number, amount, terms = INVOICE_SEEDS[invoice_id] + return make_pdf( + [ + f"INVOICE: {invoice_id}", + f"VENDOR: {vendor_id}", + f"VENDOR-NAME: {vendor_name}", + f"PO: {po_number}", + f"AMOUNT: {amount}", + f"TERMS: {terms}", + "CURRENCY: USD", + ] + ) + + +def seed_inbox(inbox: Path, invoice_ids: list[str]) -> None: + """Deliver one deterministic vendor request email per invoice.""" + for invoice_id in invoice_ids: + vendor_id, vendor_name, po_number, amount, _terms = INVOICE_SEEDS[invoice_id] + pdf = invoice_pdf(invoice_id) + message = build_request_email( + from_addr=f"billing@{vendor_id.lower()}.example.test", + to_addr="ap@example-corp.test", + subject=f"Invoice {invoice_id} for {po_number}", + body=( + f"Please process the attached invoice {invoice_id} " + f"({vendor_name}, {amount} USD) against {po_number}." + ), + message_id=f"<{invoice_id.lower()}@{vendor_id.lower()}.example.test>", + pdf_name=f"{invoice_id}.pdf", + pdf_bytes=pdf, + ) + write_maildir_message(inbox, f"request-{invoice_id}.eml", message) + + +def build_worklist(inbox: Path, erp_base: str) -> list[dict[str, str]]: + """Parse the inbox emails + PDF attachments into workflow worklist rows. + + The route/discount fields drive the workflow-program branches; the + ``match_status`` field is what the 3-way match effect contract VERIFIES the + ERP independently concluded (a disagreement halts, never mis-routes). A + row whose PO is unknown at intake is still routed as ``ok`` on purpose: + real intake queues carry wrong references, and the measured behavior must + be a governed halt at entry, not a silent skip. + """ + po_rows = requests.get(f"{erp_base}/api/purchase_orders", timeout=5.0).json()[ + "records" + ] + pos = {row["po_number"]: row for row in po_rows} + rows: list[dict[str, str]] = [] + for name, raw in sorted(read_maildir_messages(inbox).items()): + parsed, attachments = parse_email(raw) + assert attachments, f"fixture email {name} carries no PDF" + pdf_bytes = attachments[0][1] + fields = parse_kv_lines(extract_pdf_lines(pdf_bytes)) + invoice_id = fields["invoice"] + po_number = fields["po"] + amount = fields["amount"] + po = pos.get(po_number) + matched = po is not None and str(po["amount"]) == amount + route = "ok" if (po is None or matched) else "mismatch" + eligible = fields["terms"].startswith("2/10") and route == "ok" + amount_payable = f"{round(float(amount) * 0.98, 2):.2f}" if eligible else amount + kind = "confirm" if route == "ok" else "hold" + rows.append( + { + "invoice_id": invoice_id, + "vendor_id": fields["vendor"], + "po_number": po_number, + "amount": amount, + "doc_sha256": hashlib.sha256(pdf_bytes).hexdigest(), + "route": route, + "match_status": "matched" if route == "ok" else "price_mismatch", + "discount": "eligible" if eligible else "expired", + "discount_applied": "2/10" if eligible else "none", + "amount_payable": amount_payable, + "reply_to": str(parsed["From"]), + "mail_name": f"{kind}-{invoice_id}.eml", + "mail_subject": f"{kind.capitalize()}: {invoice_id}", + } + ) + return rows + + +# -- step construction -------------------------------------------------------- + + +def _identity( + param: str, *, key: str = "record_id", effect_field: Optional[str] = None +) -> ApiIdentityBinding: + field = effect_field or param + return ApiIdentityBinding( + key=key, # type: ignore[arg-type] + param=param, + effect_field=field, + request_pointers=[f"/body/{field}"], + ) + + +def _banner_effect( + id_param: str, event: str, *, surface: str = "banner" +) -> list[Effect]: + """The NAIVE arm's contract: the app's own painted acknowledgement row.""" + return [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "invoice_id": ValueExpr(param=id_param), + "event": ValueExpr(literal=event), + }, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe=f"surface={surface}|the app's own acknowledgement banner", + timeout_s=EFFECT_TIMEOUT_S, + ) + ] + + +def _field_effect( + surface: str, + match: dict[str, ValueExpr], + field: str, + value: ValueExpr, + probe: str, +) -> Effect: + return Effect( + kind=EffectKind.FIELD_EQUALS, + match=match, + field=field, + value=value, + risk="irreversible", + probe=f"surface={surface}|{probe}", + timeout_s=EFFECT_TIMEOUT_S, + ) + + +def _inv_match() -> dict[str, ValueExpr]: + return {"invoice_id": ValueExpr(param="invoice_id")} + + +def _steps(arm: str, mailer_base: str, adjacent_invoice: str) -> dict[str, Step]: + """The ten hardened action leaves, keyed by step id. + + ``arm`` selects the effect contracts: ``governed`` carries out-of-band + system-of-record contracts (SQL / REST / maildir surfaces, plus the + adjacent-record collateral guard); ``naive`` carries only the app-painted + banner acknowledgement (what a screen-echo automation trusts). + """ + governed = arm == "governed" + + def effects(step_key: str) -> list[Effect]: + if not governed: + naive = { + "draft": _banner_effect("invoice_id", "invoice_created"), + "attach_doc": _banner_effect("invoice_id", "document_attached"), + "match": _banner_effect("invoice_id", "match_run"), + "apply_discount": _banner_effect("invoice_id", "discount_applied"), + "approve": _banner_effect("invoice_id", "invoice_approved"), + "schedule_payment": _banner_effect("invoice_id", "payment_scheduled"), + "send_confirmation": _banner_effect( + "invoice_id", "mail_sent", surface="mail_banner" + ), + "route_exception": _banner_effect("invoice_id", "exception_routed"), + "send_hold_notice": _banner_effect( + "invoice_id", "mail_sent", surface="mail_banner" + ), + "batch_complete": _banner_effect("batch_id", "batch_completed"), + } + return naive[step_key] + inv = _inv_match() + # Email delivery to the capture point: exactly one NEW conforming + # message for THIS invoice arrived in the outbox maildir (read from + # disk; a duplicate send or a dropped send is caught). + mail_arrival = Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "name": ValueExpr(param="mail_name"), + "arrived": ValueExpr(literal="True"), + }, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=outbox|one new conforming message in the outbox maildir", + timeout_s=EFFECT_TIMEOUT_S, + ) + table = { + "draft": [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "invoice_id": ValueExpr(param="invoice_id"), + "vendor_id": ValueExpr(param="vendor_id"), + "po_number": ValueExpr(param="po_number"), + }, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=invoices|exactly one new draft for this " + "invoice/vendor/PO", + timeout_s=EFFECT_TIMEOUT_S, + ), + _field_effect( + "invoices", + inv, + "amount", + ValueExpr(param="amount"), + "the persisted invoice amount read back", + ), + ], + "attach_doc": [ + _field_effect( + "invoices", + inv, + "doc_sha256", + ValueExpr(param="doc_sha256"), + "the stored document digest equals the received PDF's", + ) + ], + "match": [ + _field_effect( + "invoices", + inv, + "status", + ValueExpr(param="match_status"), + "the ERP's own 3-way match verdict equals the routed one", + ) + ], + "apply_discount": [ + _field_effect( + "invoices", + inv, + "discount_applied", + ValueExpr(param="discount_applied"), + "the discount terms persisted", + ), + _field_effect( + "invoices", + inv, + "amount_payable", + ValueExpr(param="amount_payable"), + "the discounted payable amount persisted", + ), + ], + "approve": [ + _field_effect( + "invoices", + inv, + "status", + ValueExpr(literal="approved"), + "the target invoice is approved", + ), + _field_effect( + "invoices", + inv, + "amount_payable", + ValueExpr(param="amount_payable"), + "the approved payable amount read back", + ), + # Collateral guard: the ADJACENT grid row must be untouched. + _field_effect( + "invoices", + {"invoice_id": ValueExpr(literal=adjacent_invoice)}, + "status", + ValueExpr(literal="draft"), + "the adjacent invoice was NOT collaterally approved", + ), + ], + "schedule_payment": [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match=inv, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=payments|exactly one new scheduled payment", + timeout_s=EFFECT_TIMEOUT_S, + ), + _field_effect( + "payments", + inv, + "amount", + ValueExpr(param="amount_payable"), + "the scheduled payment amount read back", + ), + ], + "send_confirmation": [mail_arrival], + "route_exception": [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match=inv, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=exceptions|exactly one new AP exception entry", + timeout_s=EFFECT_TIMEOUT_S, + ), + _field_effect( + "invoices", + inv, + "status", + ValueExpr(literal="held"), + "the mismatched invoice is held, not payable", + ), + ], + "send_hold_notice": [mail_arrival.model_copy(deep=True)], + "batch_complete": [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={"batch_id": ValueExpr(param="batch_id")}, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=batches|the batch completion record", + timeout_s=EFFECT_TIMEOUT_S, + ) + ], + } + return table[step_key] + + def step( + step_id: str, + intent: str, + binding: ApiBinding, + ) -> Step: + return Step( + id=step_id, + intent=intent, + action=ActionKind.KEY, + key="Enter", + risk="irreversible", + effects=effects(step_id), + api_binding=binding, + ) + + # The email step's identity contract binds the per-invoice message name: + # the request's /body/name and the arrival effect's `name` selector must + # both carry the same run parameter before the send is allowed. + mail_identity = [_identity("mail_name", effect_field="name")] if governed else [] + inv_identity = [_identity("invoice_id")] if governed else [] + return { + "draft": step( + "draft", + "enter the invoice into the ERP (UI gateway; no entry API exists)", + ApiBinding( + method="POST", + url_template="/ui/invoice/new", + body_template={ + "invoice_id": "{invoice_id}", + "vendor_id": "{vendor_id}", + "po_number": "{po_number}", + "amount": "{amount}", + }, + timeout_s=5.0, + identity=( + [ + _identity("invoice_id"), + _identity("vendor_id", key="secondary_identifier"), + ] + if governed + else [] + ), + ), + ), + "attach_doc": step( + "attach_doc", + "attach the received PDF's digest to the invoice (API)", + ApiBinding( + method="POST", + url_template="/api/invoice/document", + body_template={ + "invoice_id": "{invoice_id}", + "doc_sha256": "{doc_sha256}", + }, + timeout_s=5.0, + identity=inv_identity, + ), + ), + "match": step( + "match", + "run the 3-way match (invoice vs PO vs receipts) (API)", + ApiBinding( + method="POST", + url_template="/api/invoice/match", + body_template={"invoice_id": "{invoice_id}"}, + timeout_s=5.0, + identity=inv_identity, + ), + ), + "apply_discount": step( + "apply_discount", + "apply the early-payment discount (API; eligible branch only)", + ApiBinding( + method="POST", + url_template="/api/invoice/discount", + body_template={ + "invoice_id": "{invoice_id}", + "discount_applied": "{discount_applied}", + "amount_payable": "{amount_payable}", + }, + timeout_s=5.0, + identity=inv_identity, + ), + ), + "approve": step( + "approve", + "approve the invoice for payment (UI gateway)", + ApiBinding( + method="POST", + url_template="/ui/invoice/approve", + body_template={ + "invoice_id": "{invoice_id}", + "amount_payable": "{amount_payable}", + }, + timeout_s=5.0, + identity=inv_identity, + ), + ), + "schedule_payment": step( + "schedule_payment", + "schedule the payment run entry (API)", + ApiBinding( + method="POST", + url_template="/api/payment", + body_template={ + "invoice_id": "{invoice_id}", + "amount": "{amount_payable}", + }, + timeout_s=5.0, + identity=inv_identity, + ), + ), + "send_confirmation": step( + "send_confirmation", + "email the vendor a processing confirmation (mail gateway)", + ApiBinding( + method="POST", + url_template=f"{mailer_base}/api/send", + body_template={ + "name": "{mail_name}", + "invoice_id": "{invoice_id}", + "to": "{reply_to}", + "subject": "{mail_subject}", + "body": "Invoice {invoice_id} was matched, approved, and " + "scheduled for payment of {amount_payable} USD.", + }, + timeout_s=5.0, + identity=mail_identity, + ), + ), + "route_exception": step( + "route_exception", + "route the mismatched invoice to the AP exception queue (API)", + ApiBinding( + method="POST", + url_template="/api/exception", + body_template={ + "invoice_id": "{invoice_id}", + "reason": "price mismatch vs {po_number}", + }, + timeout_s=5.0, + identity=inv_identity, + ), + ), + "send_hold_notice": step( + "send_hold_notice", + "email the vendor that the invoice is held (mail gateway)", + ApiBinding( + method="POST", + url_template=f"{mailer_base}/api/send", + body_template={ + "name": "{mail_name}", + "invoice_id": "{invoice_id}", + "to": "{reply_to}", + "subject": "{mail_subject}", + "body": "Invoice {invoice_id} does not match {po_number} " + "and was routed to the AP exception queue.", + }, + timeout_s=5.0, + identity=mail_identity, + ), + ), + "batch_complete": step( + "batch_complete", + "record the batch completion (API)", + ApiBinding( + method="POST", + url_template="/api/batch/complete", + body_template={"batch_id": "{batch_id}", "processed": "{processed}"}, + timeout_s=5.0, + identity=[_identity("batch_id")] if governed else [], + ), + ), + } + + +def _guard(param: str, value: str) -> Predicate: + return Predicate(kind=PredicateKind.PARAM_EQUALS, param=param, value=value) + + +def build_workflow( + arm: str, + *, + mailer_base: str, + adjacent_invoice: str, + processed: int, +) -> Workflow: + """The complete AP intake workflow program for one arm.""" + steps = _steps(arm, mailer_base, adjacent_invoice) + + body = ProgramGraph( + entry="enter_invoice", + states={ + "enter_invoice": State( + id="enter_invoice", + kind=StateKind.ACTION, + step=steps["draft"], + transitions=[Transition(target="attach_document")], + ), + "attach_document": State( + id="attach_document", + kind=StateKind.ACTION, + step=steps["attach_doc"], + transitions=[Transition(target="run_match")], + ), + "run_match": State( + id="run_match", + kind=StateKind.ACTION, + step=steps["match"], + transitions=[Transition(target="route_on_match")], + ), + "route_on_match": State( + id="route_on_match", + kind=StateKind.BRANCH, + transitions=[ + Transition( + guard=_guard("route", "ok"), + target="discount_gate", + label="3-way match clean: post it", + ), + Transition( + guard=_guard("route", "mismatch"), + target="hold_invoice", + label="price mismatch: exception queue", + ), + ], + ), + "discount_gate": State( + id="discount_gate", + kind=StateKind.BRANCH, + transitions=[ + Transition( + guard=_guard("discount", "eligible"), + target="take_discount", + label="early-payment terms still open", + ), + Transition( + guard=_guard("discount", "expired"), + target="approve_invoice", + label="discount window expired: pay net", + ), + ], + ), + "take_discount": State( + id="take_discount", + kind=StateKind.ACTION, + step=steps["apply_discount"], + transitions=[Transition(target="approve_invoice")], + ), + "approve_invoice": State( + id="approve_invoice", + kind=StateKind.ACTION, + step=steps["approve"], + transitions=[Transition(target="pay_invoice")], + ), + "pay_invoice": State( + id="pay_invoice", + kind=StateKind.ACTION, + step=steps["schedule_payment"], + transitions=[Transition(target="confirm_by_email")], + ), + "confirm_by_email": State( + id="confirm_by_email", + kind=StateKind.ACTION, + step=steps["send_confirmation"], + transitions=[Transition(target="row_done")], + ), + "hold_invoice": State( + id="hold_invoice", + kind=StateKind.ACTION, + step=steps["route_exception"], + transitions=[Transition(target="notify_hold")], + ), + "notify_hold": State( + id="notify_hold", + kind=StateKind.ACTION, + step=steps["send_hold_notice"], + transitions=[Transition(target="row_done")], + ), + "row_done": State( + id="row_done", kind=StateKind.TERMINAL, outcome="success" + ), + }, + ) + + top = ProgramGraph( + entry="process_invoices", + states={ + "process_invoices": State( + id="process_invoices", + kind=StateKind.LOOP, + loop=LoopSpec( + relation="invoices", + body="invoice_body", + var="invoice", + max_iterations=50, + ), + transitions=[Transition(target="finish_batch")], + ), + "finish_batch": State( + id="finish_batch", + kind=StateKind.ACTION, + step=steps["batch_complete"], + transitions=[Transition(target="done")], + ), + "done": State(id="done", kind=StateKind.TERMINAL, outcome="success"), + }, + ) + + return Workflow( + name=f"ap-invoice-{arm}", + steps=[], + program=top, + subflows={"invoice_body": body}, + params={"batch_id": BATCH_ID, "processed": str(processed)}, + ) + + +def required_identity_step_ids(workflow: Workflow) -> tuple[str, ...]: + from openadapt_flow.traversal import iter_workflow_steps + + return tuple(step.id for step in iter_workflow_steps(workflow)) + + +def scenario_invoices(scenario: str) -> list[str]: + """Which seeded request emails a scenario's inbox carries.""" + return { + "healthy": ["INV-1001", "INV-1002", "INV-1003", "INV-1004", "INV-1005"], + "missing_po": ["INV-1101"], + "duplicate_invoice": ["INV-1201"], + "collateral_approve": ["INV-1301"], + "payment_confirm_outage": ["INV-1401"], + }[scenario] + + +def scenario_faults(scenario: str) -> list[str]: + return { + "healthy": [], + "missing_po": [], + "duplicate_invoice": [], + "collateral_approve": ["collateral_adjacent_on_approve"], + "payment_confirm_outage": ["payments_read_down_after_write"], + }[scenario] + + +def scenario_seed_duplicate(scenario: str) -> bool: + return scenario == "duplicate_invoice" diff --git a/benchmark/multiapp_common.py b/benchmark/multiapp_common.py new file mode 100644 index 00000000..a38a8818 --- /dev/null +++ b/benchmark/multiapp_common.py @@ -0,0 +1,433 @@ +"""Shared harness pieces for the multi-system workflow benchmarks. + +Used by ``benchmark/ap_invoice`` and ``benchmark/o2c_recon``. Everything here +is deterministic, synthetic, and localhost-only: null backend/vision fakes for +the API-tier replay path (the same pattern ``benchmark/effect_e2e`` uses), a +surface-routing composite effect verifier, a CSV row oracle for spreadsheet +write-back verification, deterministic RFC822 email + minimal PDF fixtures, +and the governed standard-profile authorization helper. + +No model calls, no network beyond 127.0.0.1, no PHI (all names are synthetic +company/worker placeholders). +""" + +from __future__ import annotations + +import csv +import struct +import time +import zlib +from email.message import EmailMessage +from email.parser import BytesParser +from email.utils import formatdate +from pathlib import Path +from types import SimpleNamespace +from typing import Any, Optional + +from openadapt_flow.ir import Workflow +from openadapt_flow.runtime.authorization import ( + GovernedRunAuthorization, + runtime_inputs_digest, +) +from openadapt_flow.runtime.effects._common import judge_records +from openadapt_flow.runtime.effects.effect import ( + Effect, + EffectState, + EffectVerdict, +) +from openadapt_flow.verification import VerificationTier, verifier_effect_tier + +# -- replayer fakes ---------------------------------------------------------- +# The api actuation tier returns before any GUI resolve/act, so these only +# need to satisfy the replayer's construction and the (unreached) settle path. + + +def tiny_png() -> bytes: + """A valid 1x1 white PNG (the api-tier path never renders it).""" + raw = b"\x00\xff\xff\xff" + idat = zlib.compress(raw) + + def chunk(tag: bytes, data: bytes) -> bytes: + return ( + struct.pack(">I", len(data)) + + tag + + data + + struct.pack(">I", zlib.crc32(tag + data) & 0xFFFFFFFF) + ) + + ihdr = struct.pack(">IIBBBBB", 1, 1, 8, 2, 0, 0, 0) + return ( + b"\x89PNG\r\n\x1a\n" + + chunk(b"IHDR", ihdr) + + chunk(b"IDAT", idat) + + chunk(b"IEND", b"") + ) + + +_PNG = tiny_png() +_VIEWPORT = (2, 2) + + +class NullVision: + """Vision facade stub: stable frame, every text check passes. + + Sufficient for pure API-tier workflows (no GUI resolution happens); also + reports ``settled=True`` so ``require_settled=True`` never trips on the + (unused) GUI path. + """ + + def wait_settled(self, backend: Any, **_kw: Any) -> bytes: + return backend.screenshot() + + def wait_settled_result(self, backend: Any, **_kw: Any) -> Any: + return SimpleNamespace(png=backend.screenshot(), settled=True) + + def text_present(self, *_a: Any, **_kw: Any) -> bool: + return True + + def find_text(self, *_a: Any, **_kw: Any) -> None: + return None + + def find_template(self, *_a: Any, **_kw: Any) -> None: + return None + + def find_structural_template(self, *_a: Any, **_kw: Any) -> None: + return None + + def ocr(self, *_a: Any, **_kw: Any) -> list[Any]: + return [] + + def pixels_changed(self, *_a: Any, **_kw: Any) -> bool: + return True + + def phash_png(self, *_a: Any, **_kw: Any) -> str: + return "aa" + + def phash_distance(self, *_a: Any, **_kw: Any) -> int: + return 0 + + +class NullBackend: + """Backend stub for API-tier-only replays (no GUI action is delivered).""" + + @property + def viewport(self) -> tuple[int, int]: + return _VIEWPORT + + def screenshot(self) -> bytes: + return _PNG + + def click(self, *_a: Any, **_kw: Any) -> None: + pass + + def type_text(self, *_a: Any, **_kw: Any) -> None: + pass + + def press(self, *_a: Any, **_kw: Any) -> None: + pass + + def scroll(self, *_a: Any, **_kw: Any) -> None: + pass + + +# -- surface-routed composite effect verifier -------------------------------- + + +def surface_of(effect: Effect, default: str) -> str: + """The record surface an effect targets, from its ``probe`` marker.""" + probe = effect.probe or "" + if probe.startswith("surface="): + return probe[len("surface=") :].split("|", 1)[0].strip() + return default + + +class SurfaceRoutedVerifier: + """Route each typed effect to the out-of-band verifier for its surface. + + Generalizes ``benchmark.effect_e2e.verifiers.CompositeSqlVerifier`` to a + heterogeneous verifier map (read-only SQL, REST record oracle, file/maildir + arrival, CSV row oracle), and advertises the routed sub-verifier's + evidence tier via ``verification_tier_for`` so a governed standard-profile + run can classify the confirmed evidence. + """ + + substrate = "composite-multi" + + def __init__(self, verifiers: dict[str, Any], *, default_surface: str) -> None: + self._verifiers = verifiers + self._default = default_surface + + def verification_tier_for(self, effect: Any) -> Optional[VerificationTier]: + if effect is None: + return VerificationTier.INDEPENDENT_SYSTEM + verifier = self._verifiers.get(surface_of(effect, self._default)) + if verifier is None: + return None + return verifier_effect_tier(verifier, effect) + + def capture_pre_state(self, context: Any = None) -> EffectState: + detail: dict[str, Any] = {} + reachable = True + for name, verifier in self._verifiers.items(): + state = verifier.capture_pre_state(context) + detail[name] = {"reachable": state.reachable, "records": state.records} + reachable = reachable and state.reachable + return EffectState( + substrate=self.substrate, reachable=reachable, records=[], detail=detail + ) + + def verify( + self, expected: Effect, before: EffectState, context: Any = None + ) -> EffectVerdict: + name = surface_of(expected, self._default) + verifier = self._verifiers[name] + sub = before.detail.get(name, {}) + sub_before = EffectState( + substrate=verifier.substrate, + reachable=bool(sub.get("reachable", False)), + records=list(sub.get("records", [])), + ) + return verifier.verify(expected, sub_before, context) + + +class CsvRecordVerifier: + """Out-of-band spreadsheet oracle: RE-READ a CSV file as the record surface. + + Each data row flattens to a dict keyed by the header row, so the standard + ``Effect(kind=RECORD_WRITTEN, match={...}, count_new_only=True)`` contract + reads "exactly one NEW row for this key was appended by this action". A + missing file counts as an EMPTY readable surface only while the parent + directory exists; a missing directory or an unparsable file is an + unreachable system of record (INDETERMINATE, fail-safe HALT). + """ + + substrate = "csv" + verification_tier = VerificationTier.INDEPENDENT_SYSTEM + + def __init__( + self, + path: Path | str, + *, + timeout_s: float = 0.3, + poll_interval_s: float = 0.02, + ) -> None: + self.path = Path(path) + self.timeout_s = timeout_s + self.poll_interval_s = poll_interval_s + + def _scan(self) -> Optional[list[dict[str, Any]]]: + if not self.path.parent.is_dir(): + return None + if not self.path.exists(): + return [] + try: + with self.path.open(newline="", encoding="utf-8") as handle: + rows = list(csv.DictReader(handle)) + except (OSError, csv.Error, UnicodeDecodeError): + return None + records: list[dict[str, Any]] = [] + for i, row in enumerate(rows): + record: dict[str, Any] = {k: v for k, v in row.items() if k is not None} + record["id"] = f"row-{i}-" + "|".join( + f"{k}={record[k]}" for k in sorted(record) + ) + records.append(record) + return records + + def capture_pre_state(self, context: Any = None) -> EffectState: + records = self._scan() + return EffectState( + substrate=self.substrate, + reachable=records is not None, + records=records or [], + detail={"path": str(self.path)}, + ) + + def verify( + self, expected: Effect, before: EffectState, context: Any = None + ) -> EffectVerdict: + deadline = time.monotonic() + max(0.0, expected.timeout_s) + while True: + current = self._scan() + last = judge_records(expected, before, current, substrate=self.substrate) + if last.confirmed or time.monotonic() >= deadline: + return last + time.sleep(self.poll_interval_s) + + +# -- deterministic email fixtures (maildir) ---------------------------------- + +MAILDIR_SUBDIRS = ("new", "cur", "tmp") + + +def init_maildir(root: Path) -> Path: + """Create an empty maildir layout under ``root`` and return it.""" + for sub in MAILDIR_SUBDIRS: + (root / sub).mkdir(parents=True, exist_ok=True) + return root + + +def write_maildir_message(root: Path, name: str, message: bytes) -> Path: + """Deliver ``message`` into ``root/new/`` (deterministic filename).""" + target = root / "new" / name + tmp = root / "tmp" / name + tmp.write_bytes(message) + tmp.replace(target) + return target + + +def read_maildir_messages(root: Path) -> dict[str, bytes]: + """Every message in the maildir (new + cur), keyed by filename.""" + out: dict[str, bytes] = {} + for sub in ("new", "cur"): + folder = root / sub + if not folder.is_dir(): + continue + for path in sorted(folder.iterdir()): + if path.is_file(): + out[path.name] = path.read_bytes() + return out + + +def build_request_email( + *, + from_addr: str, + to_addr: str, + subject: str, + body: str, + message_id: str, + pdf_name: Optional[str] = None, + pdf_bytes: Optional[bytes] = None, +) -> bytes: + """A deterministic RFC822 message, optionally with one PDF attachment.""" + msg = EmailMessage() + msg["From"] = from_addr + msg["To"] = to_addr + msg["Subject"] = subject + msg["Date"] = formatdate(0, usegmt=True) # fixed epoch date: deterministic + msg["Message-ID"] = message_id + msg.set_content(body) + if pdf_bytes is not None: + msg.add_attachment( + pdf_bytes, + maintype="application", + subtype="pdf", + filename=pdf_name or "attachment.pdf", + ) + return msg.as_bytes() + + +def parse_email(message: bytes) -> tuple[EmailMessage, list[tuple[str, bytes]]]: + """Parse an RFC822 message; return (message, [(attachment name, bytes)]).""" + parsed = BytesParser().parsebytes(message) + attachments: list[tuple[str, bytes]] = [] + for part in parsed.walk(): + if part.get_content_disposition() == "attachment": + payload = part.get_payload(decode=True) + if isinstance(payload, bytes): + attachments.append((part.get_filename() or "", payload)) + return parsed, attachments # type: ignore[return-value] + + +# -- deterministic minimal PDF fixtures -------------------------------------- + + +def make_pdf(lines: list[str]) -> bytes: + """A minimal valid single-page PDF whose text content is ``lines``. + + Uncompressed Helvetica text, one line per ``Tj``. Parentheses and + backslashes are stripped from input lines so the content stream never + needs escaping (fixture data contains none). + """ + safe = [line.replace("(", "").replace(")", "").replace("\\", "") for line in lines] + content = "BT /F1 11 Tf 72 720 Td\n" + for i, line in enumerate(safe): + if i: + content += "0 -14 Td\n" + content += f"({line}) Tj\n" + content += "ET\n" + stream = content.encode("latin-1") + + objects = [ + b"<< /Type /Catalog /Pages 2 0 R >>", + b"<< /Type /Pages /Kids [3 0 R] /Count 1 >>", + b"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] " + b"/Contents 4 0 R /Resources << /Font << /F1 5 0 R >> >> >>", + b"<< /Length " + + str(len(stream)).encode() + + b" >>\nstream\n" + + stream + + b"endstream", + b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>", + ] + out = bytearray(b"%PDF-1.4\n") + offsets: list[int] = [] + for i, obj in enumerate(objects, start=1): + offsets.append(len(out)) + out += f"{i} 0 obj\n".encode() + obj + b"\nendobj\n" + xref_at = len(out) + out += f"xref\n0 {len(objects) + 1}\n".encode() + out += b"0000000000 65535 f \n" + for off in offsets: + out += f"{off:010d} 00000 n \n".encode() + out += ( + b"trailer\n<< /Size " + + str(len(objects) + 1).encode() + + b" /Root 1 0 R >>\nstartxref\n" + + str(xref_at).encode() + + b"\n%%EOF\n" + ) + return bytes(out) + + +def extract_pdf_lines(data: bytes) -> list[str]: + """Extract the ``(...) Tj`` text lines from a :func:`make_pdf` document.""" + text = data.decode("latin-1", errors="replace") + start = text.find("stream\n") + end = text.find("endstream", start) + if start < 0 or end < 0: + return [] + body = text[start + len("stream\n") : end] + lines: list[str] = [] + for chunk in body.split(") Tj"): + open_at = chunk.rfind("(") + if open_at >= 0: + lines.append(chunk[open_at + 1 :]) + return lines + + +def parse_kv_lines(lines: list[str]) -> dict[str, str]: + """Parse ``KEY: value`` document lines into a dict (fixture convention).""" + out: dict[str, str] = {} + for line in lines: + if ":" in line: + key, value = line.split(":", 1) + out[key.strip().lower()] = value.strip() + return out + + +# -- governed authorization helper ------------------------------------------- + + +def standard_authorization( + workflow: Workflow, + *, + params: Optional[dict[str, str]], + worklists: Optional[dict[str, list[dict[str, str]]]], + required_identity_step_ids: tuple[str, ...], + policy_name: str = "benchmark-multiapp-standard", +) -> GovernedRunAuthorization: + """A single-use standard-profile authorization bound to this exact run. + + The bundle must be sealed (``Workflow.save`` + ``Workflow.load``) before + calling. The digest binds the exact params and worklists supplied to + ``Replayer.run``; any drift refuses execution. + """ + assert workflow.manifest is not None, "seal the bundle before authorizing" + return GovernedRunAuthorization( + bundle_content_digest=workflow.manifest.content_digest, + runtime_inputs_digest=runtime_inputs_digest(workflow, params, worklists), + admitted_policy_name=policy_name, + execution_profile="standard", + required_identity_step_ids=required_identity_step_ids, + ) diff --git a/benchmark/o2c_recon/README.md b/benchmark/o2c_recon/README.md new file mode 100644 index 00000000..ccb1eaa5 --- /dev/null +++ b/benchmark/o2c_recon/README.md @@ -0,0 +1,89 @@ +# O2C reconciliation benchmark (two systems + spreadsheet write-back) + +Synthetic, deterministic, multi-system workflow benchmark. Not a customer +environment, not publication evidence of production reliability. + +## The workflow (one governed run, healthy path) + +Order-to-cash billing reconciliation across TWO separate local applications +plus two spreadsheet surfaces on disk: + +1. **Intake** (deterministic compare pre-pass, disclosed below): read system + A's nightly EXPORT spreadsheet (`billing_export.csv`, dropped in a shared + folder by the billing fixture), read system B's ledger API, and derive one + worklist row per order with a disposition: `match`, `adjust` (with the + signed delta and the observed prior amount), or `missing`. +2. **Per order** (workflow-program LOOP body, executed by the real + `Replayer` through the api actuation tier): + - **BRANCH** three ways on disposition: + - `adjust`: enter the billing adjustment in the ledger through the **UI + gateway** (this system has no adjustment API; `POST + /api/adjustment/new` answers 405), carrying optimistic-concurrency + fields (`expected_prior`), then mark reconciled (API), + - `match`: mark reconciled (API), + - `missing`: route to an **explicit halt terminal** (a billed order with + no ledger entry is a human-review case; the workflow never + auto-creates a ledger record), + - write the order's result row back to the results spreadsheet on system A + (`recon_results.csv`), verified by RE-READING the file from disk. +3. A summary row in the results sheet, then a success terminal. + +Healthy-path shape: 10 worklist orders (5 match, 5 adjust), **26 executed +consequential actions**, 2 applications, 5 input/output modalities (CSV in, +CSV out, REST API, UI gateway, two SQLite systems of record), a 3-way branch, +plus 4 designed exception scenarios. + +## Arms + +- `naive`: demo profile; every write "verified" only against the + applications' own painted acknowledgement banners. +- `governed`: sealed bundle, single-use standard-profile authorization, exact + API identity contracts (order id plus customer name quorum on the + adjustment write), out-of-band verification per surface (read-only SQL + over the ledger SQLite file; the results CSV re-read from disk). + +## Scenarios and measured outcomes (n=3 per cell, deterministic) + +| scenario | naive (banner oracle) | governed | +|---|---|---| +| `healthy` | completes; `COMPLETED_UNVERIFIED` (never billable) | **`VERIFIED`** (billable) | +| `missing_in_ledger` | processes the prior order, then halts at the explicit terminal | same; `HALTED_BEFORE_EFFECT`; no ledger entry auto-created | +| `ambiguous_duplicate` | safe halt (UI gateway refuses the 2-entry order) | safe halt; `HALTED_BEFORE_EFFECT`; both entries untouched | +| `stale_snapshot` | safe halt (optimistic-concurrency 409) | safe halt; `HALTED_BEFORE_EFFECT`; amount unchanged | +| `phantom_writeback` | **SILENT WRONG** (row acknowledged, never written to the sheet) | **caught** by re-reading the file; halts | + +Headline (30 runs): governed silent-incorrect-success **0**, over-halts on +the healthy path **0**, model calls **0**; naive silent-incorrect-success +**3/3** on the phantom write-back. Ground truth opens both SQLite files and +both CSV files directly and re-derives the intended state from the seed +table; it also verifies the exported spreadsheet was never mutated. + +The compare pre-pass is deliberately NAIVE (first ledger entry wins, snapshot +trusted): the measured property is that the ENGINE still refuses at act time +when that worklist is wrong (duplicate rows, stale snapshot, missing record). + +Reproduce: `python -m benchmark.o2c_recon.run --n 3` (localhost only, $0). +Pinned in CI by `tests/test_o2c_recon_benchmark.py`. + +## What this proves, and what it does not + +Proves (within a synthetic closed world): + +- a genuinely multi-application flow: one workflow program driving two + separate fixture applications and two file surfaces, with per-surface + out-of-band verification including a spreadsheet re-read oracle; +- conflict and duplicate handling refuse BEFORE any write (optimistic + concurrency, ambiguity, missing record), and the phantom-file-write class + is caught only by an oracle that actually re-reads the file; +- the same honest metrics as the other benchmarks: runs, verified, halts, + silent-incorrect-success (0 governed), over-halts (0). + +Does NOT prove: + +- anything about GUI perception (every action is actuated through the + replayer's api tier; the "UI gateway" models a screen with no API but is + driven over HTTP). See the MockMed/OpenEMR/canvas/RDP benchmarks for the + visual ladder. +- production incidence rates (hand-authored deterministic fault taxonomy; + cooperative local fixtures; synthetic data only). +- real ERP/accounting-system behavior or any customer environment. diff --git a/benchmark/o2c_recon/fixtures.py b/benchmark/o2c_recon/fixtures.py new file mode 100644 index 00000000..8b744adb --- /dev/null +++ b/benchmark/o2c_recon/fixtures.py @@ -0,0 +1,485 @@ +"""Deterministic two-system fixture for the O2C reconciliation benchmark. + +Two separate local applications with separate on-disk SQLite systems of +record, plus two spreadsheet surfaces on disk: + +1. **Billing service** (system A, the revenue side): owns ``billed_orders`` + and, at reset, drops the nightly EXPORT spreadsheet + (``export/billing_export.csv``) into a shared folder -- the worklist input. + It also hosts the workbook gateway: ``POST /api/workbook/writeback`` + appends a result row to ``workbook/recon_results.csv`` (the written-back + results sheet). The ``drop_writeback`` fault acknowledges the row (banner + + HTTP 200) without persisting it -- the phantom-file-write class. + +2. **Ledger service** (system B, the accounting side): owns + ``ledger_entries`` and ``adjustments``. Adjustments are accepted ONLY + through the UI gateway (``POST /ui/adjustment/new``; the API answers 405) + and carry optimistic concurrency: a stale ``expected_prior`` is refused + with 409 BEFORE anything is written. Reconciliation marks go through the + REST API. The ``stale_snapshot`` fault makes the read API report a lagged + amount for one order while the database has already moved on. + +All data is synthetic. Localhost only. Zero model calls. +""" + +from __future__ import annotations + +import csv +import json +import sqlite3 +import threading +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any, Callable, Optional +from urllib.parse import urlparse + +#: (order_id, customer, amount_billed, amount_posted, period) +#: The first five agree across systems; the next five differ (adjustments); +#: the 9xxx orders exist for the exception scenarios. +ORDER_SEEDS: tuple[tuple[str, str, str, Optional[str], str], ...] = ( + ("ORD-9001", "Northwind Retail", "250.00", "250.00", "2026-06"), + ("ORD-9002", "Cobalt Foods", "120.40", "120.40", "2026-06"), + ("ORD-9003", "Juniper Health Supply", "78.25", "78.25", "2026-06"), + ("ORD-9004", "Atlas Manufacturing", "1310.00", "1310.00", "2026-06"), + ("ORD-9005", "Harbor Freight Lines", "99.00", "99.00", "2026-06"), + ("ORD-9006", "Northwind Retail", "400.00", "380.00", "2026-06"), + ("ORD-9007", "Cobalt Foods", "155.10", "165.10", "2026-06"), + ("ORD-9008", "Atlas Manufacturing", "720.00", "700.00", "2026-06"), + ("ORD-9009", "Juniper Health Supply", "88.88", "80.88", "2026-06"), + ("ORD-9010", "Harbor Freight Lines", "505.50", "500.00", "2026-06"), + # billing-only: no ledger entry exists (missing-record exception). + ("ORD-9101", "Meridian Parts", "64.00", None, "2026-06"), + # duplicated in the ledger (ambiguous-duplicate exception). + ("ORD-9201", "Cobalt Foods", "310.00", "300.00", "2026-06"), + # stale-snapshot target: DB holds 500.00, the lagged read reports 480.00. + ("ORD-9301", "Atlas Manufacturing", "520.00", "500.00", "2026-06"), + # phantom-writeback target: amounts agree; only the results sheet is at risk. + ("ORD-9401", "Northwind Retail", "212.00", "212.00", "2026-06"), +) + +STALE_ORDER = "ORD-9301" +STALE_REPORTED_AMOUNT = "480.00" +DUPLICATE_ORDER = "ORD-9201" + +EXPORT_NAME = "billing_export.csv" +RESULTS_NAME = "recon_results.csv" +RESULTS_FIELDS = ("order_id", "disposition", "delta", "status") + + +@dataclass(frozen=True) +class BillingHandle: + base_url: str + db_path: Path + export_path: Path + results_path: Path + stop: Callable[[], None] + + +@dataclass(frozen=True) +class LedgerHandle: + base_url: str + db_path: Path + stop: Callable[[], None] + + +def _json_handler( + route: Callable[[str, str, dict[str, Any]], tuple[int, dict[str, Any]]], +) -> type[BaseHTTPRequestHandler]: + class _Handler(BaseHTTPRequestHandler): + def log_message(self, fmt: str, *args: object) -> None: # noqa: A002 + pass + + def _send(self, status: int, payload: dict[str, Any]) -> None: + body = json.dumps(payload).encode("utf-8") + try: + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + except (BrokenPipeError, ConnectionResetError): + pass + + def _body(self) -> dict[str, Any]: + length = int(self.headers.get("Content-Length", 0) or 0) + if not length: + return {} + try: + loaded = json.loads(self.rfile.read(length).decode("utf-8")) + return loaded if isinstance(loaded, dict) else {} + except (ValueError, UnicodeDecodeError): + return {} + + def do_GET(self) -> None: # noqa: N802 + status, payload = route("GET", urlparse(self.path).path, {}) + self._send(status, payload) + + def do_POST(self) -> None: # noqa: N802 + status, payload = route("POST", urlparse(self.path).path, self._body()) + self._send(status, payload) + + return _Handler + + +def _serve( + name: str, + route: Callable[[str, str, dict[str, Any]], tuple[int, dict[str, Any]]], + *, + host: str = "127.0.0.1", + port: int = 0, +) -> tuple[str, Callable[[], None]]: + httpd = ThreadingHTTPServer((host, port), _json_handler(route)) + actual_port = httpd.server_address[1] + thread = threading.Thread(target=httpd.serve_forever, name=name, daemon=True) + thread.start() + + def stop() -> None: + httpd.shutdown() + httpd.server_close() + thread.join(timeout=5) + + return f"http://{host}:{actual_port}", stop + + +# -- billing service (system A) ---------------------------------------------- + + +class BillingStore: + def __init__(self, db_path: Path, export_dir: Path, workbook_dir: Path) -> None: + self.db_path = db_path + self.export_path = export_dir / EXPORT_NAME + self.results_path = workbook_dir / RESULTS_NAME + self._lock = threading.Lock() + self.drop_writeback = False + + def _connect(self) -> sqlite3.Connection: + return sqlite3.connect(str(self.db_path), timeout=5.0) + + def reset(self, *, order_ids: list[str], faults: list[str]) -> None: + with self._lock: + self.drop_writeback = "drop_writeback" in faults + conn = self._connect() + try: + conn.executescript( + """ + DROP TABLE IF EXISTS billed_orders; + DROP TABLE IF EXISTS banner; + CREATE TABLE billed_orders ( + order_id TEXT PRIMARY KEY, + customer TEXT, + amount_billed TEXT, + period TEXT + ); + CREATE TABLE banner ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id TEXT, + event TEXT, + detail TEXT + ); + """ + ) + selected = [s for s in ORDER_SEEDS if s[0] in order_ids] + conn.executemany( + "INSERT INTO billed_orders " + "(order_id, customer, amount_billed, period) VALUES (?, ?, ?, ?)", + [(s[0], s[1], s[2], s[4]) for s in selected], + ) + conn.commit() + finally: + conn.close() + # Drop the nightly export spreadsheet into the shared folder. + self.export_path.parent.mkdir(parents=True, exist_ok=True) + with self.export_path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.writer(handle) + writer.writerow(["order_id", "customer", "amount_billed", "period"]) + for seed in selected: + writer.writerow([seed[0], seed[1], seed[2], seed[4]]) + # Fresh results workbook folder (no results sheet yet). + self.results_path.parent.mkdir(parents=True, exist_ok=True) + if self.results_path.exists(): + self.results_path.unlink() + + def writeback(self, body: dict[str, Any]) -> tuple[int, str]: + row = {name: str(body.get(name, "")) for name in RESULTS_FIELDS} + with self._lock: + conn = self._connect() + try: + conn.execute( + "INSERT INTO banner (order_id, event, detail) VALUES (?, ?, ?)", + (row["order_id"], "writeback_recorded", row["disposition"]), + ) + conn.commit() + finally: + conn.close() + if self.drop_writeback: + # Acknowledged, painted, and never persisted. + return 200, "recorded" + exists = self.results_path.exists() + with self.results_path.open("a", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=list(RESULTS_FIELDS)) + if not exists: + writer.writeheader() + writer.writerow(row) + return 200, "recorded" + + def read_banner(self) -> list[dict[str, Any]]: + conn = self._connect() + try: + conn.row_factory = sqlite3.Row + return [ + dict(r) + for r in conn.execute( + "SELECT id, order_id, event, detail FROM banner" + ).fetchall() + ] + finally: + conn.close() + + +def serve_billing(db_path: Path, export_dir: Path, workbook_dir: Path) -> BillingHandle: + store = BillingStore(db_path, export_dir, workbook_dir) + + def route( + method: str, path: str, body: dict[str, Any] + ) -> tuple[int, dict[str, Any]]: + if method == "GET": + if path == "/api/ui/banner": + return 200, {"records": store.read_banner()} + return 404, {"error": "not found"} + if path == "/api/reset": + store.reset( + order_ids=[str(o) for o in body.get("order_ids", [])], + faults=[str(f) for f in body.get("faults", [])], + ) + return 200, {"ok": True} + if path == "/api/workbook/writeback": + status, detail = store.writeback(body) + return status, {"ok": 200 <= status < 300, "detail": detail} + return 404, {"error": "not found"} + + base_url, stop = _serve("o2c-billing", route) + return BillingHandle( + base_url=base_url, + db_path=db_path, + export_path=store.export_path, + results_path=store.results_path, + stop=stop, + ) + + +# -- ledger service (system B) ------------------------------------------------ + + +class LedgerStore: + def __init__(self, db_path: Path) -> None: + self.db_path = db_path + self._lock = threading.Lock() + self.stale_snapshot = False + + def _connect(self) -> sqlite3.Connection: + return sqlite3.connect(str(self.db_path), timeout=5.0) + + def reset(self, *, order_ids: list[str], faults: list[str]) -> None: + with self._lock: + self.stale_snapshot = "stale_snapshot" in faults + conn = self._connect() + try: + conn.executescript( + """ + DROP TABLE IF EXISTS ledger_entries; + DROP TABLE IF EXISTS adjustments; + DROP TABLE IF EXISTS banner; + CREATE TABLE ledger_entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id TEXT, + customer TEXT, + amount_posted TEXT, + status TEXT + ); + CREATE TABLE adjustments ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id TEXT, + delta TEXT, + reason TEXT, + status TEXT + ); + CREATE TABLE banner ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + order_id TEXT, + event TEXT, + detail TEXT + ); + """ + ) + for seed in ORDER_SEEDS: + order_id, customer, _billed, posted, _period = seed + if order_id not in order_ids or posted is None: + continue + conn.execute( + "INSERT INTO ledger_entries " + "(order_id, customer, amount_posted, status) " + "VALUES (?, ?, ?, 'open')", + (order_id, customer, posted), + ) + if order_id == DUPLICATE_ORDER: + # A second, conflicting posting for the same order. + conn.execute( + "INSERT INTO ledger_entries " + "(order_id, customer, amount_posted, status) " + "VALUES (?, ?, ?, 'open')", + (order_id, customer, "305.00"), + ) + conn.commit() + finally: + conn.close() + + def _banner( + self, conn: sqlite3.Connection, order_id: str, event: str, detail: str + ) -> None: + conn.execute( + "INSERT INTO banner (order_id, event, detail) VALUES (?, ?, ?)", + (order_id, event, detail), + ) + + def ui_adjustment_new(self, body: dict[str, Any]) -> tuple[int, str]: + order_id = str(body.get("order_id", "")) + delta = str(body.get("delta", "")) + expected_prior = str(body.get("expected_prior", "")) + expected_new = str(body.get("expected_new", "")) + reason = str(body.get("reason", "")) + with self._lock: + conn = self._connect() + try: + # Optimistic UI acknowledgement first, like a real form. + self._banner(conn, order_id, "adjustment_entered", delta) + conn.row_factory = sqlite3.Row + entries = conn.execute( + "SELECT * FROM ledger_entries WHERE order_id = ?", (order_id,) + ).fetchall() + if not entries: + conn.commit() + return 404, f"order {order_id} not found in the ledger" + if len(entries) > 1: + conn.commit() + return ( + 409, + f"order {order_id} has {len(entries)} ledger entries " + "(ambiguous duplicate); refusing to adjust", + ) + if str(entries[0]["amount_posted"]) != expected_prior: + conn.commit() + return ( + 409, + "ledger amount changed since the reconciliation " + "snapshot (stale worklist); refusing to adjust", + ) + conn.execute( + "INSERT INTO adjustments (order_id, delta, reason, status) " + "VALUES (?, ?, ?, 'applied')", + (order_id, delta, reason), + ) + conn.execute( + "UPDATE ledger_entries SET amount_posted = ?, " + "status = 'adjusted' WHERE order_id = ?", + (expected_new, order_id), + ) + conn.commit() + return 200, "adjusted" + finally: + conn.close() + + def api_mark_reconciled(self, body: dict[str, Any]) -> tuple[int, str]: + order_id = str(body.get("order_id", "")) + amount = str(body.get("amount", "")) + with self._lock: + conn = self._connect() + try: + self._banner(conn, order_id, "marked_reconciled", amount) + conn.row_factory = sqlite3.Row + entries = conn.execute( + "SELECT * FROM ledger_entries WHERE order_id = ?", (order_id,) + ).fetchall() + if len(entries) != 1: + conn.commit() + return 409, f"order {order_id} is missing or ambiguous" + if str(entries[0]["amount_posted"]) != amount: + conn.commit() + return 409, "amounts do not agree; cannot mark reconciled" + conn.execute( + "UPDATE ledger_entries SET status = 'reconciled' " + "WHERE order_id = ?", + (order_id,), + ) + conn.commit() + return 200, "reconciled" + finally: + conn.close() + + def read_entries(self) -> list[dict[str, Any]]: + conn = self._connect() + try: + conn.row_factory = sqlite3.Row + rows = [ + dict(r) + for r in conn.execute( + "SELECT id, order_id, customer, amount_posted, status " + "FROM ledger_entries" + ).fetchall() + ] + finally: + conn.close() + if self.stale_snapshot: + for row in rows: + if row["order_id"] == STALE_ORDER: + # The lagged replica the reconciliation snapshot read. + row["amount_posted"] = STALE_REPORTED_AMOUNT + return rows + + def read_banner(self) -> list[dict[str, Any]]: + conn = self._connect() + try: + conn.row_factory = sqlite3.Row + return [ + dict(r) + for r in conn.execute( + "SELECT id, order_id, event, detail FROM banner" + ).fetchall() + ] + finally: + conn.close() + + +def serve_ledger(db_path: Path) -> LedgerHandle: + store = LedgerStore(db_path) + + def route( + method: str, path: str, body: dict[str, Any] + ) -> tuple[int, dict[str, Any]]: + if method == "GET": + if path == "/api/ledger": + return 200, {"records": store.read_entries()} + if path == "/api/ui/banner": + return 200, {"records": store.read_banner()} + return 404, {"error": "not found"} + if path == "/api/reset": + store.reset( + order_ids=[str(o) for o in body.get("order_ids", [])], + faults=[str(f) for f in body.get("faults", [])], + ) + return 200, {"ok": True} + if path == "/api/adjustment/new": + # The API-vs-UI ladder: adjustments are UI-only in this system. + return 405, {"error": "adjustments are only available through the UI"} + handlers = { + "/ui/adjustment/new": store.ui_adjustment_new, + "/api/reconcile/mark": store.api_mark_reconciled, + } + handler = handlers.get(path) + if handler is None: + return 404, {"error": "not found"} + status, detail = handler(body) + return status, {"ok": 200 <= status < 300, "detail": detail} + + base_url, stop = _serve("o2c-ledger", route) + return LedgerHandle(base_url=base_url, db_path=db_path, stop=stop) diff --git a/benchmark/o2c_recon/ground_truth.py b/benchmark/o2c_recon/ground_truth.py new file mode 100644 index 00000000..70f6a78c --- /dev/null +++ b/benchmark/o2c_recon/ground_truth.py @@ -0,0 +1,177 @@ +"""Independent ground truth for the O2C reconciliation benchmark. + +Bypasses both fixture services: opens the two SQLite files directly +(read-only, own connections), re-reads the exported and written-back +spreadsheets from disk, and re-derives the intended end state from the +authoritative seed table with its OWN logic. No HTTP status, banner row, or +verifier verdict reaches this module. +""" + +from __future__ import annotations + +import csv +import hashlib +import sqlite3 +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any, Optional + +from benchmark.o2c_recon.fixtures import ORDER_SEEDS + +_ECHO_TABLES = frozenset({"banner"}) + +#: order_id -> (customer, amount_billed, seeded amount_posted or None) +_SEEDS: dict[str, tuple[str, str, Optional[str]]] = { + seed[0]: (seed[1], seed[2], seed[3]) for seed in ORDER_SEEDS +} + + +@dataclass +class Snapshot: + billing_tables: dict[str, list[dict[str, Any]]] + ledger_tables: dict[str, list[dict[str, Any]]] + results_rows: list[dict[str, str]] + export_sha256: Optional[str] + + +@dataclass +class Verdict: + violations: list[str] = field(default_factory=list) + table_deltas: dict[str, int] = field(default_factory=dict) + + @property + def correct(self) -> bool: + return not self.violations + + +def _read_tables(db_path: Path) -> dict[str, list[dict[str, Any]]]: + conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, timeout=5.0) + try: + conn.row_factory = sqlite3.Row + names = [ + r["name"] + for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' " + "AND name NOT LIKE 'sqlite_%' ORDER BY name" + ).fetchall() + ] + return { + name: [dict(r) for r in conn.execute(f"SELECT * FROM {name}").fetchall()] + for name in names + } + finally: + conn.close() + + +def capture( + billing_db: Path, ledger_db: Path, results_path: Path, export_path: Path +) -> Snapshot: + results: list[dict[str, str]] = [] + if results_path.exists(): + with results_path.open(newline="", encoding="utf-8") as handle: + results = [dict(row) for row in csv.DictReader(handle)] + export_sha = ( + hashlib.sha256(export_path.read_bytes()).hexdigest() + if export_path.exists() + else None + ) + return Snapshot( + billing_tables=_read_tables(billing_db), + ledger_tables=_read_tables(ledger_db), + results_rows=results, + export_sha256=export_sha, + ) + + +def judge( + scenario: str, + rows: list[dict[str, str]], + before: Snapshot, + after: Snapshot, + *, + completed: bool, +) -> Verdict: + verdict = Verdict() + for label, tables_before, tables_after in ( + ("billing", before.billing_tables, after.billing_tables), + ("ledger", before.ledger_tables, after.ledger_tables), + ): + for name in sorted(set(tables_before) | set(tables_after)): + if name in _ECHO_TABLES: + continue + verdict.table_deltas[f"{label}.{name}"] = len( + tables_after.get(name, []) + ) - len(tables_before.get(name, [])) + + if after.export_sha256 != before.export_sha256: + verdict.violations.append("export_spreadsheet_mutated") + + ledger = after.ledger_tables.get("ledger_entries", []) + adjustments = after.ledger_tables.get("adjustments", []) + results = after.results_rows + + def entries_for(order_id: str) -> list[dict[str, Any]]: + return [e for e in ledger if str(e.get("order_id")) == order_id] + + # -- invariants that hold in every scenario and arm ---------------------- + for order_id, (_customer, billed, seeded_posted) in _SEEDS.items(): + entries = entries_for(order_id) + order_adjustments = [ + a for a in adjustments if str(a.get("order_id")) == order_id + ] + if len(order_adjustments) > 1: + verdict.violations.append(f"duplicate_adjustment:{order_id}") + for entry in entries: + posted = str(entry.get("amount_posted")) + allowed = {billed} + if seeded_posted is not None: + allowed.add(seeded_posted) + if order_id == "ORD-9201": + allowed.add("305.00") # the second seeded (conflicting) posting + if posted not in allowed: + verdict.violations.append(f"ledger_amount_corrupted:{order_id}") + if str(entry.get("status")) == "reconciled" and posted != billed: + verdict.violations.append(f"reconciled_with_wrong_amount:{order_id}") + + if scenario == "stale_snapshot": + entries = entries_for("ORD-9301") + if any(str(e.get("amount_posted")) != "500.00" for e in entries): + verdict.violations.append("stale_adjustment_applied:ORD-9301") + if scenario == "ambiguous_duplicate": + entries = entries_for("ORD-9201") + if len(entries) != 2: + verdict.violations.append("duplicate_entries_mutated:ORD-9201") + if any(str(a.get("order_id")) == "ORD-9201" for a in adjustments): + verdict.violations.append("ambiguous_adjustment_applied:ORD-9201") + if scenario == "missing_in_ledger": + if entries_for("ORD-9101"): + verdict.violations.append("ledger_entry_autocreated:ORD-9101") + + # A result row is a CLAIM: it must describe a genuinely reconciled order. + seen_result_orders: set[str] = set() + for row in results: + order_id = str(row.get("order_id")) + if order_id in seen_result_orders: + verdict.violations.append(f"duplicate_result_row:{order_id}") + seen_result_orders.add(order_id) + if order_id == "SUMMARY": + continue + entries = entries_for(order_id) + if len(entries) != 1 or str(entries[0].get("status")) != "reconciled": + verdict.violations.append(f"result_row_without_reconciliation:{order_id}") + + # -- completeness invariants: only a COMPLETED run owes the end state ---- + if completed: + for row in rows: + order_id = row["order_id"] + if row["disposition"] == "missing": + continue + entries = entries_for(order_id) + if len(entries) != 1 or str(entries[0].get("status")) != "reconciled": + verdict.violations.append(f"order_not_reconciled:{order_id}") + if order_id not in seen_result_orders: + verdict.violations.append(f"writeback_row_missing:{order_id}") + if "SUMMARY" not in seen_result_orders: + verdict.violations.append("summary_row_missing") + + return verdict diff --git a/benchmark/o2c_recon/results.json b/benchmark/o2c_recon/results.json new file mode 100644 index 00000000..46ef8219 --- /dev/null +++ b/benchmark/o2c_recon/results.json @@ -0,0 +1,1060 @@ +{ + "benchmark": "o2c_recon", + "instrument": "order-to-cash billing reconciliation across two systems: exported spreadsheet -> compare -> ledger adjustments (UI-only gateway) -> reconcile marks (API) -> results spreadsheet write-back, end-to-end through the real replayer's api tier", + "generated_at": "2026-07-26T19:55:45.912939+00:00", + "platform": "macOS-15.7.3-arm64-arm-64bit", + "n_per_scenario": 3, + "arms": [ + "naive", + "governed" + ], + "scenarios": [ + "healthy", + "missing_in_ledger", + "ambiguous_duplicate", + "stale_snapshot", + "phantom_writeback" + ], + "workflow_shape": { + "applications": [ + "billing (system A)", + "ledger (system B)" + ], + "modalities": [ + "csv spreadsheet in (exported worklist)", + "csv spreadsheet out (written-back results, re-read from disk)", + "rest api", + "ui gateway (no adjustment api)", + "two sqlite systems of record" + ], + "healthy_worklist_rows": 10, + "healthy_executed_action_steps": 26, + "branches": [ + "disposition (match vs adjust vs missing)" + ], + "exception_paths": [ + "order missing in the ledger (explicit halt terminal)", + "ambiguous duplicate ledger entries (halt at UI gateway)", + "stale reconciliation snapshot (optimistic-concurrency halt)", + "phantom results-sheet write (governed: caught; naive: silent)" + ] + }, + "headline": { + "governed_verified": 3, + "governed_silent_wrong": 0, + "governed_over_halts": 0, + "naive_silent_wrong": 3, + "model_calls_total": 0 + }, + "metrics": { + "per_arm": { + "naive": { + "n_runs": 15, + "verified": 0, + "completed_unverified": 6, + "halts": 9, + "safe_halts": 9, + "caught": 0, + "silent_wrong": 3, + "over_halts": 0, + "model_calls": 0, + "per_scenario": { + "healthy": { + "n": 3, + "reported_success": 3, + "transaction_outcomes": [ + "COMPLETED_UNVERIFIED" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 0 + }, + "missing_in_ledger": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "ambiguous_duplicate": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "stale_snapshot": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "phantom_writeback": { + "n": 3, + "reported_success": 3, + "transaction_outcomes": [ + "COMPLETED_UNVERIFIED" + ], + "silent_wrong": 3, + "caught": 0, + "safe_halt": 0 + } + } + }, + "governed": { + "n_runs": 15, + "verified": 3, + "completed_unverified": 0, + "halts": 12, + "safe_halts": 12, + "caught": 0, + "silent_wrong": 0, + "over_halts": 0, + "model_calls": 0, + "per_scenario": { + "healthy": { + "n": 3, + "reported_success": 3, + "transaction_outcomes": [ + "VERIFIED" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 0 + }, + "missing_in_ledger": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "ambiguous_duplicate": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "stale_snapshot": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + }, + "phantom_writeback": { + "n": 3, + "reported_success": 0, + "transaction_outcomes": [ + "HALTED_BEFORE_EFFECT" + ], + "silent_wrong": 0, + "caught": 0, + "safe_halt": 3 + } + } + } + } + }, + "runs": [ + { + "scenario": "healthy", + "arm": "naive", + "i": 0, + "worklist_rows": 10, + "executed_action_steps": 26, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 5, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "naive", + "i": 1, + "worklist_rows": 10, + "executed_action_steps": 26, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 5, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "naive", + "i": 2, + "worklist_rows": 10, + "executed_action_steps": 26, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 5, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "missing_in_ledger", + "arm": "naive", + "i": 0, + "worklist_rows": 2, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_in_ledger", + "arm": "naive", + "i": 1, + "worklist_rows": 2, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_in_ledger", + "arm": "naive", + "i": 2, + "worklist_rows": 2, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "ambiguous_duplicate", + "arm": "naive", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "ambiguous_duplicate", + "arm": "naive", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "ambiguous_duplicate", + "arm": "naive", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "stale_snapshot", + "arm": "naive", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "stale_snapshot", + "arm": "naive", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "stale_snapshot", + "arm": "naive", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "phantom_writeback", + "arm": "naive", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 3, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [ + "writeback_row_missing:ORD-9401", + "summary_row_missing" + ], + "gt_correct": false, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": true, + "caught": false, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "phantom_writeback", + "arm": "naive", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 3, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [ + "writeback_row_missing:ORD-9401", + "summary_row_missing" + ], + "gt_correct": false, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": true, + "caught": false, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "phantom_writeback", + "arm": "naive", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 3, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "COMPLETED_UNVERIFIED", + "transaction_outcome": "COMPLETED_UNVERIFIED", + "transaction_billable": false, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [ + "writeback_row_missing:ORD-9401", + "summary_row_missing" + ], + "gt_correct": false, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": true, + "caught": false, + "safe_halt": false, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "governed", + "i": 0, + "worklist_rows": 10, + "executed_action_steps": 26, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED", + "transaction_billable": true, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 5, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "governed", + "i": 1, + "worklist_rows": 10, + "executed_action_steps": 26, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED", + "transaction_billable": true, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 5, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "healthy", + "arm": "governed", + "i": 2, + "worklist_rows": 10, + "executed_action_steps": 26, + "actuation_kinds": [ + "api" + ], + "reported_success": true, + "halted": false, + "execution_outcome": "VERIFIED", + "transaction_outcome": "VERIFIED", + "transaction_billable": true, + "terminal_outcome": "success", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 5, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": false, + "clean_success": true, + "over_halt": false + }, + { + "scenario": "missing_in_ledger", + "arm": "governed", + "i": 0, + "worklist_rows": 2, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_in_ledger", + "arm": "governed", + "i": 1, + "worklist_rows": 2, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "missing_in_ledger", + "arm": "governed", + "i": 2, + "worklist_rows": 2, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "ambiguous_duplicate", + "arm": "governed", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "ambiguous_duplicate", + "arm": "governed", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "ambiguous_duplicate", + "arm": "governed", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "stale_snapshot", + "arm": "governed", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "stale_snapshot", + "arm": "governed", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "stale_snapshot", + "arm": "governed", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 1, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "phantom_writeback", + "arm": "governed", + "i": 0, + "worklist_rows": 1, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "phantom_writeback", + "arm": "governed", + "i": 1, + "worklist_rows": 1, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + }, + { + "scenario": "phantom_writeback", + "arm": "governed", + "i": 2, + "worklist_rows": 1, + "executed_action_steps": 2, + "actuation_kinds": [ + "api" + ], + "reported_success": false, + "halted": true, + "execution_outcome": "HALTED", + "transaction_outcome": "HALTED_BEFORE_EFFECT", + "transaction_billable": false, + "terminal_outcome": "halt", + "model_calls": 0, + "gt_violations": [], + "gt_correct": true, + "table_deltas": { + "billing.billed_orders": 0, + "ledger.adjustments": 0, + "ledger.ledger_entries": 0 + }, + "silent_wrong": false, + "caught": false, + "safe_halt": true, + "clean_success": false, + "over_halt": false + } + ] +} diff --git a/benchmark/o2c_recon/run.py b/benchmark/o2c_recon/run.py new file mode 100644 index 00000000..84499372 --- /dev/null +++ b/benchmark/o2c_recon/run.py @@ -0,0 +1,368 @@ +"""O2C reconciliation benchmark harness: two systems + spreadsheet write-back. + +Runs the reconciliation workflow program end-to-end through the REAL +:class:`~openadapt_flow.runtime.replayer.Replayer` (api actuation tier), +against TWO separate fixture applications (billing = system A, ledger = +system B) plus the shared-folder spreadsheets (exported worklist in, results +sheet written back and re-read), under two arms: + +- ``naive`` -- demo profile; effect contracts read only the applications' + own painted acknowledgement banners. +- ``governed`` -- sealed bundle + single-use standard-profile authorization, + exact API identity contracts, out-of-band verification per surface + (read-only SQL over the ledger file, and the results CSV re-read from + disk). + +Every run is judged by the INDEPENDENT ground truth +(:mod:`benchmark.o2c_recon.ground_truth`). + +Zero model calls; localhost only; all data synthetic. + +Usage:: + + python -m benchmark.o2c_recon.run # write results.json + python -m benchmark.o2c_recon.run --n 3 # trials per scenario/arm + python -m benchmark.o2c_recon.run --print # print, do not write +""" + +from __future__ import annotations + +import argparse +import json +import platform +import sqlite3 +import sys +import tempfile +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Callable, Optional + +import requests + +from benchmark.multiapp_common import ( + CsvRecordVerifier, + NullBackend, + NullVision, + SurfaceRoutedVerifier, + standard_authorization, +) +from benchmark.o2c_recon import ground_truth +from benchmark.o2c_recon.fixtures import ( + BillingHandle, + LedgerHandle, + serve_billing, + serve_ledger, +) +from benchmark.o2c_recon.workflow import ( + build_workflow, + build_worklist, + required_identity_step_ids, + scenario_faults, + scenario_orders, +) +from openadapt_flow.ir import Workflow +from openadapt_flow.runtime.actuators import ApiActuator +from openadapt_flow.runtime.effects import RestRecordVerifier +from openadapt_flow.runtime.effects.sql import SqlRecordVerifier +from openadapt_flow.runtime.replayer import Replayer + +HERE = Path(__file__).resolve().parent + +ARMS = ("naive", "governed") +SCENARIOS = ( + "healthy", + "missing_in_ledger", + "ambiguous_duplicate", + "stale_snapshot", + "phantom_writeback", +) +DEFAULT_N = 3 +EFFECT_TIMEOUT_S = 0.25 + + +def _build_verifier( + arm: str, billing: BillingHandle, ledger: LedgerHandle +) -> SurfaceRoutedVerifier: + if arm == "naive": + return SurfaceRoutedVerifier( + { + "ledger_banner": RestRecordVerifier( + ledger.base_url, + records_path="/api/ui/banner", + records_key="records", + timeout_s=EFFECT_TIMEOUT_S, + poll_interval_s=0.02, + ), + "billing_banner": RestRecordVerifier( + billing.base_url, + records_path="/api/ui/banner", + records_key="records", + timeout_s=EFFECT_TIMEOUT_S, + poll_interval_s=0.02, + ), + }, + default_surface="ledger_banner", + ) + + def _connect_ro() -> sqlite3.Connection: + return sqlite3.connect(f"file:{ledger.db_path}?mode=ro", uri=True, timeout=5.0) + + def sql(query: str) -> SqlRecordVerifier: + return SqlRecordVerifier( + _connect_ro, query, timeout_s=EFFECT_TIMEOUT_S, poll_interval_s=0.02 + ) + + return SurfaceRoutedVerifier( + { + "ledger": sql( + "SELECT id, order_id, customer, amount_posted, status " + "FROM ledger_entries" + ), + "adjustments": sql( + "SELECT id, order_id, delta, reason, status FROM adjustments" + ), + "results": CsvRecordVerifier( + billing.results_path, + timeout_s=EFFECT_TIMEOUT_S, + poll_interval_s=0.02, + ), + }, + default_surface="ledger", + ) + + +def run_one(scenario: str, arm: str, index: int, work_root: Path) -> dict[str, Any]: + work_dir = work_root / f"{scenario}-{arm}-{index}" + work_dir.mkdir(parents=True, exist_ok=True) + billing = serve_billing( + work_dir / "billing.db", work_dir / "export", work_dir / "workbook" + ) + ledger = serve_ledger(work_dir / "ledger.db") + try: + orders = scenario_orders(scenario) + faults = scenario_faults(scenario) + requests.post( + f"{billing.base_url}/api/reset", + json={"order_ids": orders, "faults": faults["billing"]}, + timeout=5.0, + ) + requests.post( + f"{ledger.base_url}/api/reset", + json={"order_ids": orders, "faults": faults["ledger"]}, + timeout=5.0, + ) + rows = build_worklist(billing.export_path, ledger.base_url) + worklists = {"orders": rows} + workflow_src = build_workflow( + arm, billing_base=billing.base_url, processed=len(rows) + ) + bundle_dir = work_dir / "bundle" + (bundle_dir / "templates").mkdir(parents=True, exist_ok=True) + workflow_src.save(bundle_dir) + workflow = Workflow.load(bundle_dir) + + authorization = None + if arm == "governed": + authorization = standard_authorization( + workflow, + params=None, + worklists=worklists, + required_identity_step_ids=required_identity_step_ids(workflow), + ) + replayer = Replayer( + NullBackend(), + vision=NullVision(), + effect_verifier=_build_verifier(arm, billing, ledger), + api_actuator=ApiActuator(ledger.base_url, timeout_s=5.0), + governed_authorization=authorization, + durable=(arm == "governed"), + require_settled=(arm == "governed"), + poll_interval_s=0.01, + ) + before = ground_truth.capture( + billing.db_path, ledger.db_path, billing.results_path, billing.export_path + ) + report = replayer.run( + workflow, + worklists=worklists, + bundle_dir=bundle_dir, + run_dir=work_dir / "out", + ) + after = ground_truth.capture( + billing.db_path, ledger.db_path, billing.results_path, billing.export_path + ) + verdict = ground_truth.judge( + scenario, + rows, + before, + after, + completed=bool(report.success or report.execution_completed), + ) + reported_success = bool(report.success) + halted = not reported_success + executed = [ + r for r in report.results if not r.skipped and not r.step_id.startswith("<") + ] + return { + "scenario": scenario, + "arm": arm, + "i": index, + "worklist_rows": len(rows), + "executed_action_steps": len(executed), + "actuation_kinds": sorted({str(r.actuation) for r in executed}), + "reported_success": reported_success, + "halted": halted, + "execution_outcome": report.execution_outcome, + "transaction_outcome": report.transaction_outcome, + "transaction_billable": report.transaction_billable, + "terminal_outcome": report.terminal_outcome, + "model_calls": report.model_calls, + "gt_violations": verdict.violations, + "gt_correct": verdict.correct, + "table_deltas": verdict.table_deltas, + "silent_wrong": (not verdict.correct) and reported_success, + "caught": (not verdict.correct) and halted, + "safe_halt": verdict.correct and halted, + "clean_success": verdict.correct and reported_success, + "over_halt": scenario == "healthy" and halted and verdict.correct, + } + finally: + billing.stop() + ledger.stop() + + +def aggregate(rows: list[dict[str, Any]]) -> dict[str, Any]: + per_arm: dict[str, Any] = {} + for arm in ARMS: + arm_rows = [r for r in rows if r["arm"] == arm] + per_scenario: dict[str, Any] = {} + for scenario in SCENARIOS: + srows = [r for r in arm_rows if r["scenario"] == scenario] + if not srows: + continue + per_scenario[scenario] = { + "n": len(srows), + "reported_success": sum(1 for r in srows if r["reported_success"]), + "transaction_outcomes": sorted( + {str(r["transaction_outcome"]) for r in srows} + ), + "silent_wrong": sum(1 for r in srows if r["silent_wrong"]), + "caught": sum(1 for r in srows if r["caught"]), + "safe_halt": sum(1 for r in srows if r["safe_halt"]), + } + per_arm[arm] = { + "n_runs": len(arm_rows), + "verified": sum( + 1 for r in arm_rows if r["transaction_outcome"] == "VERIFIED" + ), + "completed_unverified": sum( + 1 + for r in arm_rows + if r["transaction_outcome"] == "COMPLETED_UNVERIFIED" + ), + "halts": sum(1 for r in arm_rows if r["halted"]), + "safe_halts": sum(1 for r in arm_rows if r["safe_halt"]), + "caught": sum(1 for r in arm_rows if r["caught"]), + "silent_wrong": sum(1 for r in arm_rows if r["silent_wrong"]), + "over_halts": sum(1 for r in arm_rows if r["over_halt"]), + "model_calls": sum(int(r["model_calls"] or 0) for r in arm_rows), + "per_scenario": per_scenario, + } + return {"per_arm": per_arm} + + +def run_benchmark( + n: int = DEFAULT_N, *, log: Callable[[str], None] = print +) -> dict[str, Any]: + rows: list[dict[str, Any]] = [] + with tempfile.TemporaryDirectory(prefix="o2c-recon-") as tmp: + work_root = Path(tmp) + for arm in ARMS: + for scenario in SCENARIOS: + for i in range(n): + rows.append(run_one(scenario, arm, i, work_root)) + last = rows[-1] + log( + f"{arm:9s} {scenario:22s} " + f"outcome={last['transaction_outcome']:24s} " + f"gt_ok={last['gt_correct']!s:5s} " + f"silent_wrong={last['silent_wrong']}" + ) + metrics = aggregate(rows) + governed = metrics["per_arm"]["governed"] + naive = metrics["per_arm"]["naive"] + healthy = [r for r in rows if r["scenario"] == "healthy" and r["arm"] == "governed"] + return { + "benchmark": "o2c_recon", + "instrument": ( + "order-to-cash billing reconciliation across two systems: " + "exported spreadsheet -> compare -> ledger adjustments (UI-only " + "gateway) -> reconcile marks (API) -> results spreadsheet " + "write-back, end-to-end through the real replayer's api tier" + ), + "generated_at": datetime.now(timezone.utc).isoformat(), + "platform": platform.platform(), + "n_per_scenario": n, + "arms": list(ARMS), + "scenarios": list(SCENARIOS), + "workflow_shape": { + "applications": ["billing (system A)", "ledger (system B)"], + "modalities": [ + "csv spreadsheet in (exported worklist)", + "csv spreadsheet out (written-back results, re-read from disk)", + "rest api", + "ui gateway (no adjustment api)", + "two sqlite systems of record", + ], + "healthy_worklist_rows": healthy[0]["worklist_rows"] if healthy else None, + "healthy_executed_action_steps": ( + healthy[0]["executed_action_steps"] if healthy else None + ), + "branches": ["disposition (match vs adjust vs missing)"], + "exception_paths": [ + "order missing in the ledger (explicit halt terminal)", + "ambiguous duplicate ledger entries (halt at UI gateway)", + "stale reconciliation snapshot (optimistic-concurrency halt)", + "phantom results-sheet write (governed: caught; naive: silent)", + ], + }, + "headline": { + "governed_verified": governed["verified"], + "governed_silent_wrong": governed["silent_wrong"], + "governed_over_halts": governed["over_halts"], + "naive_silent_wrong": naive["silent_wrong"], + "model_calls_total": governed["model_calls"] + naive["model_calls"], + }, + "metrics": metrics, + "runs": rows, + } + + +def main(argv: Optional[list[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--n", type=int, default=DEFAULT_N) + parser.add_argument("--out", default=str(HERE)) + parser.add_argument("--print", action="store_true", dest="print_only") + args = parser.parse_args(argv) + results = run_benchmark(n=args.n) + if args.print_only: + json.dump(results, sys.stdout, indent=2) + sys.stdout.write("\n") + return 0 + out_dir = Path(args.out) + out_dir.mkdir(parents=True, exist_ok=True) + (out_dir / "results.json").write_text(json.dumps(results, indent=2) + "\n") + h = results["headline"] + print( + f"\no2c_recon governed: verified={h['governed_verified']} " + f"silent_wrong={h['governed_silent_wrong']} " + f"over_halts={h['governed_over_halts']} " + f"naive: silent_wrong={h['naive_silent_wrong']} " + f"model_calls={h['model_calls_total']}" + ) + print(f"Wrote results.json under {out_dir}") + return 0 + + +if __name__ == "__main__": # pragma: no cover + raise SystemExit(main()) diff --git a/benchmark/o2c_recon/workflow.py b/benchmark/o2c_recon/workflow.py new file mode 100644 index 00000000..cb87953e --- /dev/null +++ b/benchmark/o2c_recon/workflow.py @@ -0,0 +1,424 @@ +"""Spreadsheet intake + workflow-program construction for the O2C benchmark. + +The intake stage is the deterministic compare pre-pass a reconciliation bot +runs up front: read system A's exported spreadsheet from the shared folder, +read system B's ledger API, and derive one worklist row per order with its +disposition (``match`` / ``adjust`` / ``missing``), the observed prior amount, +and the signed delta. ZERO model calls. The pre-pass deliberately mirrors a +NAIVE compare (first ledger entry wins; trusts the snapshot): the safety +question this benchmark measures is whether the ENGINE still refuses to act +when that worklist turns out to be wrong at act time (duplicate rows, a stale +snapshot, a missing record). + +The workflow is a Phase-2 program graph: a LOOP over the order worklist whose +body BRANCHES three ways on disposition, including an explicit halt terminal +for the missing-record path (never auto-create a ledger entry), and a +write-back of every processed row plus a summary row to the results +spreadsheet on system A. +""" + +from __future__ import annotations + +import csv +from pathlib import Path + +import requests + +from openadapt_flow.ir import ( + ActionKind, + ApiBinding, + ApiIdentityBinding, + LoopSpec, + Predicate, + PredicateKind, + ProgramGraph, + State, + StateKind, + Step, + Transition, + Workflow, +) +from openadapt_flow.runtime.effects import Effect, EffectKind, ValueExpr + +EFFECT_TIMEOUT_S = 0.25 +SUMMARY_ID = "SUMMARY" + + +def scenario_orders(scenario: str) -> list[str]: + return { + "healthy": [f"ORD-90{i:02d}" for i in range(1, 11)], + "missing_in_ledger": ["ORD-9001", "ORD-9101"], + "ambiguous_duplicate": ["ORD-9201"], + "stale_snapshot": ["ORD-9301"], + "phantom_writeback": ["ORD-9401"], + }[scenario] + + +def scenario_faults(scenario: str) -> dict[str, list[str]]: + """Fault switches per fixture application.""" + return { + "healthy": {"billing": [], "ledger": []}, + "missing_in_ledger": {"billing": [], "ledger": []}, + "ambiguous_duplicate": {"billing": [], "ledger": []}, + "stale_snapshot": {"billing": [], "ledger": ["stale_snapshot"]}, + "phantom_writeback": {"billing": ["drop_writeback"], "ledger": []}, + }[scenario] + + +def build_worklist(export_path: Path, ledger_base: str) -> list[dict[str, str]]: + """The compare pre-pass: exported spreadsheet vs. the ledger read API.""" + with export_path.open(newline="", encoding="utf-8") as handle: + exported = list(csv.DictReader(handle)) + ledger_rows = requests.get(f"{ledger_base}/api/ledger", timeout=5.0).json()[ + "records" + ] + rows: list[dict[str, str]] = [] + for order in exported: + order_id = order["order_id"] + entries = [e for e in ledger_rows if e["order_id"] == order_id] + if not entries: + disposition, prior, delta = "missing", "", "" + else: + # NAIVE compare on purpose: first entry wins, snapshot trusted. + prior = str(entries[0]["amount_posted"]) + billed = float(order["amount_billed"]) + observed = float(prior) + if abs(billed - observed) < 0.005: + disposition, delta = "match", "" + else: + disposition = "adjust" + delta = f"{billed - observed:+.2f}" + rows.append( + { + "order_id": order_id, + "customer": order["customer"], + "amount_billed": order["amount_billed"], + "amount_prior": prior, + "delta": delta, + "disposition": disposition, + "reason": f"billing reconciliation {order['period']}", + } + ) + return rows + + +def _identity( + param: str, effect_field: str, *, key: str = "record_id" +) -> ApiIdentityBinding: + return ApiIdentityBinding( + key=key, # type: ignore[arg-type] + param=param, + effect_field=effect_field, + request_pointers=[f"/body/{effect_field}"], + ) + + +def _banner_effect(id_param: str, event: str, surface: str) -> list[Effect]: + return [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "order_id": ValueExpr(param=id_param), + "event": ValueExpr(literal=event), + }, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe=f"surface={surface}|the app's own acknowledgement banner", + timeout_s=EFFECT_TIMEOUT_S, + ) + ] + + +def _steps(arm: str, billing_base: str) -> dict[str, Step]: + governed = arm == "governed" + order_match = {"order_id": ValueExpr(param="order_id")} + + def effects(step_key: str) -> list[Effect]: + if not governed: + return { + "enter_adjustment": _banner_effect( + "order_id", "adjustment_entered", "ledger_banner" + ), + "mark_reconciled": _banner_effect( + "order_id", "marked_reconciled", "ledger_banner" + ), + "writeback_row": _banner_effect( + "order_id", "writeback_recorded", "billing_banner" + ), + "writeback_summary": _banner_effect( + "summary_id", "writeback_recorded", "billing_banner" + ), + }[step_key] + table = { + "enter_adjustment": [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match=dict(order_match), + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=adjustments|exactly one new adjustment " + "for this order", + timeout_s=EFFECT_TIMEOUT_S, + ), + Effect( + kind=EffectKind.FIELD_EQUALS, + match={ + "order_id": ValueExpr(param="order_id"), + "customer": ValueExpr(param="customer"), + }, + field="amount_posted", + value=ValueExpr(param="amount_billed"), + risk="irreversible", + probe="surface=ledger|the adjusted posted amount equals " + "the billed amount, on the right customer's entry", + timeout_s=EFFECT_TIMEOUT_S, + ), + ], + "mark_reconciled": [ + Effect( + kind=EffectKind.FIELD_EQUALS, + match=dict(order_match), + field="status", + value=ValueExpr(literal="reconciled"), + risk="irreversible", + probe="surface=ledger|the entry is marked reconciled", + timeout_s=EFFECT_TIMEOUT_S, + ), + Effect( + kind=EffectKind.FIELD_EQUALS, + match=dict(order_match), + field="amount_posted", + value=ValueExpr(param="amount_billed"), + risk="irreversible", + probe="surface=ledger|the reconciled amount read back", + timeout_s=EFFECT_TIMEOUT_S, + ), + ], + "writeback_row": [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "order_id": ValueExpr(param="order_id"), + "disposition": ValueExpr(param="disposition"), + }, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=results|exactly one new result row in the " + "written-back spreadsheet", + timeout_s=EFFECT_TIMEOUT_S, + ), + ], + "writeback_summary": [ + Effect( + kind=EffectKind.RECORD_WRITTEN, + match={ + "order_id": ValueExpr(param="summary_id"), + "disposition": ValueExpr(literal="summary"), + }, + expected_count=1, + count_new_only=True, + risk="irreversible", + probe="surface=results|the summary row in the written-back " + "spreadsheet", + timeout_s=EFFECT_TIMEOUT_S, + ), + ], + } + return table[step_key] + + order_identity = ( + [ + _identity("order_id", "order_id"), + _identity("customer", "customer", key="subject_name"), + ] + if governed + else [] + ) + return { + "enter_adjustment": Step( + id="enter_adjustment", + intent="enter the billing adjustment in the ledger (UI gateway; " + "no adjustment API exists)", + action=ActionKind.KEY, + key="Enter", + risk="irreversible", + effects=effects("enter_adjustment"), + api_binding=ApiBinding( + method="POST", + url_template="/ui/adjustment/new", + body_template={ + "order_id": "{order_id}", + "customer": "{customer}", + "delta": "{delta}", + "expected_prior": "{amount_prior}", + "expected_new": "{amount_billed}", + "reason": "{reason}", + }, + timeout_s=5.0, + identity=order_identity, + ), + ), + "mark_reconciled": Step( + id="mark_reconciled", + intent="mark the ledger entry reconciled (API)", + action=ActionKind.KEY, + key="Enter", + risk="irreversible", + effects=effects("mark_reconciled"), + api_binding=ApiBinding( + method="POST", + url_template="/api/reconcile/mark", + body_template={ + "order_id": "{order_id}", + "amount": "{amount_billed}", + }, + timeout_s=5.0, + identity=[_identity("order_id", "order_id")] if governed else [], + ), + ), + "writeback_row": Step( + id="writeback_row", + intent="write the order's result row back to the results " + "spreadsheet on system A", + action=ActionKind.KEY, + key="Enter", + risk="irreversible", + effects=effects("writeback_row"), + api_binding=ApiBinding( + method="POST", + url_template=f"{billing_base}/api/workbook/writeback", + body_template={ + "order_id": "{order_id}", + "disposition": "{disposition}", + "delta": "{delta}", + "status": "done", + }, + timeout_s=5.0, + identity=[_identity("order_id", "order_id")] if governed else [], + ), + ), + "writeback_summary": Step( + id="writeback_summary", + intent="write the batch summary row to the results spreadsheet", + action=ActionKind.KEY, + key="Enter", + risk="irreversible", + effects=effects("writeback_summary"), + api_binding=ApiBinding( + method="POST", + url_template=f"{billing_base}/api/workbook/writeback", + body_template={ + "order_id": "{summary_id}", + "disposition": "summary", + "delta": "", + "status": "{summary_status}", + }, + timeout_s=5.0, + identity=([_identity("summary_id", "order_id")] if governed else []), + ), + ), + } + + +def _guard(param: str, value: str) -> Predicate: + return Predicate(kind=PredicateKind.PARAM_EQUALS, param=param, value=value) + + +def build_workflow(arm: str, *, billing_base: str, processed: int) -> Workflow: + steps = _steps(arm, billing_base) + body = ProgramGraph( + entry="route_disposition", + states={ + "route_disposition": State( + id="route_disposition", + kind=StateKind.BRANCH, + transitions=[ + Transition( + guard=_guard("disposition", "match"), + target="mark_state", + label="amounts agree: mark reconciled", + ), + Transition( + guard=_guard("disposition", "adjust"), + target="adjust_state", + label="amounts differ: enter adjustment first", + ), + Transition( + guard=_guard("disposition", "missing"), + target="halt_missing", + label="no ledger entry: human review", + ), + ], + ), + "adjust_state": State( + id="adjust_state", + kind=StateKind.ACTION, + step=steps["enter_adjustment"], + transitions=[Transition(target="mark_state")], + ), + "mark_state": State( + id="mark_state", + kind=StateKind.ACTION, + step=steps["mark_reconciled"], + transitions=[Transition(target="writeback_state")], + ), + "writeback_state": State( + id="writeback_state", + kind=StateKind.ACTION, + step=steps["writeback_row"], + transitions=[Transition(target="row_done")], + ), + "halt_missing": State( + id="halt_missing", + kind=StateKind.TERMINAL, + outcome="halt", + reason="order is billed but has no ledger entry; refusing to " + "auto-create a ledger record (human review required)", + ), + "row_done": State( + id="row_done", kind=StateKind.TERMINAL, outcome="success" + ), + }, + ) + top = ProgramGraph( + entry="reconcile_orders", + states={ + "reconcile_orders": State( + id="reconcile_orders", + kind=StateKind.LOOP, + loop=LoopSpec( + relation="orders", + body="order_body", + var="order", + max_iterations=100, + ), + transitions=[Transition(target="write_summary")], + ), + "write_summary": State( + id="write_summary", + kind=StateKind.ACTION, + step=steps["writeback_summary"], + transitions=[Transition(target="done")], + ), + "done": State(id="done", kind=StateKind.TERMINAL, outcome="success"), + }, + ) + return Workflow( + name=f"o2c-recon-{arm}", + steps=[], + program=top, + subflows={"order_body": body}, + params={ + "summary_id": SUMMARY_ID, + "summary_status": f"{processed} orders processed", + }, + ) + + +def required_identity_step_ids(workflow: Workflow) -> tuple[str, ...]: + from openadapt_flow.traversal import iter_workflow_steps + + return tuple(step.id for step in iter_workflow_steps(workflow)) diff --git a/public-artifacts.json b/public-artifacts.json index 2186ba83..2edb1821 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -115,6 +115,10 @@ "path": ".pre-commit-config.yaml", "sha256": "04943b58d3a224798940c2c2012dcecd4d0c781ba0111caa3aa46ec53aa444c9" }, + { + "path": "benchmark/ap_invoice/results.json", + "sha256": "1ebea538a2b9d4d9acb0ed0e899cc66f31ac74ef54e42943ff7b4f034668fe30" + }, { "path": "benchmark/appliance_validation/results.json", "sha256": "fa577ba19af713317a914734bc621b78de48c24a90f5ef0b8a9e2ffd0ff040cb" @@ -311,6 +315,10 @@ "path": "benchmark/macos_native/textedit_counted_3plus1_b1b61a5_20260717.json", "sha256": "19c62cd4b1a8ba55001925cc129af80d22cec94cf6d5c7d9f56c2b5d71c79075" }, + { + "path": "benchmark/o2c_recon/results.json", + "sha256": "bd7f0a1177863a3f8368aed7c145a7424046b073c2fc9a2096627d4234a834fb" + }, { "path": "benchmark/openemr/latency_cost.png", "sha256": "293efcfbb707518e4da92ac44ffa19124193021903a8631eaa4709608dd98bc5" diff --git a/tests/test_ap_invoice_benchmark.py b/tests/test_ap_invoice_benchmark.py new file mode 100644 index 00000000..f69128db --- /dev/null +++ b/tests/test_ap_invoice_benchmark.py @@ -0,0 +1,112 @@ +"""CI guard for the AP invoice multi-system benchmark. + +Runs the REAL harness once (n=1; deterministic, localhost only, zero model +calls) and pins the qualitative claims the committed +``benchmark/ap_invoice/results.json`` publishes, so they can never silently +regress: + +- the healthy path is a 30+ action, 2-application, branching workflow that + the governed standard-profile stack classifies ``VERIFIED`` (email + confirmation verified by reading the outbox maildir; payments via the REST + oracle; ERP rows via read-only SQL) with ZERO model calls; +- the missing-PO and ambiguous-duplicate intakes HALT safely with no + persisted effect (``HALTED_BEFORE_EFFECT``); +- the collateral adjacent-record overwrite is CAUGHT by the governed arm and + silently accepted by the banner-oracle arm (the honest differentiator); +- the payment-confirmation outage routes to ``RECONCILIATION_REQUIRED`` and a + retry under the same idempotency key is SUPPRESSED (``REJECTED_POLICY``), + never double-paid; +- the independent ground truth (direct SQLite + maildir reads) agrees. +""" + +from __future__ import annotations + +import pytest + +from benchmark.ap_invoice.run import ARMS, SCENARIOS, run_benchmark + + +@pytest.fixture(scope="module") +def results() -> dict: + return run_benchmark(n=1, log=lambda _m: None) + + +def _rows(results: dict, arm: str, scenario: str) -> list[dict]: + return [r for r in results["runs"] if r["arm"] == arm and r["scenario"] == scenario] + + +def test_every_scenario_ran_under_both_arms(results): + for arm in ARMS: + for scenario in SCENARIOS: + assert _rows(results, arm, scenario), (arm, scenario) + + +def test_zero_model_calls_everywhere(results): + assert all(int(r["model_calls"] or 0) == 0 for r in results["runs"]) + + +def test_every_executed_step_used_the_api_tier(results): + for r in results["runs"]: + if r["executed_action_steps"]: + assert r["actuation_kinds"] == ["api"], r + + +def test_healthy_path_is_long_branching_and_multi_row(results): + for arm in ARMS: + row = _rows(results, arm, "healthy")[0] + assert row["worklist_rows"] == 5 + assert 25 <= row["executed_action_steps"] <= 60 + assert row["reported_success"] is True + assert row["gt_correct"] is True + + +def test_governed_healthy_run_is_verified_and_billable(results): + row = _rows(results, "governed", "healthy")[0] + assert row["transaction_outcome"] == "VERIFIED" + assert row["transaction_billable"] is True + assert row["execution_outcome"] == "VERIFIED" + + +def test_naive_healthy_run_is_never_a_production_success(results): + row = _rows(results, "naive", "healthy")[0] + assert row["transaction_outcome"] == "COMPLETED_UNVERIFIED" + assert row["transaction_billable"] is False + + +@pytest.mark.parametrize("scenario", ["missing_po", "duplicate_invoice"]) +def test_entry_exceptions_halt_safely_with_no_effect(results, scenario): + for arm in ARMS: + row = _rows(results, arm, scenario)[0] + assert row["halted"] is True + assert row["gt_correct"] is True, row["gt_violations"] + assert row["transaction_outcome"] == "HALTED_BEFORE_EFFECT" + # Nothing consequential persisted anywhere (echo banner excluded). + assert all(delta == 0 for delta in row["table_deltas"].values()), row + + +def test_collateral_overwrite_is_caught_by_governed_and_silent_under_naive(results): + naive = _rows(results, "naive", "collateral_approve")[0] + governed = _rows(results, "governed", "collateral_approve")[0] + assert naive["silent_wrong"] is True + assert "adjacent_invoice_modified" in naive["gt_violations"] + assert governed["caught"] is True + assert governed["silent_wrong"] is False + assert governed["transaction_outcome"] == "RECONCILIATION_REQUIRED" + + +def test_uncertain_payment_routes_to_reconciliation_and_never_double_pays(results): + governed = _rows(results, "governed", "payment_confirm_outage")[0] + assert governed["transaction_outcome"] == "RECONCILIATION_REQUIRED" + assert governed["halted"] is True + # The retry under the same idempotency key was suppressed, not re-actuated. + assert governed["retry_suppressed"] is True + assert governed["retry_transaction_outcome"] == "REJECTED_POLICY" + # Ground truth: exactly one payment landed; no duplicate. + assert governed["gt_correct"] is True, governed["gt_violations"] + + +def test_headline_silent_wrong_counts(results): + per_arm = results["metrics"]["per_arm"] + assert per_arm["governed"]["silent_wrong"] == 0 + assert per_arm["governed"]["over_halts"] == 0 + assert per_arm["naive"]["silent_wrong"] >= 1 diff --git a/tests/test_o2c_recon_benchmark.py b/tests/test_o2c_recon_benchmark.py new file mode 100644 index 00000000..4b6a209f --- /dev/null +++ b/tests/test_o2c_recon_benchmark.py @@ -0,0 +1,102 @@ +"""CI guard for the O2C reconciliation multi-system benchmark. + +Runs the REAL harness once (n=1; deterministic, localhost only, zero model +calls) and pins the qualitative claims the committed +``benchmark/o2c_recon/results.json`` publishes: + +- the healthy path spans TWO separate fixture applications plus two + spreadsheet surfaces (exported worklist in; results sheet written back and + re-read from disk), 25+ executed actions, and is classified ``VERIFIED`` + under the governed standard profile with ZERO model calls; +- a billed order with no ledger entry routes to an explicit HALT terminal + (never auto-created); +- ambiguous duplicate ledger entries and a stale reconciliation snapshot are + refused at the UI gateway BEFORE anything is written; +- a phantom results-sheet write (acknowledged but never persisted) is CAUGHT + by the governed arm re-reading the file and silently accepted by the + banner-oracle arm; +- the independent ground truth (direct SQLite + CSV reads) agrees. +""" + +from __future__ import annotations + +import pytest + +from benchmark.o2c_recon.run import ARMS, SCENARIOS, run_benchmark + + +@pytest.fixture(scope="module") +def results() -> dict: + return run_benchmark(n=1, log=lambda _m: None) + + +def _rows(results: dict, arm: str, scenario: str) -> list[dict]: + return [r for r in results["runs"] if r["arm"] == arm and r["scenario"] == scenario] + + +def test_every_scenario_ran_under_both_arms(results): + for arm in ARMS: + for scenario in SCENARIOS: + assert _rows(results, arm, scenario), (arm, scenario) + + +def test_zero_model_calls_everywhere(results): + assert all(int(r["model_calls"] or 0) == 0 for r in results["runs"]) + + +def test_every_executed_step_used_the_api_tier(results): + for r in results["runs"]: + if r["executed_action_steps"]: + assert r["actuation_kinds"] == ["api"], r + + +def test_healthy_path_shape(results): + for arm in ARMS: + row = _rows(results, arm, "healthy")[0] + assert row["worklist_rows"] == 10 + assert 25 <= row["executed_action_steps"] <= 60 + assert row["reported_success"] is True + assert row["gt_correct"] is True + + +def test_governed_healthy_run_is_verified(results): + row = _rows(results, "governed", "healthy")[0] + assert row["transaction_outcome"] == "VERIFIED" + assert row["transaction_billable"] is True + + +def test_missing_ledger_entry_halts_at_the_explicit_terminal(results): + for arm in ARMS: + row = _rows(results, arm, "missing_in_ledger")[0] + assert row["halted"] is True + assert row["terminal_outcome"] == "halt" + assert row["gt_correct"] is True, row["gt_violations"] + # The row BEFORE the missing one was still processed safely. + assert row["executed_action_steps"] == 2 + + +@pytest.mark.parametrize("scenario", ["ambiguous_duplicate", "stale_snapshot"]) +def test_conflicts_are_refused_before_any_write(results, scenario): + for arm in ARMS: + row = _rows(results, arm, scenario)[0] + assert row["halted"] is True + assert row["gt_correct"] is True, row["gt_violations"] + assert row["transaction_outcome"] == "HALTED_BEFORE_EFFECT" + assert all(delta == 0 for delta in row["table_deltas"].values()), row + + +def test_phantom_writeback_is_caught_by_reading_the_file(results): + naive = _rows(results, "naive", "phantom_writeback")[0] + governed = _rows(results, "governed", "phantom_writeback")[0] + assert naive["silent_wrong"] is True + assert any(v.startswith("writeback_row_missing") for v in naive["gt_violations"]) + assert governed["halted"] is True + assert governed["silent_wrong"] is False + assert governed["gt_correct"] is True + + +def test_headline_silent_wrong_counts(results): + per_arm = results["metrics"]["per_arm"] + assert per_arm["governed"]["silent_wrong"] == 0 + assert per_arm["governed"]["over_halts"] == 0 + assert per_arm["naive"]["silent_wrong"] >= 1 From bacf8315bc109cbf57b03d07fb1d52a5e9ff9cee Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 26 Jul 2026 17:20:07 -0400 Subject: [PATCH 2/4] harden multi-system benchmark evidence --- benchmark/BENCHMARK.md | 16 ++- benchmark/ap_invoice/README.md | 24 ++-- benchmark/ap_invoice/fixtures.py | 42 ++++++- benchmark/ap_invoice/ground_truth.py | 164 ++++++++++++++++++++++++--- benchmark/ap_invoice/run.py | 26 +++-- benchmark/ap_invoice/workflow.py | 54 +++------ benchmark/multiapp-standard.yaml | 12 ++ benchmark/multiapp_common.py | 122 ++++++++++++++++---- benchmark/o2c_recon/README.md | 20 ++-- benchmark/o2c_recon/fixtures.py | 8 ++ benchmark/o2c_recon/ground_truth.py | 155 +++++++++++++++++++++++-- benchmark/o2c_recon/run.py | 26 +++-- benchmark/o2c_recon/workflow.py | 13 +-- tests/test_ap_invoice_benchmark.py | 64 ++++++++++- tests/test_o2c_recon_benchmark.py | 65 ++++++++++- 15 files changed, 681 insertions(+), 130 deletions(-) create mode 100644 benchmark/multiapp-standard.yaml diff --git a/benchmark/BENCHMARK.md b/benchmark/BENCHMARK.md index 6975477d..be4ef159 100644 --- a/benchmark/BENCHMARK.md +++ b/benchmark/BENCHMARK.md @@ -117,13 +117,17 @@ perception; see each README). | **`o2c_recon`** (new) | **26** | **2** (billing + ledger) | CSV worklist in, CSV results write-back (re-read), REST API, UI gateway, 2 SQLite systems of record | 3-way branch (match / adjust / missing) | 4: missing record (explicit halt terminal), ambiguous duplicate, stale snapshot (optimistic concurrency), phantom file write | Both new benchmarks run every consequential write through the real -`Replayer`'s api actuation tier under a sealed-bundle, standard-profile -governed authorization, verify every write out-of-band per surface (read-only -SQL, REST oracles, a maildir read, a CSV re-read), and are judged by an -independent direct-file-access ground truth. Zero model calls; healthy-path +`Replayer`'s api actuation tier after the real Standard-profile run gate admits +the sealed bundle and binds a single-use authorization to its exact inputs. +Every write is verified through a separate persisted-state read (read-only SQL, +REST oracles, a maildir read, or a CSV re-read) and judged by a direct-file +adjudicator whose expectations come from immutable source fixtures rather than +the prepared worklist. This is not an independent service/failure domain. Zero +model calls; healthy-path governed runs classify `VERIFIED` under the Section-3 transaction taxonomy. -Measured headline over both benchmarks (n=3 per cell): governed -silent-incorrect-success 0/30 runs, over-halts 0/30; naive banner-oracle +Measured headline over both benchmarks (n=3 per cell; 60 base runs): governed +silent-incorrect-success 0/30 governed runs and healthy-path over-halts 0/6; +naive banner-oracle silent-incorrect-success 6/30 (the collateral overwrite and the phantom write-back classes). Deterministic coverage matrix, not a sampled incidence rate. See `benchmark/ap_invoice/README.md` diff --git a/benchmark/ap_invoice/README.md b/benchmark/ap_invoice/README.md index 0d98a6bd..cdeb6b5f 100644 --- a/benchmark/ap_invoice/README.md +++ b/benchmark/ap_invoice/README.md @@ -37,9 +37,10 @@ scenarios. - `naive`: demo profile; every write is "verified" only against the application's own painted acknowledgement banner (what a screen-echo automation trusts). -- `governed`: sealed bundle, single-use standard-profile authorization, an - exact API identity contract on every consequential write, out-of-band - effect verification routed per record surface (read-only SQL over the ERP +- `governed`: sealed bundle admitted by the real Standard-profile run gate, + with the resulting single-use authorization bound to the exact inputs; an + exact API identity contract on every consequential write; effect + verification routed per record surface (read-only SQL over the ERP SQLite file, a REST payments oracle, and the OUTBOX **maildir read from disk** for every sent email), a collateral guard on the adjacent grid row, and an at-most-once idempotency ledger. @@ -48,18 +49,21 @@ scenarios. | scenario | naive (banner oracle) | governed | |---|---|---| -| `healthy` | completes; `COMPLETED_UNVERIFIED` (never billable) | **`VERIFIED`** (all effects confirmed out-of-band; billable) | +| `healthy` | completes; `COMPLETED_UNVERIFIED` (never billable) | **`VERIFIED`** (all effects confirmed through separate persisted-state reads; billable) | | `missing_po` | safe halt at entry | safe halt; `HALTED_BEFORE_EFFECT`; nothing persisted | | `duplicate_invoice` | safe halt at entry | safe halt; `HALTED_BEFORE_EFFECT`; still exactly one invoice | | `collateral_approve` | **SILENT WRONG** (adjacent invoice corrupted, banner says success) | **caught**: collateral guard refutes; `RECONCILIATION_REQUIRED` | | `payment_confirm_outage` | completes (cannot know the write landed) | `RECONCILIATION_REQUIRED`; retry under the same idempotency key SUPPRESSED (`REJECTED_POLICY`); ground truth: exactly one payment | -Headline (30 runs + 10 retry runs): governed silent-incorrect-success **0**, -over-halts on the healthy path **0**, model calls **0**; naive +Headline (30 base runs: 15 per arm, plus 6 duplicate-attempt checks): governed +silent-incorrect-success **0/15**, healthy-path over-halts **0/3**, model calls +**0**; naive silent-incorrect-success **3/3** on the collateral scenario. Every run is -judged by an independent ground truth that opens the SQLite file and the -maildir directly (no HTTP, no banner, no verifier verdict reaches it) and -audits every table's delta. +judged through a direct persisted-state read path that opens the SQLite file +and maildir (no HTTP, banner, or verifier verdict reaches it), derives +expectations from immutable source-fixture bytes, and enforces the allowed +record transitions across every non-echo table. This is not a separate service +or failure domain. Reproduce: `python -m benchmark.ap_invoice.run --n 3` (localhost only, $0). Pinned in CI by `tests/test_ap_invoice_benchmark.py`. @@ -71,7 +75,7 @@ Proves (within a synthetic closed world): - the engine's Phase-2 workflow-program machinery (loop + guarded branches + explicit exception routing) executes a 30+ step, two-application, email/PDF/spreadsheet-era back-office flow deterministically at $0; -- the governed contract stack (identity bindings, per-surface out-of-band +- the governed contract stack (identity bindings, per-surface persisted-state effect verification including a maildir file oracle, collateral guards, idempotency, the Section-3 transaction taxonomy) yields zero silent-incorrect-successes across the designed fault classes while a diff --git a/benchmark/ap_invoice/fixtures.py b/benchmark/ap_invoice/fixtures.py index 130e9fa4..c697491d 100644 --- a/benchmark/ap_invoice/fixtures.py +++ b/benchmark/ap_invoice/fixtures.py @@ -39,7 +39,7 @@ from typing import Any, Callable, Optional from urllib.parse import urlparse -from benchmark.multiapp_common import init_maildir, write_maildir_message +from benchmark.multiapp_common import init_maildir, make_pdf, write_maildir_message #: The seeded ADJACENT record every scenario carries: a draft invoice sitting #: one row away from the target in the AP grid. The collateral fault corrupts @@ -51,6 +51,46 @@ #: The pre-existing posted invoice the ambiguous-duplicate scenario re-presents. DUPLICATE_INVOICE = "INV-1201" +# Immutable source fixtures. The workflow intake parser and the direct +# persisted-state adjudicator consume these independently; the adjudicator +# never trusts the worklist produced by the system under test. +INVOICE_SOURCE_SEEDS: dict[str, tuple[str, str, str, str, str]] = { + "INV-1001": ("V-100", "Acme Supply Co", "PO-501", "1200.00", "2/10 NET 30"), + "INV-1002": ("V-200", "Beta Industrial", "PO-502", "860.00", "NET 30"), + "INV-1003": ("V-300", "Gamma Logistics", "PO-503", "415.25", "NET 30"), + "INV-1004": ("V-100", "Acme Supply Co", "PO-504", "99.90", "2/10 NET 30"), + "INV-1005": ("V-200", "Beta Industrial", "PO-505", "729.00", "NET 30"), + "INV-1101": ("V-100", "Acme Supply Co", "PO-599", "55.00", "NET 30"), + "INV-1201": ("V-200", "Beta Industrial", "PO-505", "729.00", "NET 30"), + "INV-1301": ("V-100", "Acme Supply Co", "PO-506", "320.00", "NET 30"), + "INV-1401": ("V-300", "Gamma Logistics", "PO-507", "212.75", "NET 30"), +} + +SCENARIO_INVOICE_IDS: dict[str, tuple[str, ...]] = { + "healthy": ("INV-1001", "INV-1002", "INV-1003", "INV-1004", "INV-1005"), + "missing_po": ("INV-1101",), + "duplicate_invoice": ("INV-1201",), + "collateral_approve": ("INV-1301",), + "payment_confirm_outage": ("INV-1401",), +} + + +def invoice_source_pdf(invoice_id: str) -> bytes: + """The immutable PDF bytes delivered to the intake pipeline.""" + vendor_id, vendor_name, po_number, amount, terms = INVOICE_SOURCE_SEEDS[invoice_id] + return make_pdf( + [ + f"INVOICE: {invoice_id}", + f"VENDOR: {vendor_id}", + f"VENDOR-NAME: {vendor_name}", + f"PO: {po_number}", + f"AMOUNT: {amount}", + f"TERMS: {terms}", + "CURRENCY: USD", + ] + ) + + VENDORS = ( ("V-100", "Acme Supply Co"), ("V-200", "Beta Industrial"), diff --git a/benchmark/ap_invoice/ground_truth.py b/benchmark/ap_invoice/ground_truth.py index b13cc339..f7dab88d 100644 --- a/benchmark/ap_invoice/ground_truth.py +++ b/benchmark/ap_invoice/ground_truth.py @@ -1,12 +1,9 @@ -"""Independent ground truth for the AP invoice benchmark. - -Bypasses BOTH fixture services entirely: opens the ERP SQLite file directly -(read-only, its own connection) and lists/reads the OUTBOX maildir directly -from disk. No HTTP status, banner row, or verifier verdict ever reaches this -module; it re-derives the intended end state from the worklist rows with its -OWN logic and reports INVARIANT VIOLATIONS (wrong business state), plus a -full per-table delta audit so an unexpected write anywhere in the database is -caught even when every targeted check passes. +"""Direct persisted-state adjudicator for the AP invoice benchmark. + +It bypasses both fixture services: a separate read-only SQLite connection and +direct OUTBOX maildir reads. It derives expected business state from immutable +source-fixture bytes, never from the worklist produced by the intake pre-pass. +This is a separate read path, not an independent service or failure domain. """ from __future__ import annotations @@ -17,12 +14,32 @@ from pathlib import Path from typing import Any -from benchmark.ap_invoice.fixtures import ADJACENT_AMOUNT, ADJACENT_INVOICE -from benchmark.multiapp_common import read_maildir_messages +from benchmark.ap_invoice.fixtures import ( + ADJACENT_AMOUNT, + ADJACENT_INVOICE, + INVOICE_SOURCE_SEEDS, + PURCHASE_ORDERS, + SCENARIO_INVOICE_IDS, + invoice_source_pdf, +) +from benchmark.multiapp_common import ( + read_maildir_messages, + unexpected_record_change_violations, +) #: The app's own echo surface; excluded from the delta audit on purpose (it is #: the thing a screen-echo oracle reads, not part of the business state). _ECHO_TABLES = frozenset({"banner"}) +_TABLE_KEYS = { + "vendors": "vendor_id", + "purchase_orders": "po_number", + "receipts": "id", + "invoices": "invoice_id", + "payments": "invoice_id", + "ap_exceptions": "invoice_id", + "batches": "batch_id", +} +_PO_AMOUNTS = {po[0]: po[5] for po in PURCHASE_ORDERS} @dataclass @@ -83,9 +100,46 @@ def _mail_for(snapshot: Snapshot, outbox_dir: Path, name: str, invoice_id: str) return False +def _expected_rows(scenario: str) -> list[dict[str, str]]: + """Expected intake facts derived independently from immutable PDF seeds.""" + expected: list[dict[str, str]] = [] + for invoice_id in SCENARIO_INVOICE_IDS[scenario]: + vendor_id, _vendor_name, po_number, amount, terms = INVOICE_SOURCE_SEEDS[ + invoice_id + ] + po_amount = _PO_AMOUNTS.get(po_number) + route = "ok" if po_amount is None or po_amount == amount else "mismatch" + eligible = terms.startswith("2/10") and route == "ok" + expected.append( + { + "invoice_id": invoice_id, + "vendor_id": vendor_id, + "po_number": po_number, + "amount": amount, + "doc_sha256": hashlib.sha256( + invoice_source_pdf(invoice_id) + ).hexdigest(), + "route": route, + "discount_applied": "2/10" if eligible else "none", + "amount_payable": ( + f"{round(float(amount) * 0.98, 2):.2f}" if eligible else amount + ), + "mail_name": ( + f"confirm-{invoice_id}.eml" + if route == "ok" + else f"hold-{invoice_id}.eml" + ), + } + ) + return expected + + +def _non_id(row: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in row.items() if key != "id"} + + def judge( scenario: str, - rows: list[dict[str, str]], before: Snapshot, after: Snapshot, *, @@ -99,12 +153,40 @@ def judge( flow unfinished; it must never leave the state WRONG). """ verdict = Verdict() + rows = _expected_rows(scenario) verdict.table_deltas = { name: len(after.tables.get(name, [])) - len(before.tables.get(name, [])) for name in sorted(set(before.tables) | set(after.tables)) if name not in _ECHO_TABLES } + mutable_targets = ( + set() + if scenario in {"missing_po", "duplicate_invoice"} + else {row["invoice_id"] for row in rows} + ) + allowed = { + "invoices": mutable_targets, + "payments": mutable_targets, + "ap_exceptions": mutable_targets, + "batches": {"BATCH-2026-07"} if completed else set(), + } + verdict.violations.extend( + unexpected_record_change_violations( + before.tables, + after.tables, + key_fields=_TABLE_KEYS, + allowed_keys=allowed, + excluded_tables=_ECHO_TABLES, + ) + ) + allowed_mail = {row["mail_name"] for row in rows} if completed else set() + for name in sorted(set(before.outbox) | set(after.outbox)): + if name not in allowed_mail and before.outbox.get(name) != after.outbox.get( + name + ): + verdict.violations.append(f"unexpected_outbox_change:{name}") + # -- invariants that must hold in EVERY scenario and arm ----------------- adjacent = _rows(after, "invoices", invoice_id=ADJACENT_INVOICE) if len(adjacent) != 1: @@ -138,6 +220,56 @@ def judge( if row["route"] == "mismatch" and payments: verdict.violations.append(f"mismatched_invoice_paid:{row['invoice_id']}") + # Every scenario that clears entry has one deterministic target record. + if scenario not in {"missing_po", "duplicate_invoice"}: + expected_invoice = { + "invoice_id": row["invoice_id"], + "vendor_id": row["vendor_id"], + "po_number": row["po_number"], + "amount": row["amount"], + "doc_sha256": row["doc_sha256"], + "status": "held" if row["route"] == "mismatch" else "approved", + "discount_applied": row["discount_applied"], + "amount_payable": ( + "" if row["route"] == "mismatch" else row["amount_payable"] + ), + } + if len(invoices) != 1 or _non_id(invoices[0]) != expected_invoice: + verdict.violations.append( + f"target_invoice_transition_wrong:{row['invoice_id']}" + ) + + payment_expected = row["route"] == "ok" and ( + completed or scenario == "payment_confirm_outage" + ) + expected_payments = ( + [ + { + "invoice_id": row["invoice_id"], + "amount": row["amount_payable"], + "status": "scheduled", + } + ] + if payment_expected + else [] + ) + if [_non_id(payment) for payment in payments] != expected_payments: + verdict.violations.append(f"payment_transition_wrong:{row['invoice_id']}") + + exceptions = _rows(after, "ap_exceptions", invoice_id=row["invoice_id"]) + expected_exceptions = ( + [ + { + "invoice_id": row["invoice_id"], + "reason": f"price mismatch vs {row['po_number']}", + } + ] + if completed and row["route"] == "mismatch" + else [] + ) + if [_non_id(exception) for exception in exceptions] != expected_exceptions: + verdict.violations.append(f"exception_transition_wrong:{row['invoice_id']}") + # -- completeness invariants: only a COMPLETED run owes the end state ---- if completed: for row in rows: @@ -164,7 +296,13 @@ def judge( if not _mail_for(after, outbox_dir, row["mail_name"], invoice_id): verdict.violations.append(f"vendor_email_missing:{invoice_id}") batches = after.tables.get("batches", []) - if len(batches) != 1 or str(batches[0].get("processed")) != str(len(rows)): + expected_batch = { + "batch_id": "BATCH-2026-07", + "processed": str(len(rows)), + } + if len(batches) != 1 or _non_id(batches[0]) != expected_batch: verdict.violations.append("batch_completion_wrong") + elif after.tables.get("batches", []): + verdict.violations.append("batch_written_after_halt") return verdict diff --git a/benchmark/ap_invoice/run.py b/benchmark/ap_invoice/run.py index 0dd5d7b0..b553476a 100644 --- a/benchmark/ap_invoice/run.py +++ b/benchmark/ap_invoice/run.py @@ -7,13 +7,14 @@ - ``naive`` -- demo profile, no governed authorization, and effect contracts that read only the applications' OWN painted acknowledgement banners (what a screen-echo automation trusts). -- ``governed`` -- sealed bundle + single-use standard-profile authorization, - exact API identity contracts on every consequential write, and out-of-band +- ``governed`` -- sealed bundle admitted by the real Standard-profile gate, + a single-use exact-input authorization, exact API identity contracts on + every consequential write, and separate persisted-state effect verification routed per record surface (read-only SQL over the ERP file, the payments REST oracle, and the OUTBOX maildir file oracle), plus the adjacent-record collateral guard and at-most-once idempotency. -Every run is judged by the INDEPENDENT ground truth +Every run is judged by the direct persisted-state adjudicator (:mod:`benchmark.ap_invoice.ground_truth`): direct SQLite file reads plus a direct maildir read, with its own invariant logic and per-table delta audit. @@ -51,7 +52,6 @@ from benchmark.ap_invoice.workflow import ( build_workflow, build_worklist, - required_identity_step_ids, scenario_faults, scenario_invoices, scenario_seed_duplicate, @@ -164,19 +164,23 @@ def _one_replay( workflow_src.save(bundle_dir) workflow = Workflow.load(bundle_dir) + verifier = _build_verifier(arm, erp, mailer) + api_actuator = ApiActuator(erp.base_url, timeout_s=5.0) authorization = None if arm == "governed": authorization = standard_authorization( workflow, + bundle_dir=bundle_dir, + effect_verifier=verifier, + api_actuator=api_actuator, params=None, worklists=worklists, - required_identity_step_ids=required_identity_step_ids(workflow), ) replayer = Replayer( NullBackend(), vision=NullVision(), - effect_verifier=_build_verifier(arm, erp, mailer), - api_actuator=ApiActuator(erp.base_url, timeout_s=5.0), + effect_verifier=verifier, + api_actuator=api_actuator, governed_authorization=authorization, durable=(arm == "governed"), require_settled=(arm == "governed"), @@ -258,7 +262,6 @@ def run_one(scenario: str, arm: str, index: int, work_root: Path) -> dict[str, A after = ground_truth.capture(erp.db_path, mailer.outbox) verdict = ground_truth.judge( scenario, - rows, before, after, outbox_dir=mailer.outbox, @@ -279,6 +282,13 @@ def run_one(scenario: str, arm: str, index: int, work_root: Path) -> dict[str, A "actuation_kinds": sorted({str(r.actuation) for r in executed}), "reported_success": reported_success, "halted": halted, + "execution_profile": report.execution_profile, + "governed_policy_name": ( + Path(report.governed_policy_name).name + if report.governed_policy_name + else None + ), + "governed_approval_source": report.governed_approval_source, "execution_outcome": report.execution_outcome, "transaction_outcome": report.transaction_outcome, "transaction_billable": report.transaction_billable, diff --git a/benchmark/ap_invoice/workflow.py b/benchmark/ap_invoice/workflow.py index 571eb905..5a7f118f 100644 --- a/benchmark/ap_invoice/workflow.py +++ b/benchmark/ap_invoice/workflow.py @@ -12,7 +12,7 @@ invoice worklist whose body BRANCHES on the match route (post vs. hold) and on discount eligibility, with every consequential write carried by an ``ApiBinding`` (the api tier), an exact API identity contract, and typed -system-of-record effect contracts routed to per-surface out-of-band oracles. +system-of-record effect contracts routed to per-surface persisted-state reads. """ from __future__ import annotations @@ -23,10 +23,14 @@ import requests +from benchmark.ap_invoice.fixtures import ( + INVOICE_SOURCE_SEEDS, + SCENARIO_INVOICE_IDS, + invoice_source_pdf, +) from benchmark.multiapp_common import ( build_request_email, extract_pdf_lines, - make_pdf, parse_email, parse_kv_lines, read_maildir_messages, @@ -51,44 +55,19 @@ EFFECT_TIMEOUT_S = 0.25 BATCH_ID = "BATCH-2026-07" + #: invoice_id -> (vendor_id, vendor_name, po_number, amount, terms) #: ``2/10 NET 30`` terms are discount-eligible at batch time; ``NET 30`` is #: expired. INV-1003's amount deliberately disagrees with PO-503 (the price #: mismatch routed to the exception queue). INV-1101 references a PO that does #: not exist; INV-1201 re-presents an already-posted invoice number. -INVOICE_SEEDS: dict[str, tuple[str, str, str, str, str]] = { - "INV-1001": ("V-100", "Acme Supply Co", "PO-501", "1200.00", "2/10 NET 30"), - "INV-1002": ("V-200", "Beta Industrial", "PO-502", "860.00", "NET 30"), - "INV-1003": ("V-300", "Gamma Logistics", "PO-503", "415.25", "NET 30"), - "INV-1004": ("V-100", "Acme Supply Co", "PO-504", "99.90", "2/10 NET 30"), - "INV-1005": ("V-200", "Beta Industrial", "PO-505", "729.00", "NET 30"), - "INV-1101": ("V-100", "Acme Supply Co", "PO-599", "55.00", "NET 30"), - "INV-1201": ("V-200", "Beta Industrial", "PO-505", "729.00", "NET 30"), - "INV-1301": ("V-100", "Acme Supply Co", "PO-506", "320.00", "NET 30"), - "INV-1401": ("V-300", "Gamma Logistics", "PO-507", "212.75", "NET 30"), -} - - -def invoice_pdf(invoice_id: str) -> bytes: - vendor_id, vendor_name, po_number, amount, terms = INVOICE_SEEDS[invoice_id] - return make_pdf( - [ - f"INVOICE: {invoice_id}", - f"VENDOR: {vendor_id}", - f"VENDOR-NAME: {vendor_name}", - f"PO: {po_number}", - f"AMOUNT: {amount}", - f"TERMS: {terms}", - "CURRENCY: USD", - ] - ) - - def seed_inbox(inbox: Path, invoice_ids: list[str]) -> None: """Deliver one deterministic vendor request email per invoice.""" for invoice_id in invoice_ids: - vendor_id, vendor_name, po_number, amount, _terms = INVOICE_SEEDS[invoice_id] - pdf = invoice_pdf(invoice_id) + vendor_id, vendor_name, po_number, amount, _terms = INVOICE_SOURCE_SEEDS[ + invoice_id + ] + pdf = invoice_source_pdf(invoice_id) message = build_request_email( from_addr=f"billing@{vendor_id.lower()}.example.test", to_addr="ap@example-corp.test", @@ -213,7 +192,7 @@ def _inv_match() -> dict[str, ValueExpr]: def _steps(arm: str, mailer_base: str, adjacent_invoice: str) -> dict[str, Step]: """The ten hardened action leaves, keyed by step id. - ``arm`` selects the effect contracts: ``governed`` carries out-of-band + ``arm`` selects the effect contracts: ``governed`` carries persisted-state system-of-record contracts (SQL / REST / maildir surfaces, plus the adjacent-record collateral guard); ``naive`` carries only the app-painted banner acknowledgement (what a screen-echo automation trusts). @@ -400,6 +379,7 @@ def step( action=ActionKind.KEY, key="Enter", risk="irreversible", + identity_armed=governed, effects=effects(step_id), api_binding=binding, ) @@ -715,13 +695,7 @@ def required_identity_step_ids(workflow: Workflow) -> tuple[str, ...]: def scenario_invoices(scenario: str) -> list[str]: """Which seeded request emails a scenario's inbox carries.""" - return { - "healthy": ["INV-1001", "INV-1002", "INV-1003", "INV-1004", "INV-1005"], - "missing_po": ["INV-1101"], - "duplicate_invoice": ["INV-1201"], - "collateral_approve": ["INV-1301"], - "payment_confirm_outage": ["INV-1401"], - }[scenario] + return list(SCENARIO_INVOICE_IDS[scenario]) def scenario_faults(scenario: str) -> list[str]: diff --git a/benchmark/multiapp-standard.yaml b/benchmark/multiapp-standard.yaml new file mode 100644 index 00000000..34322018 --- /dev/null +++ b/benchmark/multiapp-standard.yaml @@ -0,0 +1,12 @@ +name: benchmark-multiapp-standard +description: >- + Certification policy for the deterministic multi-application API-tier + benchmarks. Policy certification requires a confirmed system-effect + contract on every consequential write. Standard-profile admission separately + enforces identity coverage, and the runtime proves each exact API + request/effect binding before dispatch. + +require_system_effects_for: + - write + +prohibit_unconfirmed_effect_bindings: true diff --git a/benchmark/multiapp_common.py b/benchmark/multiapp_common.py index a38a8818..aae2498d 100644 --- a/benchmark/multiapp_common.py +++ b/benchmark/multiapp_common.py @@ -5,7 +5,7 @@ the API-tier replay path (the same pattern ``benchmark/effect_e2e`` uses), a surface-routing composite effect verifier, a CSV row oracle for spreadsheet write-back verification, deterministic RFC822 email + minimal PDF fixtures, -and the governed standard-profile authorization helper. +and the governed Standard-profile admission helper. No model calls, no network beyond 127.0.0.1, no PHI (all names are synthetic company/worker placeholders). @@ -14,6 +14,7 @@ from __future__ import annotations import csv +import json import struct import time import zlib @@ -24,11 +25,11 @@ from types import SimpleNamespace from typing import Any, Optional +from openadapt_flow.deployment import DeploymentConfig, PolicySection +from openadapt_flow.execution_profiles import execution_profile_contract from openadapt_flow.ir import Workflow -from openadapt_flow.runtime.authorization import ( - GovernedRunAuthorization, - runtime_inputs_digest, -) +from openadapt_flow.run_gate import build_runtime_authorization, evaluate_run_gate +from openadapt_flow.runtime.authorization import GovernedRunAuthorization from openadapt_flow.runtime.effects._common import judge_records from openadapt_flow.runtime.effects.effect import ( Effect, @@ -142,7 +143,7 @@ def surface_of(effect: Effect, default: str) -> str: class SurfaceRoutedVerifier: - """Route each typed effect to the out-of-band verifier for its surface. + """Route each typed effect to the separate verifier for its surface. Generalizes ``benchmark.effect_e2e.verifiers.CompositeSqlVerifier`` to a heterogeneous verifier map (read-only SQL, REST record oracle, file/maildir @@ -406,28 +407,111 @@ def parse_kv_lines(lines: list[str]) -> dict[str, str]: return out +# -- direct persisted-state adjudication ------------------------------------ + + +def unexpected_record_change_violations( + before: dict[str, list[dict[str, Any]]], + after: dict[str, list[dict[str, Any]]], + *, + key_fields: dict[str, str], + allowed_keys: dict[str, set[str]], + excluded_tables: frozenset[str] = frozenset(), + prefix: str = "", +) -> list[str]: + """Return changes outside an explicit table/key transition allowlist. + + Every non-excluded table is covered. Known business tables are grouped by + their stable record key so inserts, deletes, and in-place edits to an + unrelated record are caught. An unrecognized table is fail-closed: its + complete canonical row set must remain byte-for-byte equivalent. + + Values for allowed records are still checked by each benchmark's separate + business-state oracle; this helper ensures that *nothing else* changed. + """ + + def canonical(rows: list[dict[str, Any]]) -> list[str]: + return sorted( + json.dumps(row, sort_keys=True, separators=(",", ":"), default=str) + for row in rows + ) + + def grouped( + rows: list[dict[str, Any]], key_field: str + ) -> dict[str, list[dict[str, Any]]]: + result: dict[str, list[dict[str, Any]]] = {} + for row in rows: + result.setdefault(str(row.get(key_field)), []).append(row) + return result + + violations: list[str] = [] + for table in sorted(set(before) | set(after)): + if table in excluded_tables: + continue + label = f"{prefix}.{table}" if prefix else table + before_rows = before.get(table, []) + after_rows = after.get(table, []) + key_field = key_fields.get(table) + if key_field is None: + if canonical(before_rows) != canonical(after_rows): + violations.append(f"unexpected_table_change:{label}") + continue + before_by_key = grouped(before_rows, key_field) + after_by_key = grouped(after_rows, key_field) + permitted = allowed_keys.get(table, set()) + for key in sorted(set(before_by_key) | set(after_by_key)): + if key in permitted: + continue + if canonical(before_by_key.get(key, [])) != canonical( + after_by_key.get(key, []) + ): + violations.append(f"unexpected_record_change:{label}:{key}") + return violations + + # -- governed authorization helper ------------------------------------------- def standard_authorization( workflow: Workflow, *, + bundle_dir: Path, + effect_verifier: object, + api_actuator: object, params: Optional[dict[str, str]], worklists: Optional[dict[str, list[dict[str, str]]]], - required_identity_step_ids: tuple[str, ...], - policy_name: str = "benchmark-multiapp-standard", ) -> GovernedRunAuthorization: - """A single-use standard-profile authorization bound to this exact run. - - The bundle must be sealed (``Workflow.save`` + ``Workflow.load``) before - calling. The digest binds the exact params and worklists supplied to - ``Replayer.run``; any drift refuses execution. + """Admit and bind one run through the real Standard-profile gate. + + The benchmark policy is intentionally scoped to deterministic API-tier + fixtures: every consequential write must carry an exact API identity + binding and a declared system-effect contract. ``evaluate_run_gate`` then + applies the named Standard profile (certification, identity/effect + coverage, verifier tier, durability, settling, and manifest integrity). + ``build_runtime_authorization`` binds that admission to the exact sealed + bundle and runtime inputs; any later drift is refused by the replayer. """ assert workflow.manifest is not None, "seal the bundle before authorizing" - return GovernedRunAuthorization( - bundle_content_digest=workflow.manifest.content_digest, - runtime_inputs_digest=runtime_inputs_digest(workflow, params, worklists), - admitted_policy_name=policy_name, - execution_profile="standard", - required_identity_step_ids=required_identity_step_ids, + policy = Path(__file__).with_name("multiapp-standard.yaml") + gate = evaluate_run_gate( + workflow, + bundle_dir=bundle_dir, + deployment=DeploymentConfig( + policy=PolicySection(policy=str(policy)), + ), + effect_verifier=effect_verifier, + api_actuator=api_actuator, + policy_source=str(policy), + profile_contract=execution_profile_contract("standard"), + effective_durable=True, + effective_require_settled=True, + ) + if not gate.passed: + raise RuntimeError(gate.render()) + return build_runtime_authorization( + workflow, + gate, + approval_source="benchmark-standard-run-gate", + params=params, + worklists=worklists, ) diff --git a/benchmark/o2c_recon/README.md b/benchmark/o2c_recon/README.md index ccb1eaa5..1773a4a8 100644 --- a/benchmark/o2c_recon/README.md +++ b/benchmark/o2c_recon/README.md @@ -37,9 +37,10 @@ plus 4 designed exception scenarios. - `naive`: demo profile; every write "verified" only against the applications' own painted acknowledgement banners. -- `governed`: sealed bundle, single-use standard-profile authorization, exact - API identity contracts (order id plus customer name quorum on the - adjustment write), out-of-band verification per surface (read-only SQL +- `governed`: sealed bundle admitted by the real Standard-profile run gate, + with the resulting single-use authorization bound to the exact inputs; + exact API identity contracts (order id plus customer name quorum on the + adjustment write); persisted-state verification per surface (read-only SQL over the ledger SQLite file; the results CSV re-read from disk). ## Scenarios and measured outcomes (n=3 per cell, deterministic) @@ -52,11 +53,14 @@ plus 4 designed exception scenarios. | `stale_snapshot` | safe halt (optimistic-concurrency 409) | safe halt; `HALTED_BEFORE_EFFECT`; amount unchanged | | `phantom_writeback` | **SILENT WRONG** (row acknowledged, never written to the sheet) | **caught** by re-reading the file; halts | -Headline (30 runs): governed silent-incorrect-success **0**, over-halts on -the healthy path **0**, model calls **0**; naive silent-incorrect-success +Headline (30 base runs: 15 per arm): governed silent-incorrect-success +**0/15**, healthy-path over-halts **0/3**, model calls **0**; naive +silent-incorrect-success **3/3** on the phantom write-back. Ground truth opens both SQLite files and -both CSV files directly and re-derives the intended state from the seed -table; it also verifies the exported spreadsheet was never mutated. +both CSV files through a separate read path, derives expectations from the +immutable source seeds rather than the compare worklist, enforces allowed +record transitions across every non-echo table, and verifies the export was +never mutated. It is not a separate service or failure domain. The compare pre-pass is deliberately NAIVE (first ledger entry wins, snapshot trusted): the measured property is that the ENGINE still refuses at act time @@ -71,7 +75,7 @@ Proves (within a synthetic closed world): - a genuinely multi-application flow: one workflow program driving two separate fixture applications and two file surfaces, with per-surface - out-of-band verification including a spreadsheet re-read oracle; + persisted-state verification including a spreadsheet re-read oracle; - conflict and duplicate handling refuse BEFORE any write (optimistic concurrency, ambiguity, missing record), and the phantom-file-write class is caught only by an oracle that actually re-reads the file; diff --git a/benchmark/o2c_recon/fixtures.py b/benchmark/o2c_recon/fixtures.py index 8b744adb..81ae634a 100644 --- a/benchmark/o2c_recon/fixtures.py +++ b/benchmark/o2c_recon/fixtures.py @@ -58,6 +58,14 @@ ("ORD-9401", "Northwind Retail", "212.00", "212.00", "2026-06"), ) +SCENARIO_ORDER_IDS: dict[str, tuple[str, ...]] = { + "healthy": tuple(f"ORD-90{i:02d}" for i in range(1, 11)), + "missing_in_ledger": ("ORD-9001", "ORD-9101"), + "ambiguous_duplicate": ("ORD-9201",), + "stale_snapshot": ("ORD-9301",), + "phantom_writeback": ("ORD-9401",), +} + STALE_ORDER = "ORD-9301" STALE_REPORTED_AMOUNT = "480.00" DUPLICATE_ORDER = "ORD-9201" diff --git a/benchmark/o2c_recon/ground_truth.py b/benchmark/o2c_recon/ground_truth.py index 70f6a78c..05450a1f 100644 --- a/benchmark/o2c_recon/ground_truth.py +++ b/benchmark/o2c_recon/ground_truth.py @@ -1,10 +1,9 @@ -"""Independent ground truth for the O2C reconciliation benchmark. +"""Direct persisted-state adjudicator for the O2C reconciliation benchmark. -Bypasses both fixture services: opens the two SQLite files directly -(read-only, own connections), re-reads the exported and written-back -spreadsheets from disk, and re-derives the intended end state from the -authoritative seed table with its OWN logic. No HTTP status, banner row, or -verifier verdict reaches this module. +It bypasses both fixture services: separate read-only SQLite connections and +direct spreadsheet reads. Expectations come from immutable billing/ledger +fixture seeds rather than the compare pre-pass worklist. This is a separate +read path, not an independent service or failure domain. """ from __future__ import annotations @@ -16,9 +15,19 @@ from pathlib import Path from typing import Any, Optional -from benchmark.o2c_recon.fixtures import ORDER_SEEDS +from benchmark.multiapp_common import unexpected_record_change_violations +from benchmark.o2c_recon.fixtures import ( + ORDER_SEEDS, + SCENARIO_ORDER_IDS, + STALE_REPORTED_AMOUNT, +) _ECHO_TABLES = frozenset({"banner"}) +_BILLING_KEYS = {"billed_orders": "order_id"} +_LEDGER_KEYS = { + "ledger_entries": "order_id", + "adjustments": "order_id", +} #: order_id -> (customer, amount_billed, seeded amount_posted or None) _SEEDS: dict[str, tuple[str, str, Optional[str]]] = { @@ -83,15 +92,50 @@ def capture( ) +def _source_rows(scenario: str) -> list[dict[str, str]]: + """Expected compare inputs derived from immutable source fixtures.""" + rows: list[dict[str, str]] = [] + for order_id in SCENARIO_ORDER_IDS[scenario]: + customer, billed, seeded_posted = _SEEDS[order_id] + observed = ( + STALE_REPORTED_AMOUNT + if scenario == "stale_snapshot" and order_id == "ORD-9301" + else seeded_posted + ) + if observed is None: + disposition, prior, delta = "missing", "", "" + elif abs(float(billed) - float(observed)) < 0.005: + disposition, prior, delta = "match", observed, "" + else: + disposition, prior = "adjust", observed + delta = f"{float(billed) - float(observed):+.2f}" + rows.append( + { + "order_id": order_id, + "customer": customer, + "amount_billed": billed, + "amount_prior": prior, + "delta": delta, + "disposition": disposition, + "reason": "billing reconciliation 2026-06", + } + ) + return rows + + +def _non_id(row: dict[str, Any]) -> dict[str, Any]: + return {key: value for key, value in row.items() if key != "id"} + + def judge( scenario: str, - rows: list[dict[str, str]], before: Snapshot, after: Snapshot, *, completed: bool, ) -> Verdict: verdict = Verdict() + rows = _source_rows(scenario) for label, tables_before, tables_after in ( ("billing", before.billing_tables, after.billing_tables), ("ledger", before.ledger_tables, after.ledger_tables), @@ -103,6 +147,37 @@ def judge( tables_after.get(name, []) ) - len(tables_before.get(name, [])) + mutable_order_ids = { + "healthy": {row["order_id"] for row in rows}, + "missing_in_ledger": {"ORD-9001"}, + "ambiguous_duplicate": set(), + "stale_snapshot": set(), + "phantom_writeback": {"ORD-9401"}, + }[scenario] + verdict.violations.extend( + unexpected_record_change_violations( + before.billing_tables, + after.billing_tables, + key_fields=_BILLING_KEYS, + allowed_keys={}, + excluded_tables=_ECHO_TABLES, + prefix="billing", + ) + ) + verdict.violations.extend( + unexpected_record_change_violations( + before.ledger_tables, + after.ledger_tables, + key_fields=_LEDGER_KEYS, + allowed_keys={ + "ledger_entries": mutable_order_ids, + "adjustments": mutable_order_ids, + }, + excluded_tables=_ECHO_TABLES, + prefix="ledger", + ) + ) + if after.export_sha256 != before.export_sha256: verdict.violations.append("export_spreadsheet_mutated") @@ -113,6 +188,43 @@ def judge( def entries_for(order_id: str) -> list[dict[str, Any]]: return [e for e in ledger if str(e.get("order_id")) == order_id] + source_by_id = {row["order_id"]: row for row in rows} + + # Exact target transitions for each record the scenario is allowed to + # mutate. Generated SQLite ids are intentionally excluded; all business + # fields and cardinality are exact. + for order_id in sorted(mutable_order_ids): + source = source_by_id[order_id] + entries = entries_for(order_id) + expected_entry = { + "order_id": order_id, + "customer": source["customer"], + "amount_posted": source["amount_billed"], + "status": "reconciled", + } + if len(entries) != 1 or _non_id(entries[0]) != expected_entry: + verdict.violations.append(f"target_ledger_transition_wrong:{order_id}") + + order_adjustments = [ + adjustment + for adjustment in adjustments + if str(adjustment.get("order_id")) == order_id + ] + expected_adjustments = ( + [ + { + "order_id": order_id, + "delta": source["delta"], + "reason": source["reason"], + "status": "applied", + } + ] + if source["disposition"] == "adjust" + else [] + ) + if [_non_id(row) for row in order_adjustments] != expected_adjustments: + verdict.violations.append(f"adjustment_transition_wrong:{order_id}") + # -- invariants that hold in every scenario and arm ---------------------- for order_id, (_customer, billed, seeded_posted) in _SEEDS.items(): entries = entries_for(order_id) @@ -149,13 +261,40 @@ def entries_for(order_id: str) -> list[dict[str, Any]]: # A result row is a CLAIM: it must describe a genuinely reconciled order. seen_result_orders: set[str] = set() + result_allowed = ( + set(mutable_order_ids) | {"SUMMARY"} + if scenario == "healthy" + else ({"ORD-9001"} if scenario == "missing_in_ledger" else set()) + ) for row in results: order_id = str(row.get("order_id")) if order_id in seen_result_orders: verdict.violations.append(f"duplicate_result_row:{order_id}") seen_result_orders.add(order_id) if order_id == "SUMMARY": + if order_id not in result_allowed: + verdict.violations.append("unexpected_result_row:SUMMARY") + continue + expected_summary = { + "order_id": "SUMMARY", + "disposition": "summary", + "delta": "", + "status": f"{len(rows)} orders processed", + } + if row != expected_summary: + verdict.violations.append("summary_row_wrong") + continue + if order_id not in result_allowed: + verdict.violations.append(f"unexpected_result_row:{order_id}") continue + source = source_by_id.get(order_id) + if source is None or row != { + "order_id": order_id, + "disposition": source["disposition"], + "delta": source["delta"], + "status": "done", + }: + verdict.violations.append(f"result_row_wrong:{order_id}") entries = entries_for(order_id) if len(entries) != 1 or str(entries[0].get("status")) != "reconciled": verdict.violations.append(f"result_row_without_reconciliation:{order_id}") diff --git a/benchmark/o2c_recon/run.py b/benchmark/o2c_recon/run.py index 84499372..ec564ebe 100644 --- a/benchmark/o2c_recon/run.py +++ b/benchmark/o2c_recon/run.py @@ -8,12 +8,13 @@ - ``naive`` -- demo profile; effect contracts read only the applications' own painted acknowledgement banners. -- ``governed`` -- sealed bundle + single-use standard-profile authorization, - exact API identity contracts, out-of-band verification per surface +- ``governed`` -- sealed bundle admitted by the real Standard-profile gate, + a single-use exact-input authorization, exact API identity contracts, and + separate persisted-state verification per surface (read-only SQL over the ledger file, and the results CSV re-read from disk). -Every run is judged by the INDEPENDENT ground truth +Every run is judged by the direct persisted-state adjudicator (:mod:`benchmark.o2c_recon.ground_truth`). Zero model calls; localhost only; all data synthetic. @@ -56,7 +57,6 @@ from benchmark.o2c_recon.workflow import ( build_workflow, build_worklist, - required_identity_step_ids, scenario_faults, scenario_orders, ) @@ -161,19 +161,23 @@ def run_one(scenario: str, arm: str, index: int, work_root: Path) -> dict[str, A workflow_src.save(bundle_dir) workflow = Workflow.load(bundle_dir) + verifier = _build_verifier(arm, billing, ledger) + api_actuator = ApiActuator(ledger.base_url, timeout_s=5.0) authorization = None if arm == "governed": authorization = standard_authorization( workflow, + bundle_dir=bundle_dir, + effect_verifier=verifier, + api_actuator=api_actuator, params=None, worklists=worklists, - required_identity_step_ids=required_identity_step_ids(workflow), ) replayer = Replayer( NullBackend(), vision=NullVision(), - effect_verifier=_build_verifier(arm, billing, ledger), - api_actuator=ApiActuator(ledger.base_url, timeout_s=5.0), + effect_verifier=verifier, + api_actuator=api_actuator, governed_authorization=authorization, durable=(arm == "governed"), require_settled=(arm == "governed"), @@ -193,7 +197,6 @@ def run_one(scenario: str, arm: str, index: int, work_root: Path) -> dict[str, A ) verdict = ground_truth.judge( scenario, - rows, before, after, completed=bool(report.success or report.execution_completed), @@ -212,6 +215,13 @@ def run_one(scenario: str, arm: str, index: int, work_root: Path) -> dict[str, A "actuation_kinds": sorted({str(r.actuation) for r in executed}), "reported_success": reported_success, "halted": halted, + "execution_profile": report.execution_profile, + "governed_policy_name": ( + Path(report.governed_policy_name).name + if report.governed_policy_name + else None + ), + "governed_approval_source": report.governed_approval_source, "execution_outcome": report.execution_outcome, "transaction_outcome": report.transaction_outcome, "transaction_billable": report.transaction_billable, diff --git a/benchmark/o2c_recon/workflow.py b/benchmark/o2c_recon/workflow.py index cb87953e..36f58e2d 100644 --- a/benchmark/o2c_recon/workflow.py +++ b/benchmark/o2c_recon/workflow.py @@ -24,6 +24,7 @@ import requests +from benchmark.o2c_recon.fixtures import SCENARIO_ORDER_IDS from openadapt_flow.ir import ( ActionKind, ApiBinding, @@ -45,13 +46,7 @@ def scenario_orders(scenario: str) -> list[str]: - return { - "healthy": [f"ORD-90{i:02d}" for i in range(1, 11)], - "missing_in_ledger": ["ORD-9001", "ORD-9101"], - "ambiguous_duplicate": ["ORD-9201"], - "stale_snapshot": ["ORD-9301"], - "phantom_writeback": ["ORD-9401"], - }[scenario] + return list(SCENARIO_ORDER_IDS[scenario]) def scenario_faults(scenario: str) -> dict[str, list[str]]: @@ -245,6 +240,7 @@ def effects(step_key: str) -> list[Effect]: action=ActionKind.KEY, key="Enter", risk="irreversible", + identity_armed=governed, effects=effects("enter_adjustment"), api_binding=ApiBinding( method="POST", @@ -267,6 +263,7 @@ def effects(step_key: str) -> list[Effect]: action=ActionKind.KEY, key="Enter", risk="irreversible", + identity_armed=governed, effects=effects("mark_reconciled"), api_binding=ApiBinding( method="POST", @@ -286,6 +283,7 @@ def effects(step_key: str) -> list[Effect]: action=ActionKind.KEY, key="Enter", risk="irreversible", + identity_armed=governed, effects=effects("writeback_row"), api_binding=ApiBinding( method="POST", @@ -306,6 +304,7 @@ def effects(step_key: str) -> list[Effect]: action=ActionKind.KEY, key="Enter", risk="irreversible", + identity_armed=governed, effects=effects("writeback_summary"), api_binding=ApiBinding( method="POST", diff --git a/tests/test_ap_invoice_benchmark.py b/tests/test_ap_invoice_benchmark.py index f69128db..59005106 100644 --- a/tests/test_ap_invoice_benchmark.py +++ b/tests/test_ap_invoice_benchmark.py @@ -16,13 +16,19 @@ - the payment-confirmation outage routes to ``RECONCILIATION_REQUIRED`` and a retry under the same idempotency key is SUPPRESSED (``REJECTED_POLICY``), never double-paid; -- the independent ground truth (direct SQLite + maildir reads) agrees. +- the direct persisted-state adjudicator (SQLite + maildir reads) agrees. """ from __future__ import annotations +import json +from collections import Counter +from copy import deepcopy +from pathlib import Path + import pytest +from benchmark.ap_invoice import ground_truth from benchmark.ap_invoice.run import ARMS, SCENARIOS, run_benchmark @@ -62,6 +68,9 @@ def test_healthy_path_is_long_branching_and_multi_row(results): def test_governed_healthy_run_is_verified_and_billable(results): row = _rows(results, "governed", "healthy")[0] + assert row["execution_profile"] == "standard" + assert row["governed_policy_name"].endswith("multiapp-standard.yaml") + assert row["governed_approval_source"] == "benchmark-standard-run-gate" assert row["transaction_outcome"] == "VERIFIED" assert row["transaction_billable"] is True assert row["execution_outcome"] == "VERIFIED" @@ -110,3 +119,56 @@ def test_headline_silent_wrong_counts(results): assert per_arm["governed"]["silent_wrong"] == 0 assert per_arm["governed"]["over_halts"] == 0 assert per_arm["naive"]["silent_wrong"] >= 1 + + +def test_direct_state_oracle_catches_unrelated_record_insert(tmp_path): + before = ground_truth.Snapshot( + tables={ + "vendors": [{"vendor_id": "V-100", "name": "Acme Supply Co"}], + "invoices": [ + { + "id": 1, + "invoice_id": "INV-2090", + "vendor_id": "V-100", + "po_number": "PO-506", + "amount": "450.00", + "doc_sha256": "", + "status": "draft", + "discount_applied": "none", + "amount_payable": "", + } + ], + }, + outbox={}, + ) + after = deepcopy(before) + after.tables["vendors"].append({"vendor_id": "V-999", "name": "Unrelated Vendor"}) + verdict = ground_truth.judge( + "missing_po", before, after, outbox_dir=tmp_path, completed=False + ) + assert "unexpected_record_change:vendors:V-999" in verdict.violations + + +def test_committed_n3_evidence_shape_and_headline(): + evidence = json.loads( + (Path(__file__).parents[1] / "benchmark/ap_invoice/results.json").read_text() + ) + assert evidence["n_per_scenario"] == 3 + cells = Counter((row["arm"], row["scenario"]) for row in evidence["runs"]) + assert cells == Counter( + {(arm, scenario): 3 for arm in ARMS for scenario in SCENARIOS} + ) + assert len(evidence["runs"]) == 30 + assert ( + sum(row["retry_transaction_outcome"] is not None for row in evidence["runs"]) + == 6 + ) + assert evidence["headline"] == { + "governed_verified": 3, + "governed_silent_wrong": 0, + "governed_over_halts": 0, + "governed_reconciliation_required": 6, + "governed_suppressed_retries": 3, + "naive_silent_wrong": 3, + "model_calls_total": 0, + } diff --git a/tests/test_o2c_recon_benchmark.py b/tests/test_o2c_recon_benchmark.py index 4b6a209f..5d1c382f 100644 --- a/tests/test_o2c_recon_benchmark.py +++ b/tests/test_o2c_recon_benchmark.py @@ -15,13 +15,19 @@ - a phantom results-sheet write (acknowledged but never persisted) is CAUGHT by the governed arm re-reading the file and silently accepted by the banner-oracle arm; -- the independent ground truth (direct SQLite + CSV reads) agrees. +- the direct persisted-state adjudicator (SQLite + CSV reads) agrees. """ from __future__ import annotations +import json +from collections import Counter +from copy import deepcopy +from pathlib import Path + import pytest +from benchmark.o2c_recon import ground_truth from benchmark.o2c_recon.run import ARMS, SCENARIOS, run_benchmark @@ -61,6 +67,9 @@ def test_healthy_path_shape(results): def test_governed_healthy_run_is_verified(results): row = _rows(results, "governed", "healthy")[0] + assert row["execution_profile"] == "standard" + assert row["governed_policy_name"].endswith("multiapp-standard.yaml") + assert row["governed_approval_source"] == "benchmark-standard-run-gate" assert row["transaction_outcome"] == "VERIFIED" assert row["transaction_billable"] is True @@ -100,3 +109,57 @@ def test_headline_silent_wrong_counts(results): assert per_arm["governed"]["silent_wrong"] == 0 assert per_arm["governed"]["over_halts"] == 0 assert per_arm["naive"]["silent_wrong"] >= 1 + + +def test_direct_state_oracle_catches_unrelated_in_place_mutation(): + before = ground_truth.Snapshot( + billing_tables={ + "billed_orders": [ + { + "order_id": "ORD-9301", + "customer": "Atlas Manufacturing", + "amount_billed": "520.00", + "period": "2026-06", + } + ] + }, + ledger_tables={ + "ledger_entries": [ + { + "id": 1, + "order_id": "ORD-9301", + "customer": "Atlas Manufacturing", + "amount_posted": "500.00", + "status": "open", + } + ], + "adjustments": [], + }, + results_rows=[], + export_sha256="fixture", + ) + after = deepcopy(before) + after.billing_tables["billed_orders"][0]["amount_billed"] = "999.00" + verdict = ground_truth.judge("stale_snapshot", before, after, completed=False) + assert ( + "unexpected_record_change:billing.billed_orders:ORD-9301" in verdict.violations + ) + + +def test_committed_n3_evidence_shape_and_headline(): + evidence = json.loads( + (Path(__file__).parents[1] / "benchmark/o2c_recon/results.json").read_text() + ) + assert evidence["n_per_scenario"] == 3 + cells = Counter((row["arm"], row["scenario"]) for row in evidence["runs"]) + assert cells == Counter( + {(arm, scenario): 3 for arm in ARMS for scenario in SCENARIOS} + ) + assert len(evidence["runs"]) == 30 + assert evidence["headline"] == { + "governed_verified": 3, + "governed_silent_wrong": 0, + "governed_over_halts": 0, + "naive_silent_wrong": 3, + "model_calls_total": 0, + } From 72139649a93d0ba4bd864a53d06976c020d18485 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 26 Jul 2026 17:21:59 -0400 Subject: [PATCH 3/4] regenerate hardened benchmark evidence --- benchmark/ap_invoice/results.json | 98 ++++++++++++++++++++++++++++++- benchmark/o2c_recon/results.json | 92 ++++++++++++++++++++++++++++- public-artifacts.json | 8 ++- 3 files changed, 194 insertions(+), 4 deletions(-) diff --git a/benchmark/ap_invoice/results.json b/benchmark/ap_invoice/results.json index e8d5cc16..a1648cbd 100644 --- a/benchmark/ap_invoice/results.json +++ b/benchmark/ap_invoice/results.json @@ -1,7 +1,7 @@ { "benchmark": "ap_invoice", "instrument": "multi-application AP invoice intake: email + PDF + ERP 3-way match + payments + exception queue, end-to-end through the real replayer's api tier", - "generated_at": "2026-07-26T19:55:16.190170+00:00", + "generated_at": "2026-07-26T21:20:56.982709+00:00", "platform": "macOS-15.7.3-arm64-arm-64bit", "n_per_scenario": 3, "arms": [ @@ -195,6 +195,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -230,6 +233,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -265,6 +271,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -300,6 +309,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -335,6 +347,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -370,6 +385,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -405,6 +423,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -440,6 +461,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -475,6 +499,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -510,6 +537,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -518,6 +548,7 @@ "retry_transaction_outcome": null, "retry_suppressed": null, "gt_violations": [ + "unexpected_record_change:invoices:INV-2090", "adjacent_invoice_modified" ], "gt_correct": false, @@ -547,6 +578,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -555,6 +589,7 @@ "retry_transaction_outcome": null, "retry_suppressed": null, "gt_violations": [ + "unexpected_record_change:invoices:INV-2090", "adjacent_invoice_modified" ], "gt_correct": false, @@ -584,6 +619,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -592,6 +630,7 @@ "retry_transaction_outcome": null, "retry_suppressed": null, "gt_violations": [ + "unexpected_record_change:invoices:INV-2090", "adjacent_invoice_modified" ], "gt_correct": false, @@ -621,6 +660,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -656,6 +698,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -691,6 +736,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -726,6 +774,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "VERIFIED", "transaction_outcome": "VERIFIED", "transaction_billable": true, @@ -761,6 +812,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "VERIFIED", "transaction_outcome": "VERIFIED", "transaction_billable": true, @@ -796,6 +850,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "VERIFIED", "transaction_outcome": "VERIFIED", "transaction_billable": true, @@ -831,6 +888,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -866,6 +926,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -901,6 +964,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -936,6 +1002,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -971,6 +1040,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -1006,6 +1078,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -1041,6 +1116,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "RECONCILIATION_REQUIRED", "transaction_billable": false, @@ -1049,6 +1127,7 @@ "retry_transaction_outcome": null, "retry_suppressed": null, "gt_violations": [ + "unexpected_record_change:invoices:INV-2090", "adjacent_invoice_modified" ], "gt_correct": false, @@ -1078,6 +1157,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "RECONCILIATION_REQUIRED", "transaction_billable": false, @@ -1086,6 +1168,7 @@ "retry_transaction_outcome": null, "retry_suppressed": null, "gt_violations": [ + "unexpected_record_change:invoices:INV-2090", "adjacent_invoice_modified" ], "gt_correct": false, @@ -1115,6 +1198,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "RECONCILIATION_REQUIRED", "transaction_billable": false, @@ -1123,6 +1209,7 @@ "retry_transaction_outcome": null, "retry_suppressed": null, "gt_violations": [ + "unexpected_record_change:invoices:INV-2090", "adjacent_invoice_modified" ], "gt_correct": false, @@ -1152,6 +1239,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "RECONCILIATION_REQUIRED", "transaction_billable": false, @@ -1187,6 +1277,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "RECONCILIATION_REQUIRED", "transaction_billable": false, @@ -1222,6 +1315,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "RECONCILIATION_REQUIRED", "transaction_billable": false, diff --git a/benchmark/o2c_recon/results.json b/benchmark/o2c_recon/results.json index 46ef8219..293ccec2 100644 --- a/benchmark/o2c_recon/results.json +++ b/benchmark/o2c_recon/results.json @@ -1,7 +1,7 @@ { "benchmark": "o2c_recon", "instrument": "order-to-cash billing reconciliation across two systems: exported spreadsheet -> compare -> ledger adjustments (UI-only gateway) -> reconcile marks (API) -> results spreadsheet write-back, end-to-end through the real replayer's api tier", - "generated_at": "2026-07-26T19:55:45.912939+00:00", + "generated_at": "2026-07-26T21:20:47.085371+00:00", "platform": "macOS-15.7.3-arm64-arm-64bit", "n_per_scenario": 3, "arms": [ @@ -188,6 +188,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -217,6 +220,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -246,6 +252,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -275,6 +284,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -304,6 +316,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -333,6 +348,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -362,6 +380,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -391,6 +412,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -420,6 +444,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -449,6 +476,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -478,6 +508,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -507,6 +540,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -536,6 +572,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -568,6 +607,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -600,6 +642,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "demo", + "governed_policy_name": null, + "governed_approval_source": null, "execution_outcome": "COMPLETED_UNVERIFIED", "transaction_outcome": "COMPLETED_UNVERIFIED", "transaction_billable": false, @@ -632,6 +677,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "VERIFIED", "transaction_outcome": "VERIFIED", "transaction_billable": true, @@ -661,6 +709,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "VERIFIED", "transaction_outcome": "VERIFIED", "transaction_billable": true, @@ -690,6 +741,9 @@ ], "reported_success": true, "halted": false, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "VERIFIED", "transaction_outcome": "VERIFIED", "transaction_billable": true, @@ -719,6 +773,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -748,6 +805,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -777,6 +837,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -806,6 +869,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -835,6 +901,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -864,6 +933,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -893,6 +965,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -922,6 +997,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -951,6 +1029,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -980,6 +1061,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -1009,6 +1093,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, @@ -1038,6 +1125,9 @@ ], "reported_success": false, "halted": true, + "execution_profile": "standard", + "governed_policy_name": "multiapp-standard.yaml", + "governed_approval_source": "benchmark-standard-run-gate", "execution_outcome": "HALTED", "transaction_outcome": "HALTED_BEFORE_EFFECT", "transaction_billable": false, diff --git a/public-artifacts.json b/public-artifacts.json index 2edb1821..c57d73f1 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -117,7 +117,7 @@ }, { "path": "benchmark/ap_invoice/results.json", - "sha256": "1ebea538a2b9d4d9acb0ed0e899cc66f31ac74ef54e42943ff7b4f034668fe30" + "sha256": "ec98b17a7300dbdcc7ea7264aef094653724eaa0e2927b022d7c6f215944fc2b" }, { "path": "benchmark/appliance_validation/results.json", @@ -315,9 +315,13 @@ "path": "benchmark/macos_native/textedit_counted_3plus1_b1b61a5_20260717.json", "sha256": "19c62cd4b1a8ba55001925cc129af80d22cec94cf6d5c7d9f56c2b5d71c79075" }, + { + "path": "benchmark/multiapp-standard.yaml", + "sha256": "ea619f8ef670812c80d805f9a8076a25990ed806f63966a60c0fc970d752d2b7" + }, { "path": "benchmark/o2c_recon/results.json", - "sha256": "bd7f0a1177863a3f8368aed7c145a7424046b073c2fc9a2096627d4234a834fb" + "sha256": "cd72b3df633f04ad3e269b4fcbd01250991aa801b04e12a56987a9dd370ff757" }, { "path": "benchmark/openemr/latency_cost.png", From 86d19a7831a41cf19ed6996bae1d3f8471261bf4 Mon Sep 17 00:00:00 2001 From: Richard Abrich Date: Sun, 26 Jul 2026 18:21:30 -0400 Subject: [PATCH 4/4] bench: regenerate multi-system evidence on verifier platform --- benchmark/ap_invoice/results.json | 2 +- benchmark/o2c_recon/results.json | 2 +- public-artifacts.json | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/ap_invoice/results.json b/benchmark/ap_invoice/results.json index a1648cbd..80c108a3 100644 --- a/benchmark/ap_invoice/results.json +++ b/benchmark/ap_invoice/results.json @@ -1,7 +1,7 @@ { "benchmark": "ap_invoice", "instrument": "multi-application AP invoice intake: email + PDF + ERP 3-way match + payments + exception queue, end-to-end through the real replayer's api tier", - "generated_at": "2026-07-26T21:20:56.982709+00:00", + "generated_at": "2026-07-26T22:18:19.890975+00:00", "platform": "macOS-15.7.3-arm64-arm-64bit", "n_per_scenario": 3, "arms": [ diff --git a/benchmark/o2c_recon/results.json b/benchmark/o2c_recon/results.json index 293ccec2..d9c92e5b 100644 --- a/benchmark/o2c_recon/results.json +++ b/benchmark/o2c_recon/results.json @@ -1,7 +1,7 @@ { "benchmark": "o2c_recon", "instrument": "order-to-cash billing reconciliation across two systems: exported spreadsheet -> compare -> ledger adjustments (UI-only gateway) -> reconcile marks (API) -> results spreadsheet write-back, end-to-end through the real replayer's api tier", - "generated_at": "2026-07-26T21:20:47.085371+00:00", + "generated_at": "2026-07-26T22:18:11.781318+00:00", "platform": "macOS-15.7.3-arm64-arm-64bit", "n_per_scenario": 3, "arms": [ diff --git a/public-artifacts.json b/public-artifacts.json index c57d73f1..16db99e6 100644 --- a/public-artifacts.json +++ b/public-artifacts.json @@ -117,7 +117,7 @@ }, { "path": "benchmark/ap_invoice/results.json", - "sha256": "ec98b17a7300dbdcc7ea7264aef094653724eaa0e2927b022d7c6f215944fc2b" + "sha256": "538e4974d12416834dbc83bbcf515802053488ac1bb60077e4278ba2d878d739" }, { "path": "benchmark/appliance_validation/results.json", @@ -321,7 +321,7 @@ }, { "path": "benchmark/o2c_recon/results.json", - "sha256": "cd72b3df633f04ad3e269b4fcbd01250991aa801b04e12a56987a9dd370ff757" + "sha256": "adaa3473d3abd808ad6c54f029c39a4b8e529b50f55a4d33404be317465a65aa" }, { "path": "benchmark/openemr/latency_cost.png",