diff --git a/crates/tinyflows-adaptive/examples/service.rs b/crates/tinyflows-adaptive/examples/service.rs index 5c57a583..c0597fef 100644 --- a/crates/tinyflows-adaptive/examples/service.rs +++ b/crates/tinyflows-adaptive/examples/service.rs @@ -191,466 +191,4 @@ fn spawn_device(mut from_server: mpsc::Receiver, to_server: mpsc::Sender }); } -// --------------------------------------------------------------------------- -// Inference. In production: an HTTP client routing `tier` → model. -// --------------------------------------------------------------------------- - -/// A script standing where the model client goes. The one production-relevant -/// thing about it is the match: every request carries `tier`, and routing on -/// it — select to a cheap model, judge to a strong one — is host config, not -/// crate code. -struct TierRouter; - -#[async_trait] -impl LlmProvider for TierRouter { - async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { - let shown = request["messages"][1]["content"] - .as_str() - .unwrap_or_default(); - Ok(match request["tier"].as_str().unwrap_or_default() { - // Reads the candidate listing it was shown, like a real selector. - "select" => { - let first = shown - .lines() - .find_map(|line| line.trim().strip_prefix("- id: ")); - json!({ - "workflow_id": first, - "why": "matches the goal", - "inputs": { "repo": "acme/rust-lib" }, - }) - } - // The recipe surface: steps in, never graph syntax — the - // lowering writes the graph. The ask names no concrete repo; - // the declared input carries it, which is what lets `keep` file - // the plan for the next repository. - "author" => json!({ - "why": "nothing stored fits yet", - "declared": [ - { "name": "repo", "description": "the repository to review", "required": true } - ], - "inputs": { "repo": "acme/thing" }, - "steps": [{ - "id": "review", - "ask": "Review the open pull requests and post a summary." - }], - }), - "judge" => json!({ "satisfied": true, "gap": "" }), - "generalise" => json!({ - "name": "Review a repository's pull requests", - "description": "Reviews the open pull requests on a repository \ - and posts a summary. Takes the repository as an input.", - "reusable": true, - }), - "consolidate" => json!({ "lessons": [], "corroborate": [] }), - other => json!({ "error": format!("unexpected tier {other}") }), - }) - } -} - -// --------------------------------------------------------------------------- -// The storage relay — device-master over a wire. -// --------------------------------------------------------------------------- - -/// One message of the vault wire, either direction. -/// -/// The service sends `Load` / `Put` / `Remove`; the device answers `Records` -/// or `Ack` under the echoed wire id. Same discipline as the run relay: ids -/// are minted service-side, so a late reply can never resolve the wrong -/// waiter. -#[derive(serde::Serialize, serde::Deserialize)] -#[serde(tag = "kind", rename_all = "snake_case")] -enum VaultFrame { - /// Service → device: send me your whole catalogue. - Load { wire_id: String }, - /// Device → service: the catalogue. - Records { - wire_id: String, - records: Vec, - }, - /// Service → device: write this record into YOUR store. - /// - /// Boxed: a record dwarfs every other variant, and the frame travels - /// through queues sized for the small ones. - Put { - wire_id: String, - record: Box, - }, - /// Service → device: remove this id from your store. - Remove { wire_id: String, id: String }, - /// Device → service: the write or removal settled. - Ack { - wire_id: String, - error: Option, - }, -} - -/// What a vault waiter resolves to. -enum VaultReply { - Records(Vec), - Ack(Option), -} - -/// The service's view of the DEVICE's workflow store. -/// -/// Stateless on purpose: it holds a sender, a deadline and a counter — -/// never a record. During an episode the [`Snapshot`] is the only -/// service-side copy, and it is memory; when the success gate flushes, each -/// kept workflow crosses this wire and comes to rest in the device's own -/// store, which is the one durable home the design allows it. -struct DeviceVault { - to_device: mpsc::Sender, - waiting: Mutex>>, - sequence: AtomicU64, - deadline: Duration, -} - -impl DeviceVault { - fn new(to_device: mpsc::Sender, deadline: Duration) -> Arc { - Arc::new(Self { - to_device, - waiting: Mutex::new(HashMap::new()), - sequence: AtomicU64::new(0), - deadline, - }) - } - - /// HOST: the body of your `socket.on("tinyflows:vault_reply", …)` handler. - fn deliver(&self, frame: &str) { - let Ok(frame) = serde_json::from_str::(frame) else { - eprintln!(" ! dropped an unparseable vault frame"); - return; - }; - let (wire_id, reply) = match frame { - VaultFrame::Records { wire_id, records } => (wire_id, VaultReply::Records(records)), - VaultFrame::Ack { wire_id, error } => (wire_id, VaultReply::Ack(error)), - // Requests only travel the other way. - _ => return, - }; - if let Some(tx) = self.waiting.lock().expect("vault waiters").remove(&wire_id) { - let _ = tx.send(reply); - } else { - eprintln!(" ! late or unknown vault reply `{wire_id}`"); - } - } - - async fn exchange(&self, mut frame: VaultFrame) -> Result { - let wire_id = format!("vault#{}", self.sequence.fetch_add(1, Ordering::Relaxed)); - match &mut frame { - VaultFrame::Load { wire_id: id } - | VaultFrame::Put { wire_id: id, .. } - | VaultFrame::Remove { wire_id: id, .. } => *id = wire_id.clone(), - _ => unreachable!("the service only sends requests"), - } - let payload = serde_json::to_string(&frame) - .map_err(|e| WorkflowError::Engine(format!("vault frame: {e}")))?; - - let (tx, rx) = oneshot::channel(); - self.waiting - .lock() - .expect("vault waiters") - .insert(wire_id.clone(), tx); - - if self.to_device.send(payload).await.is_err() { - self.waiting.lock().expect("vault waiters").remove(&wire_id); - return Err(WorkflowError::Engine("no device connected".to_string())); - } - match tokio::time::timeout(self.deadline, rx).await { - Ok(Ok(reply)) => Ok(reply), - Ok(Err(_)) => Err(WorkflowError::Engine( - "the delivery side dropped the vault waiter".to_string(), - )), - Err(_) => { - self.waiting.lock().expect("vault waiters").remove(&wire_id); - // Said with the consequence: a flush that cannot reach the - // device is a REPORTED failure — the learnings ledger is - // untouched, and nothing pretends the workflow landed. - Err(WorkflowError::Engine(format!( - "the device did not answer within {:?} — the record was NOT stored", - self.deadline - ))) - } - } - } -} - -#[async_trait] -impl Vault for DeviceVault { - async fn load(&self) -> Result, WorkflowError> { - match self - .exchange(VaultFrame::Load { - wire_id: String::new(), - }) - .await? - { - VaultReply::Records(records) => Ok(records), - VaultReply::Ack(_) => Err(WorkflowError::Engine( - "the device answered a load with an ack".to_string(), - )), - } - } - - async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { - println!(" → VaultPut {}", record.id); - match self - .exchange(VaultFrame::Put { - wire_id: String::new(), - record: Box::new(record.clone()), - }) - .await? - { - VaultReply::Ack(None) => Ok(()), - VaultReply::Ack(Some(error)) => Err(WorkflowError::Engine(error)), - VaultReply::Records(_) => Err(WorkflowError::Engine( - "the device answered a put with records".to_string(), - )), - } - } - - async fn remove(&self, id: &str) -> Result<(), WorkflowError> { - match self - .exchange(VaultFrame::Remove { - wire_id: String::new(), - id: id.to_string(), - }) - .await? - { - VaultReply::Ack(None) => Ok(()), - VaultReply::Ack(Some(error)) => Err(WorkflowError::Engine(error)), - VaultReply::Records(_) => Err(WorkflowError::Engine( - "the device answered a remove with records".to_string(), - )), - } - } -} - -/// The device's side of the storage relay. -/// -/// The whole obligation: deserialize, apply to the DEVICE's own store, reply. -/// In production the store is the device's real tinyflows store — the one its -/// workflow surfaces already list — and this task is your socket handler. -fn spawn_device_store( - store: Arc, - mut from_server: mpsc::Receiver, - to_server: mpsc::Sender, -) { - tokio::spawn(async move { - while let Some(frame) = from_server.recv().await { - let Ok(frame) = serde_json::from_str::(&frame) else { - continue; - }; - let reply = match frame { - VaultFrame::Load { wire_id } => match store.load().await { - Ok(records) => VaultFrame::Records { wire_id, records }, - Err(e) => VaultFrame::Ack { - wire_id, - error: Some(e.to_string()), - }, - }, - VaultFrame::Put { wire_id, record } => { - println!(" ← device stored `{}` in ITS store", record.id); - VaultFrame::Ack { - wire_id, - error: store.put(&record).await.err().map(|e| e.to_string()), - } - } - VaultFrame::Remove { wire_id, id } => VaultFrame::Ack { - wire_id, - error: store.remove(&id).await.err().map(|e| e.to_string()), - }, - // Replies only travel the other way. - _ => continue, - }; - let Ok(payload) = serde_json::to_string(&reply) else { - continue; - }; - let _ = to_server.send(payload).await; - } - }); -} - -// --------------------------------------------------------------------------- -// Small host pieces. -// --------------------------------------------------------------------------- - -struct WallClock; -impl Clock for WallClock { - fn now(&self) -> String { - // Opaque to the crate; a real host writes RFC 3339. Zero-padded so the - // episode listing's string ordering matches time ordering. - let secs = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs()) - .unwrap_or(0); - format!("{secs:020}") - } -} - -fn permissive() -> Arc { - #[derive(Debug, Default)] - struct Permissive; - impl HostPolicy for Permissive {} - Arc::new(Permissive) -} - -// --------------------------------------------------------------------------- -// The service. -// --------------------------------------------------------------------------- - -#[tokio::main(flavor = "multi_thread")] -async fn main() { - // ---- process scope: once, at boot ----------------------------------- - // HOST: MongoLedger::connect / SqliteLedger::at_default_location, a real - // HTTP-backed LlmProvider, real HostFacts from the device's probe. - let ledger_root = MemoryLedger::new(); - // The DEVICE's workflow store — the one durable home for workflows in - // this whole topology. In production: the device's real tinyflows store. - let device_store = Arc::new(MemoryVault::new()); - let caps = Capabilities { - llm: Arc::new(TierRouter), - ..mock_capabilities() - }; - let facts = HostFacts::unknown(); - - // The wire: two channel pairs where production has one socket — one for - // runs, one for the vault. - let (to_device_tx, to_device_rx) = mpsc::channel::(16); - let (to_server_tx, mut to_server_rx) = mpsc::channel::(16); - let relay = ChannelRelay::new(to_device_tx, Duration::from_secs(30)); - spawn_device(to_device_rx, to_server_tx); - { - // HOST: this task is your socket receive handler. - let relay = Arc::clone(&relay); - tokio::spawn(async move { - while let Some(frame) = to_server_rx.recv().await { - relay.deliver(&frame); - } - }); - } - let (vault_tx, vault_rx) = mpsc::channel::(16); - let (vault_reply_tx, mut vault_reply_rx) = mpsc::channel::(16); - let vault = DeviceVault::new(vault_tx, Duration::from_secs(30)); - spawn_device_store(Arc::clone(&device_store), vault_rx, vault_reply_tx); - { - // HOST: and this one is your vault-reply handler. - let vault = Arc::clone(&vault); - tokio::spawn(async move { - while let Some(frame) = vault_reply_rx.recv().await { - vault.deliver(&frame); - } - }); - } - - // ---- tenant scope: per request, free -------------------------------- - // The ledger stays tenant-scoped on the service: learnings are the - // service's property. Workflows are the DEVICE's — the vault handle IS - // the device, so tenancy is which device you are talking to. - let tenant = "user-7"; - let ledger = ledger_root.for_tenant(tenant); - - // ---- goal run 1: a cold catalogue, so the loop authors -------------- - println!("── goal run 1 · cold start ──"); - run_goal( - "ep-1", - &Goal::new("review the open pull requests on acme/thing"), - &ledger, - vault.as_ref(), - &caps, - &facts, - &relay, - ) - .await; - - // ---- goal run 2: the catalogue now holds what run 1 learned --------- - // Learned from the DEVICE: the service kept nothing between the runs. - println!("\n── goal run 2 · the loop reuses what the DEVICE now holds ──"); - run_goal( - "ep-2", - &Goal::new("review the open pull requests on acme/rust-lib"), - &ledger, - vault.as_ref(), - &caps, - &facts, - &relay, - ) - .await; - - // ---- what the DEVICE holds, and what the trail says ----------------- - // Listed from the device store directly: proof the records came to rest - // on the device, not in anything the service kept. - println!("\n── the DEVICE's store ──"); - for record in device_store.load().await.expect("device load") { - println!(" {} · {}", record.id, record.name); - } - - println!("\n── the tenant's shelf, read back over the wire ──"); - let snapshot = Snapshot::load(vault.as_ref(), permissive()) - .await - .expect("load"); - let store: Arc = Arc::new(snapshot); - for listing in inventory::shelf(&store, &ledger).await.expect("shelf") { - println!( - " {} · {:?} · run {}× satisfied {}× · learned: {}", - listing.id, - listing.standing, - listing.score.applied, - listing.score.helped, - listing.learned - ); - } - - println!("\n── the trail ──"); - for episode in ["ep-1", "ep-2"] { - for row in ledger.rows(episode).await.expect("rows") { - println!( - " {episode} attempt {} · [{}] → {}", - row.attempt, row.approach_sig, row.outcome - ); - } - } -} - -/// One goal run, end to end: fetch the catalogue, drive the loop over the -/// relay, and persist what was learned only if the goal was achieved. -async fn run_goal( - episode: &str, - goal: &Goal, - ledger: &MemoryLedger, - vault: &DeviceVault, - caps: &Capabilities, - facts: &HostFacts, - relay: &Arc, -) { - // Fetched fresh each goal run — FROM THE DEVICE, over the wire: the - // service holds no catalogue of its own to consult. - let snapshot = Snapshot::load(vault, permissive()).await.expect("load"); - let store: Arc = Arc::new(snapshot.clone()); - - let runner = Remote { - relay: relay.as_ref(), - attempt_id: episode.to_string(), - }; - let engine = Loop { - ledger, - store: &store, - caps, - facts, - runner: &runner, - clock: &WallClock, - budget: Budget::default(), - conn: None, // HOST: the tenant's credential reference - }; - - let finished = engine.run(episode, goal).await.expect("the loop ran"); - println!( - " {episode}: {:?} after {} attempt(s)", - finished.status, finished.attempts - ); - - // The success gate: the DEVICE only ever receives workflows from goal - // runs that succeeded — each pending record crosses the wire as a put - // the device writes into its own store. - if finished.status == EpisodeStatus::Satisfied && snapshot.pending() > 0 { - let landed = snapshot.flush(vault).await.expect("flush"); - println!(" flushed {landed} learned workflow(s) to the DEVICE"); - } -} +include!("service/runtime.rs"); diff --git a/crates/tinyflows-adaptive/examples/service/runtime.rs b/crates/tinyflows-adaptive/examples/service/runtime.rs new file mode 100644 index 00000000..52875021 --- /dev/null +++ b/crates/tinyflows-adaptive/examples/service/runtime.rs @@ -0,0 +1,463 @@ +// --------------------------------------------------------------------------- +// Inference. In production: an HTTP client routing `tier` → model. +// --------------------------------------------------------------------------- + +/// A script standing where the model client goes. The one production-relevant +/// thing about it is the match: every request carries `tier`, and routing on +/// it — select to a cheap model, judge to a strong one — is host config, not +/// crate code. +struct TierRouter; + +#[async_trait] +impl LlmProvider for TierRouter { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let shown = request["messages"][1]["content"] + .as_str() + .unwrap_or_default(); + Ok(match request["tier"].as_str().unwrap_or_default() { + // Reads the candidate listing it was shown, like a real selector. + "select" => { + let first = shown + .lines() + .find_map(|line| line.trim().strip_prefix("- id: ")); + json!({ + "workflow_id": first, + "why": "matches the goal", + "inputs": { "repo": "acme/rust-lib" }, + }) + } + // The recipe surface: steps in, never graph syntax — the + // lowering writes the graph. The ask names no concrete repo; + // the declared input carries it, which is what lets `keep` file + // the plan for the next repository. + "author" => json!({ + "why": "nothing stored fits yet", + "declared": [ + { "name": "repo", "description": "the repository to review", "required": true } + ], + "inputs": { "repo": "acme/thing" }, + "steps": [{ + "id": "review", + "ask": "Review the open pull requests and post a summary." + }], + }), + "judge" => json!({ "satisfied": true, "gap": "" }), + "generalise" => json!({ + "name": "Review a repository's pull requests", + "description": "Reviews the open pull requests on a repository \ + and posts a summary. Takes the repository as an input.", + "reusable": true, + }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + other => json!({ "error": format!("unexpected tier {other}") }), + }) + } +} + +// --------------------------------------------------------------------------- +// The storage relay — device-master over a wire. +// --------------------------------------------------------------------------- + +/// One message of the vault wire, either direction. +/// +/// The service sends `Load` / `Put` / `Remove`; the device answers `Records` +/// or `Ack` under the echoed wire id. Same discipline as the run relay: ids +/// are minted service-side, so a late reply can never resolve the wrong +/// waiter. +#[derive(serde::Serialize, serde::Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case")] +enum VaultFrame { + /// Service → device: send me your whole catalogue. + Load { wire_id: String }, + /// Device → service: the catalogue. + Records { + wire_id: String, + records: Vec, + }, + /// Service → device: write this record into YOUR store. + /// + /// Boxed: a record dwarfs every other variant, and the frame travels + /// through queues sized for the small ones. + Put { + wire_id: String, + record: Box, + }, + /// Service → device: remove this id from your store. + Remove { wire_id: String, id: String }, + /// Device → service: the write or removal settled. + Ack { + wire_id: String, + error: Option, + }, +} + +/// What a vault waiter resolves to. +enum VaultReply { + Records(Vec), + Ack(Option), +} + +/// The service's view of the DEVICE's workflow store. +/// +/// Stateless on purpose: it holds a sender, a deadline and a counter — +/// never a record. During an episode the [`Snapshot`] is the only +/// service-side copy, and it is memory; when the success gate flushes, each +/// kept workflow crosses this wire and comes to rest in the device's own +/// store, which is the one durable home the design allows it. +struct DeviceVault { + to_device: mpsc::Sender, + waiting: Mutex>>, + sequence: AtomicU64, + deadline: Duration, +} + +impl DeviceVault { + fn new(to_device: mpsc::Sender, deadline: Duration) -> Arc { + Arc::new(Self { + to_device, + waiting: Mutex::new(HashMap::new()), + sequence: AtomicU64::new(0), + deadline, + }) + } + + /// HOST: the body of your `socket.on("tinyflows:vault_reply", …)` handler. + fn deliver(&self, frame: &str) { + let Ok(frame) = serde_json::from_str::(frame) else { + eprintln!(" ! dropped an unparseable vault frame"); + return; + }; + let (wire_id, reply) = match frame { + VaultFrame::Records { wire_id, records } => (wire_id, VaultReply::Records(records)), + VaultFrame::Ack { wire_id, error } => (wire_id, VaultReply::Ack(error)), + // Requests only travel the other way. + _ => return, + }; + if let Some(tx) = self.waiting.lock().expect("vault waiters").remove(&wire_id) { + let _ = tx.send(reply); + } else { + eprintln!(" ! late or unknown vault reply `{wire_id}`"); + } + } + + async fn exchange(&self, mut frame: VaultFrame) -> Result { + let wire_id = format!("vault#{}", self.sequence.fetch_add(1, Ordering::Relaxed)); + match &mut frame { + VaultFrame::Load { wire_id: id } + | VaultFrame::Put { wire_id: id, .. } + | VaultFrame::Remove { wire_id: id, .. } => *id = wire_id.clone(), + _ => unreachable!("the service only sends requests"), + } + let payload = serde_json::to_string(&frame) + .map_err(|e| WorkflowError::Engine(format!("vault frame: {e}")))?; + + let (tx, rx) = oneshot::channel(); + self.waiting + .lock() + .expect("vault waiters") + .insert(wire_id.clone(), tx); + + if self.to_device.send(payload).await.is_err() { + self.waiting.lock().expect("vault waiters").remove(&wire_id); + return Err(WorkflowError::Engine("no device connected".to_string())); + } + match tokio::time::timeout(self.deadline, rx).await { + Ok(Ok(reply)) => Ok(reply), + Ok(Err(_)) => Err(WorkflowError::Engine( + "the delivery side dropped the vault waiter".to_string(), + )), + Err(_) => { + self.waiting.lock().expect("vault waiters").remove(&wire_id); + // Said with the consequence: a flush that cannot reach the + // device is a REPORTED failure — the learnings ledger is + // untouched, and nothing pretends the workflow landed. + Err(WorkflowError::Engine(format!( + "the device did not answer within {:?} — the record was NOT stored", + self.deadline + ))) + } + } + } +} + +#[async_trait] +impl Vault for DeviceVault { + async fn load(&self) -> Result, WorkflowError> { + match self + .exchange(VaultFrame::Load { + wire_id: String::new(), + }) + .await? + { + VaultReply::Records(records) => Ok(records), + VaultReply::Ack(_) => Err(WorkflowError::Engine( + "the device answered a load with an ack".to_string(), + )), + } + } + + async fn put(&self, record: &WorkflowRecord) -> Result<(), WorkflowError> { + println!(" → VaultPut {}", record.id); + match self + .exchange(VaultFrame::Put { + wire_id: String::new(), + record: Box::new(record.clone()), + }) + .await? + { + VaultReply::Ack(None) => Ok(()), + VaultReply::Ack(Some(error)) => Err(WorkflowError::Engine(error)), + VaultReply::Records(_) => Err(WorkflowError::Engine( + "the device answered a put with records".to_string(), + )), + } + } + + async fn remove(&self, id: &str) -> Result<(), WorkflowError> { + match self + .exchange(VaultFrame::Remove { + wire_id: String::new(), + id: id.to_string(), + }) + .await? + { + VaultReply::Ack(None) => Ok(()), + VaultReply::Ack(Some(error)) => Err(WorkflowError::Engine(error)), + VaultReply::Records(_) => Err(WorkflowError::Engine( + "the device answered a remove with records".to_string(), + )), + } + } +} + +/// The device's side of the storage relay. +/// +/// The whole obligation: deserialize, apply to the DEVICE's own store, reply. +/// In production the store is the device's real tinyflows store — the one its +/// workflow surfaces already list — and this task is your socket handler. +fn spawn_device_store( + store: Arc, + mut from_server: mpsc::Receiver, + to_server: mpsc::Sender, +) { + tokio::spawn(async move { + while let Some(frame) = from_server.recv().await { + let Ok(frame) = serde_json::from_str::(&frame) else { + continue; + }; + let reply = match frame { + VaultFrame::Load { wire_id } => match store.load().await { + Ok(records) => VaultFrame::Records { wire_id, records }, + Err(e) => VaultFrame::Ack { + wire_id, + error: Some(e.to_string()), + }, + }, + VaultFrame::Put { wire_id, record } => { + println!(" ← device stored `{}` in ITS store", record.id); + VaultFrame::Ack { + wire_id, + error: store.put(&record).await.err().map(|e| e.to_string()), + } + } + VaultFrame::Remove { wire_id, id } => VaultFrame::Ack { + wire_id, + error: store.remove(&id).await.err().map(|e| e.to_string()), + }, + // Replies only travel the other way. + _ => continue, + }; + let Ok(payload) = serde_json::to_string(&reply) else { + continue; + }; + let _ = to_server.send(payload).await; + } + }); +} + +// --------------------------------------------------------------------------- +// Small host pieces. +// --------------------------------------------------------------------------- + +struct WallClock; +impl Clock for WallClock { + fn now(&self) -> String { + // Opaque to the crate; a real host writes RFC 3339. Zero-padded so the + // episode listing's string ordering matches time ordering. + let secs = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + format!("{secs:020}") + } +} + +fn permissive() -> Arc { + #[derive(Debug, Default)] + struct Permissive; + impl HostPolicy for Permissive {} + Arc::new(Permissive) +} + +// --------------------------------------------------------------------------- +// The service. +// --------------------------------------------------------------------------- + +#[tokio::main(flavor = "multi_thread")] +async fn main() { + // ---- process scope: once, at boot ----------------------------------- + // HOST: MongoLedger::connect / SqliteLedger::at_default_location, a real + // HTTP-backed LlmProvider, real HostFacts from the device's probe. + let ledger_root = MemoryLedger::new(); + // The DEVICE's workflow store — the one durable home for workflows in + // this whole topology. In production: the device's real tinyflows store. + let device_store = Arc::new(MemoryVault::new()); + let caps = Capabilities { + llm: Arc::new(TierRouter), + ..mock_capabilities() + }; + let facts = HostFacts::unknown(); + + // The wire: two channel pairs where production has one socket — one for + // runs, one for the vault. + let (to_device_tx, to_device_rx) = mpsc::channel::(16); + let (to_server_tx, mut to_server_rx) = mpsc::channel::(16); + let relay = ChannelRelay::new(to_device_tx, Duration::from_secs(30)); + spawn_device(to_device_rx, to_server_tx); + { + // HOST: this task is your socket receive handler. + let relay = Arc::clone(&relay); + tokio::spawn(async move { + while let Some(frame) = to_server_rx.recv().await { + relay.deliver(&frame); + } + }); + } + let (vault_tx, vault_rx) = mpsc::channel::(16); + let (vault_reply_tx, mut vault_reply_rx) = mpsc::channel::(16); + let vault = DeviceVault::new(vault_tx, Duration::from_secs(30)); + spawn_device_store(Arc::clone(&device_store), vault_rx, vault_reply_tx); + { + // HOST: and this one is your vault-reply handler. + let vault = Arc::clone(&vault); + tokio::spawn(async move { + while let Some(frame) = vault_reply_rx.recv().await { + vault.deliver(&frame); + } + }); + } + + // ---- tenant scope: per request, free -------------------------------- + // The ledger stays tenant-scoped on the service: learnings are the + // service's property. Workflows are the DEVICE's — the vault handle IS + // the device, so tenancy is which device you are talking to. + let tenant = "user-7"; + let ledger = ledger_root.for_tenant(tenant); + + // ---- goal run 1: a cold catalogue, so the loop authors -------------- + println!("── goal run 1 · cold start ──"); + run_goal( + "ep-1", + &Goal::new("review the open pull requests on acme/thing"), + &ledger, + vault.as_ref(), + &caps, + &facts, + &relay, + ) + .await; + + // ---- goal run 2: the catalogue now holds what run 1 learned --------- + // Learned from the DEVICE: the service kept nothing between the runs. + println!("\n── goal run 2 · the loop reuses what the DEVICE now holds ──"); + run_goal( + "ep-2", + &Goal::new("review the open pull requests on acme/rust-lib"), + &ledger, + vault.as_ref(), + &caps, + &facts, + &relay, + ) + .await; + + // ---- what the DEVICE holds, and what the trail says ----------------- + // Listed from the device store directly: proof the records came to rest + // on the device, not in anything the service kept. + println!("\n── the DEVICE's store ──"); + for record in device_store.load().await.expect("device load") { + println!(" {} · {}", record.id, record.name); + } + + println!("\n── the tenant's shelf, read back over the wire ──"); + let snapshot = Snapshot::load(vault.as_ref(), permissive()) + .await + .expect("load"); + let store: Arc = Arc::new(snapshot); + for listing in inventory::shelf(&store, &ledger).await.expect("shelf") { + println!( + " {} · {:?} · run {}× satisfied {}× · learned: {}", + listing.id, + listing.standing, + listing.score.applied, + listing.score.helped, + listing.learned + ); + } + + println!("\n── the trail ──"); + for episode in ["ep-1", "ep-2"] { + for row in ledger.rows(episode).await.expect("rows") { + println!( + " {episode} attempt {} · [{}] → {}", + row.attempt, row.approach_sig, row.outcome + ); + } + } +} + +/// One goal run, end to end: fetch the catalogue, drive the loop over the +/// relay, and persist what was learned only if the goal was achieved. +async fn run_goal( + episode: &str, + goal: &Goal, + ledger: &MemoryLedger, + vault: &DeviceVault, + caps: &Capabilities, + facts: &HostFacts, + relay: &Arc, +) { + // Fetched fresh each goal run — FROM THE DEVICE, over the wire: the + // service holds no catalogue of its own to consult. + let snapshot = Snapshot::load(vault, permissive()).await.expect("load"); + let store: Arc = Arc::new(snapshot.clone()); + + let runner = Remote { + relay: relay.as_ref(), + attempt_id: episode.to_string(), + }; + let engine = Loop { + ledger, + store: &store, + caps, + facts, + runner: &runner, + clock: &WallClock, + budget: Budget::default(), + conn: None, // HOST: the tenant's credential reference + }; + + let finished = engine.run(episode, goal).await.expect("the loop ran"); + println!( + " {episode}: {:?} after {} attempt(s)", + finished.status, finished.attempts + ); + + // The success gate: the DEVICE only ever receives workflows from goal + // runs that succeeded — each pending record crosses the wire as a put + // the device writes into its own store. + if finished.status == EpisodeStatus::Satisfied && snapshot.pending() > 0 { + let landed = snapshot.flush(vault).await.expect("flush"); + println!(" flushed {landed} learned workflow(s) to the DEVICE"); + } +} diff --git a/crates/tinyflows-adaptive/src/closing/judge.rs b/crates/tinyflows-adaptive/src/closing/judge.rs index c0958cea..2daa230c 100644 --- a/crates/tinyflows-adaptive/src/closing/judge.rs +++ b/crates/tinyflows-adaptive/src/closing/judge.rs @@ -289,215 +289,5 @@ fn without_a_model(evidence: &Evidence<'_>) -> Option { } #[cfg(test)] -mod tests { - use super::*; - - /// A provider that answers the judge with a fixed blocker. - struct Says(&'static str); - - #[async_trait::async_trait] - impl tinyflows::caps::LlmProvider for Says { - async fn complete( - &self, - _request: serde_json::Value, - _conn: Option<&str>, - ) -> tinyflows::error::Result { - Ok(serde_json::json!({ - "satisfied": false, - "blocker": self.0, - "gap": "nothing was fetched", - })) - } - } - - async fn verdict_for(blocker: &'static str, failed: Option) -> Verdict { - let outcome = tinyflows::engine::RunOutcome { - // Non-empty: the mechanical pre-judge must not settle this one, - // because the point is what the MODEL's answer becomes. - output: serde_json::json!({ "nodes": { "fetch": { "json": 1 } } }), - pending_approvals: Vec::new(), - cancelled: false, - }; - let diagnosis = Diagnosis::default(); - let evidence = Evidence { - outcome: &outcome, - diagnosis: &diagnosis, - changed: String::new(), - failed, - }; - let caps = tinyflows::caps::Capabilities { - llm: std::sync::Arc::new(Says(blocker)), - ..tinyflows::caps::mock::mock_capabilities() - }; - judge(&Goal::new("do the thing"), &evidence, &caps, None) - .await - .expect("judged") - } - - #[tokio::test] - async fn a_mechanically_broken_run_cannot_be_called_terminal() { - // Field observation: a shell step exited nonzero, the judge answered - // `missing_evidence`, and the episode ended with two of its three - // attempts unused — when rewriting the script was the whole fix. - // The prompt says mechanical failures are goal_not_met; a model that - // misreads it must not get to end the episode anyway. - let verdict = verdict_for("missing_evidence", Some("script exited 5".into())).await; - assert_eq!(verdict.blocker, Blocker::GoalNotMet); - assert!(verdict.blocker.continuable()); - } - - #[tokio::test] - async fn a_run_that_completed_keeps_the_judges_terminal_verdict() { - // No mechanical failure: the judge is the authority on whether - // another attempt could help, and this downgrade must not become a - // blanket refusal to ever stand down. - let verdict = verdict_for("missing_evidence", None).await; - assert_eq!(verdict.blocker, Blocker::MissingEvidence); - } - - #[tokio::test] - async fn a_broken_run_still_waiting_on_a_person_stays_terminal() { - // NeedsInput and ExternalWait survive the downgrade: both mean - // something OUTSIDE the loop must move, which a broken run does not - // change. - let verdict = verdict_for("needs_input", Some("script exited 5".into())).await; - assert_eq!(verdict.blocker, Blocker::NeedsInput); - } - - use serde_json::json; - use tinyflows::diagnostics::{HiddenError, NeverRan, NullBinding}; - - fn outcome(output: serde_json::Value) -> RunOutcome { - RunOutcome { - output, - pending_approvals: Vec::new(), - cancelled: false, - } - } - - fn evidence<'a>(o: &'a RunOutcome, d: &'a Diagnosis) -> Evidence<'a> { - Evidence { - outcome: o, - diagnosis: d, - changed: String::new(), - failed: None, - } - } - - #[test] - fn a_parked_approval_needs_no_model() { - let mut o = outcome(json!({})); - o.pending_approvals = vec!["gate".into()]; - let d = Diagnosis::default(); - let verdict = without_a_model(&evidence(&o, &d)).expect("settled without a model"); - assert_eq!(verdict.blocker, Blocker::NeedsInput); - assert!( - verdict.advanced, - "reaching the gate is progress, not a stall" - ); - } - - #[test] - fn a_cancelled_run_did_not_fail_it_was_stopped() { - let mut o = outcome(json!({ "nodes": { "a": {} } })); - o.cancelled = true; - let d = Diagnosis::default(); - let verdict = without_a_model(&evidence(&o, &d)).expect("settled"); - assert_eq!(verdict.blocker, Blocker::ExternalWait); - assert!( - !verdict.blocker.continuable(), - "retrying now is not retrying later" - ); - } - - #[test] - fn a_run_where_nothing_ran_and_nothing_changed_is_terminal() { - let o = outcome(json!({})); - let d = Diagnosis { - never_ran: vec![NeverRan { - node_id: "work".into(), - routed_by: Some("gate".into()), - }], - ..Diagnosis::default() - }; - let verdict = without_a_model(&evidence(&o, &d)).expect("settled"); - assert_eq!(verdict.blocker, Blocker::MissingEvidence); - assert!(!verdict.blocker.continuable()); - } - - #[test] - fn a_run_that_produced_something_goes_to_the_model() { - let o = outcome(json!({ "nodes": { "a": { "items": [1] } } })); - let d = Diagnosis::default(); - assert!( - without_a_model(&evidence(&o, &d)).is_none(), - "a real outcome is a judgement, not a fact" - ); - } - - #[test] - fn an_unverifiable_null_binding_is_not_reported_as_a_finding() { - // The engine marks expressions it could not evaluate even in principle. - // Reporting those buries the ones that are real. - let o = outcome(json!({})); - let d = Diagnosis { - null_bindings: vec![NullBinding { - node_id: "a".into(), - location: "config.prompt".into(), - expression: "=nodes.x.item".into(), - unverifiable: true, - reads_from: None, - suggestion: "n/a".into(), - }], - ..Diagnosis::default() - }; - assert!(evidence(&o, &d).findings().is_empty()); - } - - #[test] - fn a_swallowed_error_reaches_the_judge() { - // The failure a naive reading misses entirely: the step is marked - // failed and its diagnostics are empty. - let o = outcome(json!({})); - let d = Diagnosis { - hidden_errors: vec![HiddenError { - node_id: "fetch".into(), - message: Some("404".into()), - }], - ..Diagnosis::default() - }; - let findings = evidence(&o, &d).findings(); - assert_eq!(findings.len(), 1); - assert!(findings[0].contains("swallowed"), "{findings:?}"); - assert!(findings[0].contains("404")); - } - - #[test] - fn a_null_binding_names_the_node_it_should_have_read_from() { - let o = outcome(json!({})); - let d = Diagnosis { - null_bindings: vec![NullBinding { - node_id: "review".into(), - location: "config.prompt".into(), - expression: "=nodes.fetch.item.body".into(), - unverifiable: false, - reads_from: Some("fetch".into()), - suggestion: "did you mean .item.json.body".into(), - }], - ..Diagnosis::default() - }; - let findings = evidence(&o, &d).findings(); - assert!(findings[0].contains("reading from `fetch`"), "{findings:?}"); - assert!( - findings[0].contains("item.json.body"), - "the suggestion carries" - ); - } - - #[test] - fn a_clean_run_says_so_rather_than_showing_an_empty_list() { - let o = outcome(json!({ "nodes": {} })); - let d = Diagnosis::default(); - assert!(evidence(&o, &d).render().contains("found nothing wrong")); - } -} +#[path = "judge_tests.rs"] +mod tests; diff --git a/crates/tinyflows-adaptive/src/closing/judge_tests.rs b/crates/tinyflows-adaptive/src/closing/judge_tests.rs new file mode 100644 index 00000000..9f6e4296 --- /dev/null +++ b/crates/tinyflows-adaptive/src/closing/judge_tests.rs @@ -0,0 +1,210 @@ +use super::*; + +/// A provider that answers the judge with a fixed blocker. +struct Says(&'static str); + +#[async_trait::async_trait] +impl tinyflows::caps::LlmProvider for Says { + async fn complete( + &self, + _request: serde_json::Value, + _conn: Option<&str>, + ) -> tinyflows::error::Result { + Ok(serde_json::json!({ + "satisfied": false, + "blocker": self.0, + "gap": "nothing was fetched", + })) + } +} + +async fn verdict_for(blocker: &'static str, failed: Option) -> Verdict { + let outcome = tinyflows::engine::RunOutcome { + // Non-empty: the mechanical pre-judge must not settle this one, + // because the point is what the MODEL's answer becomes. + output: serde_json::json!({ "nodes": { "fetch": { "json": 1 } } }), + pending_approvals: Vec::new(), + cancelled: false, + }; + let diagnosis = Diagnosis::default(); + let evidence = Evidence { + outcome: &outcome, + diagnosis: &diagnosis, + changed: String::new(), + failed, + }; + let caps = tinyflows::caps::Capabilities { + llm: std::sync::Arc::new(Says(blocker)), + ..tinyflows::caps::mock::mock_capabilities() + }; + judge(&Goal::new("do the thing"), &evidence, &caps, None) + .await + .expect("judged") +} + +#[tokio::test] +async fn a_mechanically_broken_run_cannot_be_called_terminal() { + // Field observation: a shell step exited nonzero, the judge answered + // `missing_evidence`, and the episode ended with two of its three + // attempts unused — when rewriting the script was the whole fix. + // The prompt says mechanical failures are goal_not_met; a model that + // misreads it must not get to end the episode anyway. + let verdict = verdict_for("missing_evidence", Some("script exited 5".into())).await; + assert_eq!(verdict.blocker, Blocker::GoalNotMet); + assert!(verdict.blocker.continuable()); +} + +#[tokio::test] +async fn a_run_that_completed_keeps_the_judges_terminal_verdict() { + // No mechanical failure: the judge is the authority on whether + // another attempt could help, and this downgrade must not become a + // blanket refusal to ever stand down. + let verdict = verdict_for("missing_evidence", None).await; + assert_eq!(verdict.blocker, Blocker::MissingEvidence); +} + +#[tokio::test] +async fn a_broken_run_still_waiting_on_a_person_stays_terminal() { + // NeedsInput and ExternalWait survive the downgrade: both mean + // something OUTSIDE the loop must move, which a broken run does not + // change. + let verdict = verdict_for("needs_input", Some("script exited 5".into())).await; + assert_eq!(verdict.blocker, Blocker::NeedsInput); +} + +use serde_json::json; +use tinyflows::diagnostics::{HiddenError, NeverRan, NullBinding}; + +fn outcome(output: serde_json::Value) -> RunOutcome { + RunOutcome { + output, + pending_approvals: Vec::new(), + cancelled: false, + } +} + +fn evidence<'a>(o: &'a RunOutcome, d: &'a Diagnosis) -> Evidence<'a> { + Evidence { + outcome: o, + diagnosis: d, + changed: String::new(), + failed: None, + } +} + +#[test] +fn a_parked_approval_needs_no_model() { + let mut o = outcome(json!({})); + o.pending_approvals = vec!["gate".into()]; + let d = Diagnosis::default(); + let verdict = without_a_model(&evidence(&o, &d)).expect("settled without a model"); + assert_eq!(verdict.blocker, Blocker::NeedsInput); + assert!( + verdict.advanced, + "reaching the gate is progress, not a stall" + ); +} + +#[test] +fn a_cancelled_run_did_not_fail_it_was_stopped() { + let mut o = outcome(json!({ "nodes": { "a": {} } })); + o.cancelled = true; + let d = Diagnosis::default(); + let verdict = without_a_model(&evidence(&o, &d)).expect("settled"); + assert_eq!(verdict.blocker, Blocker::ExternalWait); + assert!( + !verdict.blocker.continuable(), + "retrying now is not retrying later" + ); +} + +#[test] +fn a_run_where_nothing_ran_and_nothing_changed_is_terminal() { + let o = outcome(json!({})); + let d = Diagnosis { + never_ran: vec![NeverRan { + node_id: "work".into(), + routed_by: Some("gate".into()), + }], + ..Diagnosis::default() + }; + let verdict = without_a_model(&evidence(&o, &d)).expect("settled"); + assert_eq!(verdict.blocker, Blocker::MissingEvidence); + assert!(!verdict.blocker.continuable()); +} + +#[test] +fn a_run_that_produced_something_goes_to_the_model() { + let o = outcome(json!({ "nodes": { "a": { "items": [1] } } })); + let d = Diagnosis::default(); + assert!( + without_a_model(&evidence(&o, &d)).is_none(), + "a real outcome is a judgement, not a fact" + ); +} + +#[test] +fn an_unverifiable_null_binding_is_not_reported_as_a_finding() { + // The engine marks expressions it could not evaluate even in principle. + // Reporting those buries the ones that are real. + let o = outcome(json!({})); + let d = Diagnosis { + null_bindings: vec![NullBinding { + node_id: "a".into(), + location: "config.prompt".into(), + expression: "=nodes.x.item".into(), + unverifiable: true, + reads_from: None, + suggestion: "n/a".into(), + }], + ..Diagnosis::default() + }; + assert!(evidence(&o, &d).findings().is_empty()); +} + +#[test] +fn a_swallowed_error_reaches_the_judge() { + // The failure a naive reading misses entirely: the step is marked + // failed and its diagnostics are empty. + let o = outcome(json!({})); + let d = Diagnosis { + hidden_errors: vec![HiddenError { + node_id: "fetch".into(), + message: Some("404".into()), + }], + ..Diagnosis::default() + }; + let findings = evidence(&o, &d).findings(); + assert_eq!(findings.len(), 1); + assert!(findings[0].contains("swallowed"), "{findings:?}"); + assert!(findings[0].contains("404")); +} + +#[test] +fn a_null_binding_names_the_node_it_should_have_read_from() { + let o = outcome(json!({})); + let d = Diagnosis { + null_bindings: vec![NullBinding { + node_id: "review".into(), + location: "config.prompt".into(), + expression: "=nodes.fetch.item.body".into(), + unverifiable: false, + reads_from: Some("fetch".into()), + suggestion: "did you mean .item.json.body".into(), + }], + ..Diagnosis::default() + }; + let findings = evidence(&o, &d).findings(); + assert!(findings[0].contains("reading from `fetch`"), "{findings:?}"); + assert!( + findings[0].contains("item.json.body"), + "the suggestion carries" + ); +} + +#[test] +fn a_clean_run_says_so_rather_than_showing_an_empty_list() { + let o = outcome(json!({ "nodes": {} })); + let d = Diagnosis::default(); + assert!(evidence(&o, &d).render().contains("found nothing wrong")); +} diff --git a/crates/tinyflows-adaptive/src/contracts.rs b/crates/tinyflows-adaptive/src/contracts.rs index d29a532d..ae23260f 100644 --- a/crates/tinyflows-adaptive/src/contracts.rs +++ b/crates/tinyflows-adaptive/src/contracts.rs @@ -308,176 +308,8 @@ impl Approach { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn two_errands_in_one_episode_are_visibly_the_same_attempt() { - // Unlike two authored graphs, which may genuinely differ and are told - // apart by their fingerprints. An errand carries no plan to differ in, - // so the constant signature is what puts it in the exclusion list and - // stops an episode spending its budget on identical single turns. - let first = Approach::Errand { - why: "one turn of work".into(), - }; - let second = Approach::Errand { - why: "still one turn, honestly".into(), - }; - assert_eq!(first.signature(), "errand"); - assert_eq!(first.signature(), second.signature()); - } - - #[test] - fn an_errand_cannot_collide_with_a_stored_workflow_called_errand() { - // The namespacing that makes the constant safe: `selected:` prefixes - // every workflow id, so no shelf entry can occupy the errand slot. - let selected = Approach::Selected { - workflow_id: "errand".into(), - why: String::new(), - }; - assert_eq!(selected.signature(), "selected:errand"); - assert_ne!( - selected.signature(), - Approach::Errand { why: String::new() }.signature() - ); - } - - #[test] - fn an_unrecognised_blocker_is_continuable_rather_than_terminal() { - // `goal_not_meet` — one letter — used to end a run at attempt 3 of 12. - assert_eq!(Blocker::parse("goal_not_meet"), Blocker::GoalNotMet); - assert_eq!(Blocker::parse("something new"), Blocker::GoalNotMet); - assert!(Blocker::parse("nonsense").continuable()); - } - - #[test] - fn the_terminal_blockers_stop_a_run() { - assert!(!Blocker::MissingEvidence.continuable()); - assert!(!Blocker::NeedsInput.continuable()); - assert!(!Blocker::ExternalWait.continuable()); - } - - #[test] - fn an_empty_blocker_reads_as_no_blocker() { - assert_eq!(Blocker::parse(""), Blocker::None); - } - - fn verdict(satisfied: bool, blocker: Blocker) -> Verdict { - Verdict { - satisfied, - blocker, - gap: String::new(), - attributed_to: String::new(), - evidence: String::new(), - advanced: false, - } - } - - #[test] - fn the_stall_rule_does_not_apply_before_min_attempts() { - // Early attempts look flat while a run is still orienting. - let budget = Budget::default(); - let v = verdict(false, Blocker::GoalNotMet); - assert!( - v.should_retry(1, 5, &budget), - "attempt 1 must not be stalled out" - ); - assert!(v.should_retry(2, 5, &budget)); - assert!( - !v.should_retry(3, 2, &budget), - "past min_attempts the rule bites" - ); - } - - #[test] - fn a_converging_run_is_not_killed_by_the_counter() { - let budget = Budget::default(); - let mut v = verdict(false, Blocker::GoalNotMet); - v.advanced = true; - // `stalled` is reset by the caller on every advancing attempt, so a run - // that keeps advancing never accumulates one. - assert!(v.should_retry(9, 0, &budget)); - } - - #[test] - fn a_satisfied_verdict_never_retries() { - assert!(!verdict(true, Blocker::None).should_retry(1, 0, &Budget::default())); - } - - #[test] - fn a_terminal_blocker_stops_even_with_budget_left() { - let v = verdict(false, Blocker::NeedsInput); - assert!(!v.should_retry(1, 0, &Budget::default())); - } - - #[test] - fn the_attempt_ceiling_is_still_a_backstop() { - let budget = Budget::default(); - let mut v = verdict(false, Blocker::GoalNotMet); - v.advanced = true; - assert!(!v.should_retry(12, 0, &budget)); - } - - #[test] - fn a_signature_names_the_kind_of_attempt_not_the_task() { - let selected = Approach::Selected { - workflow_id: "pr-review".into(), - why: "matches".into(), - }; - assert_eq!(selected.signature(), "selected:pr-review"); - } - - #[test] - fn two_authored_attempts_are_told_apart_by_their_graph() { - // Before the fingerprint every authoring attempt signed as "authored", - // `tried()` folded them to one entry, and attempt four could re-author - // attempt two word for word with nothing to notice. - let first = Approach::Authored { - why: "nothing fitted".into(), - fingerprint: "1111111".into(), - }; - let second = Approach::Authored { - why: "still nothing fitted".into(), - fingerprint: "2222222".into(), - }; - assert_ne!(first.signature(), second.signature()); - assert_eq!(first.signature(), "authored:1111111"); - } - - #[test] - fn the_same_graph_authored_twice_signs_the_same_and_is_caught() { - // The other half: a differently-worded `why` around an identical graph - // is the same attempt, and must read as the repeat it is. - let first = Approach::Authored { - why: "nothing fitted".into(), - fingerprint: "1111111".into(), - }; - let again = Approach::Authored { - why: "a fresh idea, honestly".into(), - fingerprint: "1111111".into(), - }; - assert_eq!(first.signature(), again.signature()); - } - - #[test] - fn a_verdict_round_trips_through_json() { - // The judge answers in JSON and the ledger stores JSON; a field lost in - // either direction is one that works in a test and never in a run. - let v = verdict(false, Blocker::Unverified); - let back: Verdict = serde_json::from_str(&serde_json::to_string(&v).unwrap()).unwrap(); - assert_eq!(back.blocker, Blocker::Unverified); - assert!(!back.advanced); - } - - #[test] - fn advanced_defaults_to_true_when_a_model_omits_it() { - // Absent must not read as "made no progress" — that would stall a run - // for a field the model simply did not write. - let v: Verdict = - serde_json::from_str(r#"{"satisfied":false,"blocker":"goal_not_met"}"#).unwrap(); - assert!(v.advanced); - } -} +#[path = "contracts_tests.rs"] +mod tests; /// Where a failed run stopped, so a later attempt can carry on from it. /// diff --git a/crates/tinyflows-adaptive/src/contracts_tests.rs b/crates/tinyflows-adaptive/src/contracts_tests.rs new file mode 100644 index 00000000..24d08bd8 --- /dev/null +++ b/crates/tinyflows-adaptive/src/contracts_tests.rs @@ -0,0 +1,168 @@ +use super::*; + +#[test] +fn two_errands_in_one_episode_are_visibly_the_same_attempt() { + // Unlike two authored graphs, which may genuinely differ and are told + // apart by their fingerprints. An errand carries no plan to differ in, + // so the constant signature is what puts it in the exclusion list and + // stops an episode spending its budget on identical single turns. + let first = Approach::Errand { + why: "one turn of work".into(), + }; + let second = Approach::Errand { + why: "still one turn, honestly".into(), + }; + assert_eq!(first.signature(), "errand"); + assert_eq!(first.signature(), second.signature()); +} + +#[test] +fn an_errand_cannot_collide_with_a_stored_workflow_called_errand() { + // The namespacing that makes the constant safe: `selected:` prefixes + // every workflow id, so no shelf entry can occupy the errand slot. + let selected = Approach::Selected { + workflow_id: "errand".into(), + why: String::new(), + }; + assert_eq!(selected.signature(), "selected:errand"); + assert_ne!( + selected.signature(), + Approach::Errand { why: String::new() }.signature() + ); +} + +#[test] +fn an_unrecognised_blocker_is_continuable_rather_than_terminal() { + // `goal_not_meet` — one letter — used to end a run at attempt 3 of 12. + assert_eq!(Blocker::parse("goal_not_meet"), Blocker::GoalNotMet); + assert_eq!(Blocker::parse("something new"), Blocker::GoalNotMet); + assert!(Blocker::parse("nonsense").continuable()); +} + +#[test] +fn the_terminal_blockers_stop_a_run() { + assert!(!Blocker::MissingEvidence.continuable()); + assert!(!Blocker::NeedsInput.continuable()); + assert!(!Blocker::ExternalWait.continuable()); +} + +#[test] +fn an_empty_blocker_reads_as_no_blocker() { + assert_eq!(Blocker::parse(""), Blocker::None); +} + +fn verdict(satisfied: bool, blocker: Blocker) -> Verdict { + Verdict { + satisfied, + blocker, + gap: String::new(), + attributed_to: String::new(), + evidence: String::new(), + advanced: false, + } +} + +#[test] +fn the_stall_rule_does_not_apply_before_min_attempts() { + // Early attempts look flat while a run is still orienting. + let budget = Budget::default(); + let v = verdict(false, Blocker::GoalNotMet); + assert!( + v.should_retry(1, 5, &budget), + "attempt 1 must not be stalled out" + ); + assert!(v.should_retry(2, 5, &budget)); + assert!( + !v.should_retry(3, 2, &budget), + "past min_attempts the rule bites" + ); +} + +#[test] +fn a_converging_run_is_not_killed_by_the_counter() { + let budget = Budget::default(); + let mut v = verdict(false, Blocker::GoalNotMet); + v.advanced = true; + // `stalled` is reset by the caller on every advancing attempt, so a run + // that keeps advancing never accumulates one. + assert!(v.should_retry(9, 0, &budget)); +} + +#[test] +fn a_satisfied_verdict_never_retries() { + assert!(!verdict(true, Blocker::None).should_retry(1, 0, &Budget::default())); +} + +#[test] +fn a_terminal_blocker_stops_even_with_budget_left() { + let v = verdict(false, Blocker::NeedsInput); + assert!(!v.should_retry(1, 0, &Budget::default())); +} + +#[test] +fn the_attempt_ceiling_is_still_a_backstop() { + let budget = Budget::default(); + let mut v = verdict(false, Blocker::GoalNotMet); + v.advanced = true; + assert!(!v.should_retry(12, 0, &budget)); +} + +#[test] +fn a_signature_names_the_kind_of_attempt_not_the_task() { + let selected = Approach::Selected { + workflow_id: "pr-review".into(), + why: "matches".into(), + }; + assert_eq!(selected.signature(), "selected:pr-review"); +} + +#[test] +fn two_authored_attempts_are_told_apart_by_their_graph() { + // Before the fingerprint every authoring attempt signed as "authored", + // `tried()` folded them to one entry, and attempt four could re-author + // attempt two word for word with nothing to notice. + let first = Approach::Authored { + why: "nothing fitted".into(), + fingerprint: "1111111".into(), + }; + let second = Approach::Authored { + why: "still nothing fitted".into(), + fingerprint: "2222222".into(), + }; + assert_ne!(first.signature(), second.signature()); + assert_eq!(first.signature(), "authored:1111111"); +} + +#[test] +fn the_same_graph_authored_twice_signs_the_same_and_is_caught() { + // The other half: a differently-worded `why` around an identical graph + // is the same attempt, and must read as the repeat it is. + let first = Approach::Authored { + why: "nothing fitted".into(), + fingerprint: "1111111".into(), + }; + let again = Approach::Authored { + why: "a fresh idea, honestly".into(), + fingerprint: "1111111".into(), + }; + assert_eq!(first.signature(), again.signature()); +} + +#[test] +fn a_verdict_round_trips_through_json() { + // The judge answers in JSON and the ledger stores JSON; a field lost in + // either direction is one that works in a test and never in a run. + let v = verdict(false, Blocker::Unverified); + let back: Verdict = serde_json::from_str(&serde_json::to_string(&v).unwrap()).unwrap(); + assert_eq!(back.blocker, Blocker::Unverified); + assert!(!back.advanced); +} + +#[test] +fn advanced_defaults_to_true_when_a_model_omits_it() { + // Absent must not read as "made no progress" — that would stall a run + // for a field the model simply did not write. + let v: Verdict = + serde_json::from_str(r#"{"satisfied":false,"blocker":"goal_not_met"}"#).unwrap(); + assert!(v.advanced); +} diff --git a/crates/tinyflows-adaptive/src/driver.rs b/crates/tinyflows-adaptive/src/driver.rs index 057d5881..07040d0d 100644 --- a/crates/tinyflows-adaptive/src/driver.rs +++ b/crates/tinyflows-adaptive/src/driver.rs @@ -384,154 +384,5 @@ impl Loop<'_> { } #[cfg(test)] -mod tests { - use super::*; - - struct Frozen; - impl Clock for Frozen { - fn now(&self) -> String { - "2026-01-01T00:00:00Z".to_string() - } - } - - #[tokio::test] - async fn starting_an_episode_twice_does_not_restart_it() { - // A service that retries a create must not reset a goal four attempts - // in — the rows would stay and the counters would not, which reads as - // progress that never happened. - let ledger = crate::ledger::memory::MemoryLedger::new(); - let goal = Goal::new("write the weekly report"); - - let mut record = Episode { - id: "ep-1".into(), - goal: goal.clone(), - scope_key: None, - status: EpisodeStatus::Running, - attempt: 4, - stalled: 2, - started_at: "2026-01-01T00:00:00Z".into(), - updated_at: "2026-01-01T00:00:00Z".into(), - }; - ledger.save_episode(&record).await.expect("save"); - - // `start` short-circuits on an existing record, so this is what it sees. - let seen = ledger.episode("ep-1").await.expect("read").expect("exists"); - assert_eq!(seen.attempt, 4); - assert_eq!(seen.stalled, 2); - - record.attempt = 5; - ledger.save_episode(&record).await.expect("save"); - assert_eq!( - ledger - .episode("ep-1") - .await - .expect("read") - .expect("exists") - .attempt, - 5, - "a save updates rather than duplicating" - ); - } - - #[tokio::test] - async fn only_running_episodes_are_offered_for_recovery() { - let ledger = crate::ledger::memory::MemoryLedger::new(); - for (id, status) in [ - ("ep-live", EpisodeStatus::Running), - ("ep-done", EpisodeStatus::Satisfied), - ( - "ep-gave-up", - EpisodeStatus::StoodDown("out of attempts".into()), - ), - ] { - ledger - .save_episode(&Episode { - id: id.into(), - goal: Goal::new("something"), - scope_key: None, - status, - attempt: 1, - stalled: 0, - started_at: "2026-01-01T00:00:00Z".into(), - updated_at: "2026-01-01T00:00:00Z".into(), - }) - .await - .expect("save"); - } - - let running = ledger.episodes(true, Page::ALL).await.expect("episodes"); - assert_eq!(running.len(), 1); - assert_eq!(running[0].id, "ep-live"); - assert_eq!( - ledger - .episodes(false, Page::ALL) - .await - .expect("episodes") - .len(), - 3 - ); - } - - #[tokio::test] - async fn an_episode_round_trips_its_goal_and_its_reason_for_stopping() { - // Both are unrecoverable from the rows, which is the whole test for - // what belongs on the record. - let ledger = crate::ledger::memory::MemoryLedger::new(); - let mut goal = Goal::new("write the weekly report"); - goal.success_criteria = "cites the actual figures".into(); - - ledger - .save_episode(&Episode { - id: "ep-2".into(), - goal, - scope_key: None, - status: EpisodeStatus::StoodDown("3 attempts in a row made no progress".into()), - attempt: 7, - stalled: 3, - started_at: Frozen.now(), - updated_at: Frozen.now(), - }) - .await - .expect("save"); - - let back = ledger.episode("ep-2").await.expect("read").expect("exists"); - assert_eq!(back.goal.text, "write the weekly report"); - assert_eq!(back.goal.success_criteria, "cites the actual figures"); - assert_eq!(back.stalled, 3); - match back.status { - EpisodeStatus::StoodDown(reason) => assert!(reason.contains("no progress")), - other => panic!("expected a stand-down, got {other:?}"), - } - } - - #[tokio::test] - async fn one_tenants_episodes_are_invisible_to_another() { - let ledger = crate::ledger::memory::MemoryLedger::new(); - let a = ledger.for_tenant("user-a"); - let b = ledger.for_tenant("user-b"); - a.save_episode(&Episode { - id: "ep-private".into(), - goal: Goal::new("something of mine"), - scope_key: None, - status: EpisodeStatus::Running, - attempt: 1, - stalled: 0, - started_at: Frozen.now(), - updated_at: Frozen.now(), - }) - .await - .expect("save"); - - assert!(a.episode("ep-private").await.expect("read").is_some()); - assert!( - b.episode("ep-private").await.expect("read").is_none(), - "an episode carries a goal in the user's own words" - ); - assert!( - b.episodes(false, Page::ALL) - .await - .expect("episodes") - .is_empty() - ); - } -} +#[path = "driver_tests.rs"] +mod tests; diff --git a/crates/tinyflows-adaptive/src/driver_tests.rs b/crates/tinyflows-adaptive/src/driver_tests.rs new file mode 100644 index 00000000..d06c4c8b --- /dev/null +++ b/crates/tinyflows-adaptive/src/driver_tests.rs @@ -0,0 +1,149 @@ +use super::*; + +struct Frozen; +impl Clock for Frozen { + fn now(&self) -> String { + "2026-01-01T00:00:00Z".to_string() + } +} + +#[tokio::test] +async fn starting_an_episode_twice_does_not_restart_it() { + // A service that retries a create must not reset a goal four attempts + // in — the rows would stay and the counters would not, which reads as + // progress that never happened. + let ledger = crate::ledger::memory::MemoryLedger::new(); + let goal = Goal::new("write the weekly report"); + + let mut record = Episode { + id: "ep-1".into(), + goal: goal.clone(), + scope_key: None, + status: EpisodeStatus::Running, + attempt: 4, + stalled: 2, + started_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + }; + ledger.save_episode(&record).await.expect("save"); + + // `start` short-circuits on an existing record, so this is what it sees. + let seen = ledger.episode("ep-1").await.expect("read").expect("exists"); + assert_eq!(seen.attempt, 4); + assert_eq!(seen.stalled, 2); + + record.attempt = 5; + ledger.save_episode(&record).await.expect("save"); + assert_eq!( + ledger + .episode("ep-1") + .await + .expect("read") + .expect("exists") + .attempt, + 5, + "a save updates rather than duplicating" + ); +} + +#[tokio::test] +async fn only_running_episodes_are_offered_for_recovery() { + let ledger = crate::ledger::memory::MemoryLedger::new(); + for (id, status) in [ + ("ep-live", EpisodeStatus::Running), + ("ep-done", EpisodeStatus::Satisfied), + ( + "ep-gave-up", + EpisodeStatus::StoodDown("out of attempts".into()), + ), + ] { + ledger + .save_episode(&Episode { + id: id.into(), + goal: Goal::new("something"), + scope_key: None, + status, + attempt: 1, + stalled: 0, + started_at: "2026-01-01T00:00:00Z".into(), + updated_at: "2026-01-01T00:00:00Z".into(), + }) + .await + .expect("save"); + } + + let running = ledger.episodes(true, Page::ALL).await.expect("episodes"); + assert_eq!(running.len(), 1); + assert_eq!(running[0].id, "ep-live"); + assert_eq!( + ledger + .episodes(false, Page::ALL) + .await + .expect("episodes") + .len(), + 3 + ); +} + +#[tokio::test] +async fn an_episode_round_trips_its_goal_and_its_reason_for_stopping() { + // Both are unrecoverable from the rows, which is the whole test for + // what belongs on the record. + let ledger = crate::ledger::memory::MemoryLedger::new(); + let mut goal = Goal::new("write the weekly report"); + goal.success_criteria = "cites the actual figures".into(); + + ledger + .save_episode(&Episode { + id: "ep-2".into(), + goal, + scope_key: None, + status: EpisodeStatus::StoodDown("3 attempts in a row made no progress".into()), + attempt: 7, + stalled: 3, + started_at: Frozen.now(), + updated_at: Frozen.now(), + }) + .await + .expect("save"); + + let back = ledger.episode("ep-2").await.expect("read").expect("exists"); + assert_eq!(back.goal.text, "write the weekly report"); + assert_eq!(back.goal.success_criteria, "cites the actual figures"); + assert_eq!(back.stalled, 3); + match back.status { + EpisodeStatus::StoodDown(reason) => assert!(reason.contains("no progress")), + other => panic!("expected a stand-down, got {other:?}"), + } +} + +#[tokio::test] +async fn one_tenants_episodes_are_invisible_to_another() { + let ledger = crate::ledger::memory::MemoryLedger::new(); + let a = ledger.for_tenant("user-a"); + let b = ledger.for_tenant("user-b"); + a.save_episode(&Episode { + id: "ep-private".into(), + goal: Goal::new("something of mine"), + scope_key: None, + status: EpisodeStatus::Running, + attempt: 1, + stalled: 0, + started_at: Frozen.now(), + updated_at: Frozen.now(), + }) + .await + .expect("save"); + + assert!(a.episode("ep-private").await.expect("read").is_some()); + assert!( + b.episode("ep-private").await.expect("read").is_none(), + "an episode carries a goal in the user's own words" + ); + assert!( + b.episodes(false, Page::ALL) + .await + .expect("episodes") + .is_empty() + ); +} diff --git a/crates/tinyflows-adaptive/src/host.rs b/crates/tinyflows-adaptive/src/host.rs index f302c375..6381bf26 100644 --- a/crates/tinyflows-adaptive/src/host.rs +++ b/crates/tinyflows-adaptive/src/host.rs @@ -379,300 +379,5 @@ fn host_of(url: &str) -> Option<&str> { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_host_that_configured_any_rendered_fact_is_not_unknown() { - // `is_unknown` must test every field `render` prints: a fact it skips - // is one that silently never reaches the authoring prompt. - for facts in [ - HostFacts { - default_harness: Some("codex".into()), - ..HostFacts::unknown() - }, - HostFacts { - default_model: Some("gpt-5".into()), - ..HostFacts::unknown() - }, - HostFacts { - max_parallel_agents: Some(2), - ..HostFacts::unknown() - }, - HostFacts { - run_timeout_secs: Some(600), - ..HostFacts::unknown() - }, - HostFacts { - tools: vec![ToolFact { - slug: "host:shell".into(), - args: "`script` (inline) or `script_path`".into(), - }], - ..HostFacts::unknown() - }, - ] { - assert!(!facts.is_unknown(), "{facts:?}"); - assert!(!facts.render().is_empty(), "and it renders"); - } - } - - #[test] - fn host_names_compare_case_insensitively() { - // DNS is case-insensitive; `API.GitHub.com` against `github.com` must - // not cost the episode a spurious authoring round. - let facts = HostFacts { - http_allowlist: vec!["github.com".into()], - ..HostFacts::unknown() - }; - let graph = graph(vec![node( - "fetch", - NodeKind::HttpRequest, - serde_json::json!({ "url": "https://API.GitHub.com/repos/x", "method": "GET" }), - )]); - assert!(facts.check(&graph).is_empty(), "{:?}", facts.check(&graph)); - } - use serde_json::json; - use tinyflows::model::Node; - - fn node(id: &str, kind: NodeKind, config: Value) -> Node { - Node { - id: id.into(), - kind, - type_version: 1, - name: id.into(), - config, - ports: Vec::new(), - position: None, - } - } - - fn graph(nodes: Vec) -> WorkflowGraph { - WorkflowGraph { - nodes, - ..WorkflowGraph::default() - } - } - - #[test] - fn a_host_that_has_said_nothing_refuses_nothing() { - // The reading that would break every unconfigured deployment: empty - // meaning "deny" rather than "unknown". - let facts = HostFacts::unknown(); - let g = graph(vec![ - node("a", NodeKind::Agent, json!({ "agent_ref": "anyone" })), - node( - "t", - NodeKind::ToolCall, - json!({ "slug": "anything:at:all" }), - ), - node( - "c", - NodeKind::Code, - json!({ "language": "python", "source": "1" }), - ), - ]); - assert!(facts.check(&g).is_empty()); - assert!( - facts.render().is_empty(), - "nothing known renders as nothing" - ); - } - - #[test] - fn a_worker_this_host_does_not_have_is_named() { - let facts = HostFacts { - workers: vec!["laptop".into(), "ci".into()], - ..HostFacts::unknown() - }; - let problems = facts.check(&graph(vec![node( - "a", - NodeKind::Agent, - json!({ "agent_ref": "desktop" }), - )])); - assert_eq!(problems.len(), 1); - assert!(problems[0].contains("desktop"), "{problems:?}"); - assert!( - problems[0].contains("laptop, ci"), - "the alternatives are offered" - ); - } - - #[test] - fn no_default_worker_makes_agent_ref_mandatory() { - // A host fact that changes a field from optional to required. - let facts = HostFacts { - workers: vec!["laptop".into()], - default_worker: None, - ..HostFacts::unknown() - }; - let problems = facts.check(&graph(vec![node("a", NodeKind::Agent, json!({}))])); - assert_eq!(problems.len(), 1); - assert!(problems[0].contains("must name"), "{problems:?}"); - } - - #[test] - fn a_default_worker_makes_a_bare_agent_node_fine() { - let facts = HostFacts { - workers: vec!["laptop".into()], - default_worker: Some("laptop".into()), - ..HostFacts::unknown() - }; - assert!( - facts - .check(&graph(vec![node("a", NodeKind::Agent, json!({}))])) - .is_empty() - ); - } - - #[test] - fn a_slug_outside_both_lists_is_refused() { - let facts = HostFacts { - native_tools: vec!["medulla:shell".into()], - tool_allowlist: vec!["github".into()], - ..HostFacts::unknown() - }; - let g = graph(vec![ - node("ok", NodeKind::ToolCall, json!({ "slug": "medulla:shell" })), - node("no", NodeKind::ToolCall, json!({ "slug": "slack" })), - ]); - let problems = facts.check(&g); - assert_eq!(problems.len(), 1); - assert!(problems[0].contains("slack"), "{problems:?}"); - } - - #[test] - fn an_http_host_outside_the_allowlist_is_refused_but_a_subdomain_is_not() { - let facts = HostFacts { - http_allowlist: vec!["github.com".into()], - ..HostFacts::unknown() - }; - let g = graph(vec![ - node( - "ok", - NodeKind::HttpRequest, - json!({ "url": "https://api.github.com/x" }), - ), - node( - "no", - NodeKind::HttpRequest, - json!({ "url": "https://evil.test/x" }), - ), - ]); - let problems = facts.check(&g); - assert_eq!(problems.len(), 1, "{problems:?}"); - assert!(problems[0].contains("evil.test")); - } - - #[test] - fn a_url_built_from_an_expression_is_left_to_run_time() { - // Refusing it would refuse the correct way to write a parameterised - // request, which is the thing the authoring prompt asks for. - let facts = HostFacts { - http_allowlist: vec!["github.com".into()], - ..HostFacts::unknown() - }; - let g = graph(vec![node( - "u", - NodeKind::HttpRequest, - json!({ "url": "=\"https://\" + .inputs.host" }), - )]); - assert!(facts.check(&g).is_empty()); - } - - #[test] - fn disabled_code_and_refused_shell_are_both_reported() { - let facts = HostFacts { - allow_code: Some(false), - shell_available: Some(false), - ..HostFacts::unknown() - }; - let g = graph(vec![ - node( - "c", - NodeKind::Code, - json!({ "language": "python", "source": "1" }), - ), - node("s", NodeKind::Shell, json!({ "script": "ls" })), - ]); - assert_eq!( - facts.check(&g).len(), - 2, - "every failure at once, not the first" - ); - } - - #[test] - fn a_loop_above_the_host_ceiling_is_reported() { - // Otherwise it silently stops earlier than the graph says. - let facts = HostFacts { - max_loop_iterations: Some(10), - ..HostFacts::unknown() - }; - let g = graph(vec![node( - "l", - NodeKind::Loop, - json!({ "max_iterations": 50 }), - )]); - let problems = facts.check(&g); - assert_eq!(problems.len(), 1); - assert!(problems[0].contains("ceiling of 10"), "{problems:?}"); - } - - #[test] - fn a_trigger_kind_that_never_fires_is_reported() { - let facts = HostFacts { - trigger_kinds: vec!["manual".into()], - ..HostFacts::unknown() - }; - let g = graph(vec![node( - "t", - NodeKind::Trigger, - json!({ "trigger_kind": "schedule" }), - )]); - let problems = facts.check(&g); - assert_eq!(problems.len(), 1); - assert!(problems[0].contains("never dispatched"), "{problems:?}"); - } - - #[test] - fn a_tool_fact_renders_its_argument_shape_into_the_prompt() { - let facts = HostFacts { - native_tools: vec!["host:shell".into()], - tools: vec![ToolFact { - slug: "host:shell".into(), - args: "`script` (inline text) or `script_path` (a file); NOT `command`".into(), - }], - ..HostFacts::unknown() - }; - let rendered = facts.render(); - assert!( - rendered.contains("tool `host:shell` args:") && rendered.contains("script_path"), - "{rendered}" - ); - } - - #[test] - fn the_rendering_states_consequences_not_just_values() { - let facts = HostFacts { - default_worker: None, - workers: vec!["laptop".into()], - allow_code: Some(false), - notes: vec!["Only manual triggers fire here.".into()], - ..HostFacts::unknown() - }; - let rendered = facts.render(); - assert!(rendered.contains("every agent node must name config.agent_ref")); - assert!(rendered.contains("DISABLED")); - assert!(rendered.contains("Only manual triggers fire here.")); - } - - #[test] - fn a_url_without_a_scheme_still_yields_its_host() { - assert_eq!(host_of("api.github.com/x"), Some("api.github.com")); - assert_eq!( - host_of("https://user:pw@api.github.com:443/x"), - Some("api.github.com") - ); - assert_eq!(host_of(""), None); - } -} +#[path = "host_tests.rs"] +mod tests; diff --git a/crates/tinyflows-adaptive/src/host_tests.rs b/crates/tinyflows-adaptive/src/host_tests.rs new file mode 100644 index 00000000..ff0d2504 --- /dev/null +++ b/crates/tinyflows-adaptive/src/host_tests.rs @@ -0,0 +1,295 @@ +use super::*; + +#[test] +fn a_host_that_configured_any_rendered_fact_is_not_unknown() { + // `is_unknown` must test every field `render` prints: a fact it skips + // is one that silently never reaches the authoring prompt. + for facts in [ + HostFacts { + default_harness: Some("codex".into()), + ..HostFacts::unknown() + }, + HostFacts { + default_model: Some("gpt-5".into()), + ..HostFacts::unknown() + }, + HostFacts { + max_parallel_agents: Some(2), + ..HostFacts::unknown() + }, + HostFacts { + run_timeout_secs: Some(600), + ..HostFacts::unknown() + }, + HostFacts { + tools: vec![ToolFact { + slug: "host:shell".into(), + args: "`script` (inline) or `script_path`".into(), + }], + ..HostFacts::unknown() + }, + ] { + assert!(!facts.is_unknown(), "{facts:?}"); + assert!(!facts.render().is_empty(), "and it renders"); + } +} + +#[test] +fn host_names_compare_case_insensitively() { + // DNS is case-insensitive; `API.GitHub.com` against `github.com` must + // not cost the episode a spurious authoring round. + let facts = HostFacts { + http_allowlist: vec!["github.com".into()], + ..HostFacts::unknown() + }; + let graph = graph(vec![node( + "fetch", + NodeKind::HttpRequest, + serde_json::json!({ "url": "https://API.GitHub.com/repos/x", "method": "GET" }), + )]); + assert!(facts.check(&graph).is_empty(), "{:?}", facts.check(&graph)); +} +use serde_json::json; +use tinyflows::model::Node; + +fn node(id: &str, kind: NodeKind, config: Value) -> Node { + Node { + id: id.into(), + kind, + type_version: 1, + name: id.into(), + config, + ports: Vec::new(), + position: None, + } +} + +fn graph(nodes: Vec) -> WorkflowGraph { + WorkflowGraph { + nodes, + ..WorkflowGraph::default() + } +} + +#[test] +fn a_host_that_has_said_nothing_refuses_nothing() { + // The reading that would break every unconfigured deployment: empty + // meaning "deny" rather than "unknown". + let facts = HostFacts::unknown(); + let g = graph(vec![ + node("a", NodeKind::Agent, json!({ "agent_ref": "anyone" })), + node( + "t", + NodeKind::ToolCall, + json!({ "slug": "anything:at:all" }), + ), + node( + "c", + NodeKind::Code, + json!({ "language": "python", "source": "1" }), + ), + ]); + assert!(facts.check(&g).is_empty()); + assert!( + facts.render().is_empty(), + "nothing known renders as nothing" + ); +} + +#[test] +fn a_worker_this_host_does_not_have_is_named() { + let facts = HostFacts { + workers: vec!["laptop".into(), "ci".into()], + ..HostFacts::unknown() + }; + let problems = facts.check(&graph(vec![node( + "a", + NodeKind::Agent, + json!({ "agent_ref": "desktop" }), + )])); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("desktop"), "{problems:?}"); + assert!( + problems[0].contains("laptop, ci"), + "the alternatives are offered" + ); +} + +#[test] +fn no_default_worker_makes_agent_ref_mandatory() { + // A host fact that changes a field from optional to required. + let facts = HostFacts { + workers: vec!["laptop".into()], + default_worker: None, + ..HostFacts::unknown() + }; + let problems = facts.check(&graph(vec![node("a", NodeKind::Agent, json!({}))])); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("must name"), "{problems:?}"); +} + +#[test] +fn a_default_worker_makes_a_bare_agent_node_fine() { + let facts = HostFacts { + workers: vec!["laptop".into()], + default_worker: Some("laptop".into()), + ..HostFacts::unknown() + }; + assert!( + facts + .check(&graph(vec![node("a", NodeKind::Agent, json!({}))])) + .is_empty() + ); +} + +#[test] +fn a_slug_outside_both_lists_is_refused() { + let facts = HostFacts { + native_tools: vec!["medulla:shell".into()], + tool_allowlist: vec!["github".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![ + node("ok", NodeKind::ToolCall, json!({ "slug": "medulla:shell" })), + node("no", NodeKind::ToolCall, json!({ "slug": "slack" })), + ]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("slack"), "{problems:?}"); +} + +#[test] +fn an_http_host_outside_the_allowlist_is_refused_but_a_subdomain_is_not() { + let facts = HostFacts { + http_allowlist: vec!["github.com".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![ + node( + "ok", + NodeKind::HttpRequest, + json!({ "url": "https://api.github.com/x" }), + ), + node( + "no", + NodeKind::HttpRequest, + json!({ "url": "https://evil.test/x" }), + ), + ]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1, "{problems:?}"); + assert!(problems[0].contains("evil.test")); +} + +#[test] +fn a_url_built_from_an_expression_is_left_to_run_time() { + // Refusing it would refuse the correct way to write a parameterised + // request, which is the thing the authoring prompt asks for. + let facts = HostFacts { + http_allowlist: vec!["github.com".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![node( + "u", + NodeKind::HttpRequest, + json!({ "url": "=\"https://\" + .inputs.host" }), + )]); + assert!(facts.check(&g).is_empty()); +} + +#[test] +fn disabled_code_and_refused_shell_are_both_reported() { + let facts = HostFacts { + allow_code: Some(false), + shell_available: Some(false), + ..HostFacts::unknown() + }; + let g = graph(vec![ + node( + "c", + NodeKind::Code, + json!({ "language": "python", "source": "1" }), + ), + node("s", NodeKind::Shell, json!({ "script": "ls" })), + ]); + assert_eq!( + facts.check(&g).len(), + 2, + "every failure at once, not the first" + ); +} + +#[test] +fn a_loop_above_the_host_ceiling_is_reported() { + // Otherwise it silently stops earlier than the graph says. + let facts = HostFacts { + max_loop_iterations: Some(10), + ..HostFacts::unknown() + }; + let g = graph(vec![node( + "l", + NodeKind::Loop, + json!({ "max_iterations": 50 }), + )]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("ceiling of 10"), "{problems:?}"); +} + +#[test] +fn a_trigger_kind_that_never_fires_is_reported() { + let facts = HostFacts { + trigger_kinds: vec!["manual".into()], + ..HostFacts::unknown() + }; + let g = graph(vec![node( + "t", + NodeKind::Trigger, + json!({ "trigger_kind": "schedule" }), + )]); + let problems = facts.check(&g); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("never dispatched"), "{problems:?}"); +} + +#[test] +fn a_tool_fact_renders_its_argument_shape_into_the_prompt() { + let facts = HostFacts { + native_tools: vec!["host:shell".into()], + tools: vec![ToolFact { + slug: "host:shell".into(), + args: "`script` (inline text) or `script_path` (a file); NOT `command`".into(), + }], + ..HostFacts::unknown() + }; + let rendered = facts.render(); + assert!( + rendered.contains("tool `host:shell` args:") && rendered.contains("script_path"), + "{rendered}" + ); +} + +#[test] +fn the_rendering_states_consequences_not_just_values() { + let facts = HostFacts { + default_worker: None, + workers: vec!["laptop".into()], + allow_code: Some(false), + notes: vec!["Only manual triggers fire here.".into()], + ..HostFacts::unknown() + }; + let rendered = facts.render(); + assert!(rendered.contains("every agent node must name config.agent_ref")); + assert!(rendered.contains("DISABLED")); + assert!(rendered.contains("Only manual triggers fire here.")); +} + +#[test] +fn a_url_without_a_scheme_still_yields_its_host() { + assert_eq!(host_of("api.github.com/x"), Some("api.github.com")); + assert_eq!( + host_of("https://user:pw@api.github.com:443/x"), + Some("api.github.com") + ); + assert_eq!(host_of(""), None); +} diff --git a/crates/tinyflows-adaptive/src/intake/mod.rs b/crates/tinyflows-adaptive/src/intake/mod.rs index 0c538355..f2c0cc85 100644 --- a/crates/tinyflows-adaptive/src/intake/mod.rs +++ b/crates/tinyflows-adaptive/src/intake/mod.rs @@ -484,41 +484,5 @@ fn peek(value: &Value) -> String { } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn a_bare_object_is_read_as_itself() { - let answer = serde_json::json!({ "workflow_id": "pr-review" }); - assert_eq!(extract(&answer).unwrap()["workflow_id"], "pr-review"); - } - - #[test] - fn an_openai_shaped_envelope_is_unwrapped() { - let answer = serde_json::json!({ - "choices": [{ "message": { "content": "{\"workflow_id\":\"pr-review\"}" } }] - }); - assert_eq!(extract(&answer).unwrap()["workflow_id"], "pr-review"); - } - - #[test] - fn a_text_field_holding_json_is_read() { - let answer = serde_json::json!({ "text": "{\"workflow_id\":\"x\"}" }); - assert_eq!(extract(&answer).unwrap()["workflow_id"], "x"); - } - - #[test] - fn prose_around_the_object_does_not_lose_it() { - // Models do this whatever the response_format asked for. - let answer = serde_json::json!({ - "text": "Sure! Here you go:\n```json\n{\"workflow_id\":\"x\"}\n```\nHope that helps." - }); - assert_eq!(extract(&answer).unwrap()["workflow_id"], "x"); - } - - #[test] - fn an_answer_with_no_object_at_all_is_none_rather_than_a_panic() { - assert!(extract(&serde_json::json!({ "text": "I could not decide." })).is_none()); - assert!(extract(&serde_json::json!("just a string")).is_none()); - } -} +#[path = "mod_tests.rs"] +mod tests; diff --git a/crates/tinyflows-adaptive/src/intake/mod_tests.rs b/crates/tinyflows-adaptive/src/intake/mod_tests.rs new file mode 100644 index 00000000..819454ee --- /dev/null +++ b/crates/tinyflows-adaptive/src/intake/mod_tests.rs @@ -0,0 +1,36 @@ +use super::*; + +#[test] +fn a_bare_object_is_read_as_itself() { + let answer = serde_json::json!({ "workflow_id": "pr-review" }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "pr-review"); +} + +#[test] +fn an_openai_shaped_envelope_is_unwrapped() { + let answer = serde_json::json!({ + "choices": [{ "message": { "content": "{\"workflow_id\":\"pr-review\"}" } }] + }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "pr-review"); +} + +#[test] +fn a_text_field_holding_json_is_read() { + let answer = serde_json::json!({ "text": "{\"workflow_id\":\"x\"}" }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "x"); +} + +#[test] +fn prose_around_the_object_does_not_lose_it() { + // Models do this whatever the response_format asked for. + let answer = serde_json::json!({ + "text": "Sure! Here you go:\n```json\n{\"workflow_id\":\"x\"}\n```\nHope that helps." + }); + assert_eq!(extract(&answer).unwrap()["workflow_id"], "x"); +} + +#[test] +fn an_answer_with_no_object_at_all_is_none_rather_than_a_panic() { + assert!(extract(&serde_json::json!({ "text": "I could not decide." })).is_none()); + assert!(extract(&serde_json::json!("just a string")).is_none()); +} diff --git a/crates/tinyflows-adaptive/src/intake/recipe.rs b/crates/tinyflows-adaptive/src/intake/recipe.rs index 6a7654e6..58fe2e37 100644 --- a/crates/tinyflows-adaptive/src/intake/recipe.rs +++ b/crates/tinyflows-adaptive/src/intake/recipe.rs @@ -327,486 +327,7 @@ pub(crate) fn errand(goal: &str) -> Result { lower(&recipe, &[]).map(|(graph, _, _)| graph) } -/// Ask steps that restate a declared value instead of relying on the -/// attachment. Only distinctive values count — refusing a plan because an -/// ask contains the word "on" would block perfectly reusable recipes — and -/// only DECLARED inputs: undeclared entries are trimmed by the author gate -/// and never attached, so their values in an ask are just prose. -fn pasted_values( - steps: &[Step], - declared: &[(String, String, bool)], - inputs: &Map, -) -> Vec { - let mut problems = Vec::new(); - for step in steps { - let Action::Ask { prompt, .. } = &step.action else { - continue; - }; - for (name, _, _) in declared { - let Some(value) = inputs.get(name).and_then(Value::as_str).map(str::trim) else { - continue; - }; - if crate::reuse::distinctive(value) && prompt.contains(value) { - problems.push(format!( - "step `{}` pastes the value of input `{name}` into its ask — remove \ - it; declared values are attached automatically, and a pasted value \ - makes the plan single-use", - step.id - )); - } - } - } - problems -} - -/// The generated prompt expression for an ask step. -/// -/// A jq program the model never sees: the instruction as a quoted literal, -/// then every declared input, then each read step's output through the path -/// its kind actually produces — `stdout` for a script (with the engine's -/// pre-parsed `stdout_json` unnecessary here: the agent reads text), `text` -/// for an upstream agent. Missing values render as an explicit marker -/// rather than vanishing, because an agent told "output: (missing)" says so -/// instead of improvising. -fn ask_expression( - prompt: &str, - reads: &[String], - steps: &[Step], - declared: &[(String, String, bool)], -) -> String { - let mut program = format!("={}", jq_quote(prompt)); - for (name, _, _) in declared { - program.push_str(&format!( - " + {} + ((.run.inputs{} // \"(not provided)\") | tostring)", - jq_quote(&format!("\n\n# Input `{name}`\n")), - jq_field(name) - )); - } - for read in reads { - let path = format!( - "({} // \"(no output)\")", - output_of(read, kind_of(read, steps)) - ); - program.push_str(&format!( - " + {} + ({path} | tostring)", - jq_quote(&format!("\n\n# Output of step `{read}`\n")) - )); - } - program -} - -/// Which kind of step `id` is, for choosing how to read its output. -fn kind_of<'a>(id: &str, steps: &'a [Step]) -> Option<&'a Action> { - steps - .iter() - .find(|step| step.id == id) - .map(|step| &step.action) -} - -/// The jq path that yields a step's output as something readable. -/// -/// Each node kind puts its result somewhere different, and this is the one -/// place that knows where. -/// -/// **An agent's prose is at `item.text`, not `item.json.text`.** The two are -/// siblings on the envelope — `json` is the structured value, `text` is the -/// prose `text_of` derived from it — so `item.json.text` reads a `text` field -/// *inside* the structured value, which a prose reply does not have. It -/// resolved to null, and every `reads` of an agent step rendered -/// "(no output)": the exact silent-null class this whole surface exists to -/// make impossible, sitting inside the surface. A script's `stdout` genuinely -/// is nested (`json` holds `{exit_code, stdout}`), which is what made the two -/// paths look symmetric enough to write side by side. -/// -/// For a called workflow it is a projection, because a `sub_workflow` node -/// emits the child's entire final run state, every node of it, wrapped around -/// the answer. Handing an agent that whole object would bury the deliverable -/// in the child's own bookkeeping. -/// -/// The projection keeps each child step's readable leaf, labelled with the -/// step id it came from. Not just the last one: the child's node slots are a -/// JSON object, whose key order is alphabetical rather than the order the -/// steps ran, so "the last one" is not a thing this expression can ask for. -/// Everything the child produced, named, is the honest answer — and for the -/// ordinary child whose one agent step writes the deliverable, it is exactly -/// that deliverable. -fn output_of(id: &str, action: Option<&Action>) -> String { - let slot = jq_field(id); - match action { - Some(Action::Run(_)) => format!(".nodes{slot}.item.json.stdout"), - Some(Action::Use { .. }) => child_answer(id), - _ => format!(".nodes{slot}.item.text"), - } -} - -/// The projection that turns a called workflow's run state into prose. -/// -/// Written defensively at every hop — a slot with no `items`, an empty array, -/// an item whose payload is not an object — because this walks a *child's* -/// state, whose shape this graph did not choose. A failure here would take -/// down the parent node rather than report the step that produced nothing, -/// and a child always has at least one payload that is not an object: its -/// trigger slot holds the seeded item **array**. -/// -/// Guarded with an explicit `type == "object"` rather than the `?` operator, -/// which does not do what it looks like it does here. In `jaq`, `.a?` over a -/// non-object yields no output as expected, but a two-hop `.a.b?` fails the -/// whole enclosing expression instead — so the "defensive" spelling of this -/// projection resolved the entire prompt to null, and every composed plan -/// reached its combining agent with nothing. Found by evaluating against a -/// real child run state; a synthetic one whose slots were all objects passed. -fn child_answer(id: &str) -> String { - // Two different shapes in one expression, which is the whole hazard here. - // `.nodes.{id}.item` is the *scope* projection — the child's final run - // state. Inside it, `.nodes.` is a raw run-state slot, which stores - // serialized items (`{"json": …}`) rather than the bare payloads the scope - // exposes. So the outer hop drops `items`/`json` and the inner one needs - // both. - let slot = jq_field(id); - format!( - "([((.nodes{slot}.item.nodes // {{}}) | to_entries[]) \ - | . as $step \ - | ((.value.items // []) | .[-1] | .json) as $out \ - | (($out | if type == \"object\" then \ - (.text // .stdout // (.json | if type == \"object\" then .stdout else null end)) \ - else null end) // empty) as $said \ - | \"## \" + $step.key + \"\\n\" + ($said | tostring)] \ - | join(\"\\n\\n\") \ - | if . == \"\" then null else . end)" - ) -} - -/// One object key as a jq path step: `["fetch_pr"]`, never `.fetch_pr`. -/// -/// `sanitize_id` keeps `[a-z0-9_]`, which is *not* the same set as the -/// identifiers jq's dot syntax accepts: it permits a leading digit, and nothing -/// upstream rejects a step id such as `2024_report`. `.nodes.2024_report` does -/// not compile, so the whole prompt expression fails at run time — a plan -/// refused by the evaluator for the way its author spelled an id. Bracket -/// access takes any key, so this is the only spelling used for an interpolated -/// name. -fn jq_field(name: &str) -> String { - format!("[{}]", jq_quote(name)) -} - -/// A string as a jq literal: quoted, with the characters jq treats specially -/// escaped. Newlines become `\n` so the program stays one line. -fn jq_quote(text: &str) -> String { - let mut quoted = String::with_capacity(text.len() + 2); - quoted.push('"'); - for ch in text.chars() { - match ch { - '"' => quoted.push_str("\\\""), - '\\' => quoted.push_str("\\\\"), - '\n' => quoted.push_str("\\n"), - '\r' => quoted.push_str("\\r"), - '\t' => quoted.push_str("\\t"), - other => quoted.push(other), - } - } - quoted.push('"'); - quoted -} - -fn parse_steps( - answer: &Value, - callables: &[Callable], - declared: &[(String, String, bool)], -) -> Result, IntakeError> { - let raw = answer["steps"] - .as_array() - .filter(|steps| !steps.is_empty()) - .ok_or_else(|| { - IntakeError::Invalid( - "the reply has no `steps` — return at least one step with an `id` and a \ - `run` script, an `ask` instruction or a `use` workflow id" - .to_string(), - ) - })?; - - let mut problems = Vec::new(); - let mut steps: Vec = Vec::new(); - for (index, step) in raw.iter().enumerate() { - let id = sanitize_id(step["id"].as_str().unwrap_or_default()); - if id.is_empty() { - problems.push(format!("step {index} has no usable `id`")); - continue; - } - if id == "start" || steps.iter().any(|existing| existing.id == id) { - problems.push(format!("step id `{id}` is taken — ids must be unique")); - continue; - } - let run = step["run"] - .as_str() - .map(str::trim) - .filter(|s| !s.is_empty()); - let ask = step["ask"] - .as_str() - .map(str::trim) - .filter(|s| !s.is_empty()); - let called = step["use"] - .as_str() - .map(str::trim) - .filter(|s| !s.is_empty()); - let given = [run.is_some(), ask.is_some(), called.is_some()] - .iter() - .filter(|given| **given) - .count(); - if given > 1 { - problems.push(format!( - "step `{id}` has more than one of `run`, `ask` and `use` — a step does \ - exactly one thing, so split it" - )); - continue; - } - let action = match (run, ask, called) { - (Some(script), _, _) => Action::Run(script.to_string()), - (_, Some(prompt), _) => Action::Ask { - prompt: prompt.to_string(), - worker: step["worker"] - .as_str() - .map(str::trim) - .filter(|s| !s.is_empty()) - .map(ToString::to_string), - }, - (_, _, Some(workflow_id)) => { - match call(&id, workflow_id, &step["with"], callables, declared, &steps) { - Ok(action) => action, - Err(problem) => { - problems.push(problem); - continue; - } - } - } - (None, None, None) => { - problems.push(format!( - "step `{id}` has none of a `run` script, an `ask` instruction or a \ - `use` workflow id" - )); - continue; - } - }; - let mut reads = Vec::new(); - if let Some(raw_reads) = step["reads"].as_array() { - for read in raw_reads { - let read = sanitize_id(read.as_str().unwrap_or_default()); - if steps.iter().any(|existing| existing.id == read) { - reads.push(read); - } else { - problems.push(format!( - "step `{id}` reads `{read}`, which is not an EARLIER step id" - )); - } - } - } - if matches!(action, Action::Run(_)) && !reads.is_empty() { - problems.push(format!( - "step `{id}`: `reads` only works on ask steps — a run script sees nothing" - )); - } - if matches!(action, Action::Use { .. }) && !reads.is_empty() { - problems.push(format!( - "step `{id}`: `reads` only works on ask steps — a called workflow takes \ - what you pass it, so put `\"@step.\"` in `with` instead" - )); - } - steps.push(Step { id, action, reads }); - } - if !problems.is_empty() { - return Err(IntakeError::Invalid(problems.join("; "))); - } - Ok(steps) -} - -/// Build one `use` step, refusing everything that could only fail later. -/// -/// Three checks, and each stands for a run this saves: an id nobody offered -/// is a hallucination that the resolver would turn into a mid-run capability -/// error; a required input left unfilled fails the child's own declaration -/// check after the parent has already spent its earlier steps; and a `with` -/// key the child never declared is a value the model believes it is passing -/// and the child will never see. -fn call( - id: &str, - workflow_id: &str, - with: &Value, - callables: &[Callable], - declared: &[(String, String, bool)], - earlier: &[Step], -) -> Result { - let Some(callable) = callables.iter().find(|c| c.id == workflow_id) else { - let offered: Vec<&str> = callables.iter().map(|c| c.id.as_str()).collect(); - return Err(if offered.is_empty() { - format!( - "step `{id}` uses `{workflow_id}`, but this host has no saved workflows to \ - call — write the step yourself" - ) - } else { - format!( - "step `{id}` uses `{workflow_id}`, which is not one of the workflows you \ - can call ({})", - offered.join(", ") - ) - }); - }; - - let given = match with { - Value::Null => Map::new(), - Value::Object(fields) => fields.clone(), - _ => { - return Err(format!( - "step `{id}`: `with` must be an object mapping `{workflow_id}`'s input \ - names to values" - )); - } - }; - - for (name, _) in callable.inputs.iter().filter(|(_, required)| *required) { - // Present-but-empty is not supplied. `"repo": null` and `"repo": ""` - // pass a `contains_key` check and are then forwarded unchanged, so the - // child fails its own declaration check mid-run — the exact failure - // this refusal exists to move to intake. `gated` in `author.rs` already - // reads unfilled the same way; the two must not disagree. - let filled = given - .get(name) - .is_some_and(|value| !value.is_null() && value.as_str() != Some("")); - if !filled { - return Err(format!( - "step `{id}`: `{workflow_id}` requires the input `{name}` and `with` does \ - not supply it" - )); - } - } - let mut forwarded = Map::new(); - for (name, value) in given { - if !callable.inputs.iter().any(|(input, _)| input == &name) { - return Err(format!( - "step `{id}`: `{workflow_id}` declares no input `{name}` — it takes {}", - if callable.inputs.is_empty() { - "none".to_string() - } else { - callable - .inputs - .iter() - .map(|(input, _)| input.as_str()) - .collect::>() - .join(", ") - } - )); - } - forwarded.insert( - name, - forward(&value, declared, earlier).map_err(|why| format!("step `{id}`: {why}"))?, - ); - } - Ok(Action::Use { - workflow_id: workflow_id.to_string(), - with: forwarded, - }) -} - -/// Turn one `with` value into what the child should receive. -/// -/// `@input.x` and `@step.y` become the engine expressions that read them; a -/// plain value is passed as itself. The sigil exists so a model can wire a -/// child's input to live data without writing jq — the same bargain the rest -/// of this surface makes. -fn forward( - value: &Value, - declared: &[(String, String, bool)], - earlier: &[Step], -) -> Result { - let Some(reference) = value.as_str().and_then(|text| text.strip_prefix('@')) else { - return Ok(value.clone()); - }; - if let Some(name) = reference.strip_prefix("input.") { - let name = sanitize_id(name); - if !declared.iter().any(|(declared, _, _)| declared == &name) { - return Err(format!( - "`@input.{name}` names an input you did not declare — add it to `declared`" - )); - } - return Ok(Value::String(format!("=.run.inputs{}", jq_field(&name)))); - } - if let Some(step) = reference.strip_prefix("step.") { - let step = sanitize_id(step); - let Some(action) = kind_of(&step, earlier) else { - return Err(format!( - "`@step.{step}` names a step that is not an EARLIER step of this plan" - )); - }; - return Ok(Value::String(format!( - "={}", - output_of(&step, Some(action)) - ))); - } - Err(format!( - "`{reference}` is not a reference this understands — write `@input.`, \ - `@step.`, or a plain value" - )) -} - -fn parse_declared(answer: &Value) -> Vec<(String, String, bool)> { - answer["declared"] - .as_array() - .map(|declared| { - declared - .iter() - .filter_map(|input| { - let name = sanitize_id(input["name"].as_str().unwrap_or_default()); - if name.is_empty() { - return None; - } - Some(( - name, - input["description"] - .as_str() - .unwrap_or_default() - .to_string(), - input["required"].as_bool().unwrap_or(false), - )) - }) - .collect() - }) - .unwrap_or_default() -} - -/// A graph name from the recipe: the first ask step's opening words, or the -/// step ids — something a shelf listing can show, not an id. -fn graph_name(why: &str, steps: &[Step]) -> String { - let head: String = why.split_whitespace().take(6).collect::>().join(" "); - if !head.is_empty() { - return head; - } - steps - .iter() - .map(|step| step.id.as_str()) - .collect::>() - .join(" → ") -} - -/// Identifiers the engine and jq both accept: lowercase, alnum and `_`. -fn sanitize_id(raw: &str) -> String { - let mut id: String = raw - .trim() - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() { - ch.to_ascii_lowercase() - } else { - '_' - } - }) - .collect(); - while id.starts_with('_') { - id.remove(0); - } - while id.ends_with('_') { - id.pop(); - } - id -} - +include!("recipe/lowering.rs"); #[cfg(test)] #[path = "recipe_tests.rs"] mod tests; diff --git a/crates/tinyflows-adaptive/src/intake/recipe/lowering.rs b/crates/tinyflows-adaptive/src/intake/recipe/lowering.rs new file mode 100644 index 00000000..07d61e1f --- /dev/null +++ b/crates/tinyflows-adaptive/src/intake/recipe/lowering.rs @@ -0,0 +1,480 @@ +/// Ask steps that restate a declared value instead of relying on the +/// attachment. Only distinctive values count — refusing a plan because an +/// ask contains the word "on" would block perfectly reusable recipes — and +/// only DECLARED inputs: undeclared entries are trimmed by the author gate +/// and never attached, so their values in an ask are just prose. +fn pasted_values( + steps: &[Step], + declared: &[(String, String, bool)], + inputs: &Map, +) -> Vec { + let mut problems = Vec::new(); + for step in steps { + let Action::Ask { prompt, .. } = &step.action else { + continue; + }; + for (name, _, _) in declared { + let Some(value) = inputs.get(name).and_then(Value::as_str).map(str::trim) else { + continue; + }; + if crate::reuse::distinctive(value) && prompt.contains(value) { + problems.push(format!( + "step `{}` pastes the value of input `{name}` into its ask — remove \ + it; declared values are attached automatically, and a pasted value \ + makes the plan single-use", + step.id + )); + } + } + } + problems +} + +/// The generated prompt expression for an ask step. +/// +/// A jq program the model never sees: the instruction as a quoted literal, +/// then every declared input, then each read step's output through the path +/// its kind actually produces — `stdout` for a script (with the engine's +/// pre-parsed `stdout_json` unnecessary here: the agent reads text), `text` +/// for an upstream agent. Missing values render as an explicit marker +/// rather than vanishing, because an agent told "output: (missing)" says so +/// instead of improvising. +fn ask_expression( + prompt: &str, + reads: &[String], + steps: &[Step], + declared: &[(String, String, bool)], +) -> String { + let mut program = format!("={}", jq_quote(prompt)); + for (name, _, _) in declared { + program.push_str(&format!( + " + {} + ((.run.inputs{} // \"(not provided)\") | tostring)", + jq_quote(&format!("\n\n# Input `{name}`\n")), + jq_field(name) + )); + } + for read in reads { + let path = format!( + "({} // \"(no output)\")", + output_of(read, kind_of(read, steps)) + ); + program.push_str(&format!( + " + {} + ({path} | tostring)", + jq_quote(&format!("\n\n# Output of step `{read}`\n")) + )); + } + program +} + +/// Which kind of step `id` is, for choosing how to read its output. +fn kind_of<'a>(id: &str, steps: &'a [Step]) -> Option<&'a Action> { + steps + .iter() + .find(|step| step.id == id) + .map(|step| &step.action) +} + +/// The jq path that yields a step's output as something readable. +/// +/// Each node kind puts its result somewhere different, and this is the one +/// place that knows where. +/// +/// **An agent's prose is at `item.text`, not `item.json.text`.** The two are +/// siblings on the envelope — `json` is the structured value, `text` is the +/// prose `text_of` derived from it — so `item.json.text` reads a `text` field +/// *inside* the structured value, which a prose reply does not have. It +/// resolved to null, and every `reads` of an agent step rendered +/// "(no output)": the exact silent-null class this whole surface exists to +/// make impossible, sitting inside the surface. A script's `stdout` genuinely +/// is nested (`json` holds `{exit_code, stdout}`), which is what made the two +/// paths look symmetric enough to write side by side. +/// +/// For a called workflow it is a projection, because a `sub_workflow` node +/// emits the child's entire final run state, every node of it, wrapped around +/// the answer. Handing an agent that whole object would bury the deliverable +/// in the child's own bookkeeping. +/// +/// The projection keeps each child step's readable leaf, labelled with the +/// step id it came from. Not just the last one: the child's node slots are a +/// JSON object, whose key order is alphabetical rather than the order the +/// steps ran, so "the last one" is not a thing this expression can ask for. +/// Everything the child produced, named, is the honest answer — and for the +/// ordinary child whose one agent step writes the deliverable, it is exactly +/// that deliverable. +fn output_of(id: &str, action: Option<&Action>) -> String { + let slot = jq_field(id); + match action { + Some(Action::Run(_)) => format!(".nodes{slot}.item.json.stdout"), + Some(Action::Use { .. }) => child_answer(id), + _ => format!(".nodes{slot}.item.text"), + } +} + +/// The projection that turns a called workflow's run state into prose. +/// +/// Written defensively at every hop — a slot with no `items`, an empty array, +/// an item whose payload is not an object — because this walks a *child's* +/// state, whose shape this graph did not choose. A failure here would take +/// down the parent node rather than report the step that produced nothing, +/// and a child always has at least one payload that is not an object: its +/// trigger slot holds the seeded item **array**. +/// +/// Guarded with an explicit `type == "object"` rather than the `?` operator, +/// which does not do what it looks like it does here. In `jaq`, `.a?` over a +/// non-object yields no output as expected, but a two-hop `.a.b?` fails the +/// whole enclosing expression instead — so the "defensive" spelling of this +/// projection resolved the entire prompt to null, and every composed plan +/// reached its combining agent with nothing. Found by evaluating against a +/// real child run state; a synthetic one whose slots were all objects passed. +fn child_answer(id: &str) -> String { + // Two different shapes in one expression, which is the whole hazard here. + // `.nodes.{id}.item` is the *scope* projection — the child's final run + // state. Inside it, `.nodes.` is a raw run-state slot, which stores + // serialized items (`{"json": …}`) rather than the bare payloads the scope + // exposes. So the outer hop drops `items`/`json` and the inner one needs + // both. + let slot = jq_field(id); + format!( + "([((.nodes{slot}.item.nodes // {{}}) | to_entries[]) \ + | . as $step \ + | ((.value.items // []) | .[-1] | .json) as $out \ + | (($out | if type == \"object\" then \ + (.text // .stdout // (.json | if type == \"object\" then .stdout else null end)) \ + else null end) // empty) as $said \ + | \"## \" + $step.key + \"\\n\" + ($said | tostring)] \ + | join(\"\\n\\n\") \ + | if . == \"\" then null else . end)" + ) +} + +/// One object key as a jq path step: `["fetch_pr"]`, never `.fetch_pr`. +/// +/// `sanitize_id` keeps `[a-z0-9_]`, which is *not* the same set as the +/// identifiers jq's dot syntax accepts: it permits a leading digit, and nothing +/// upstream rejects a step id such as `2024_report`. `.nodes.2024_report` does +/// not compile, so the whole prompt expression fails at run time — a plan +/// refused by the evaluator for the way its author spelled an id. Bracket +/// access takes any key, so this is the only spelling used for an interpolated +/// name. +fn jq_field(name: &str) -> String { + format!("[{}]", jq_quote(name)) +} + +/// A string as a jq literal: quoted, with the characters jq treats specially +/// escaped. Newlines become `\n` so the program stays one line. +fn jq_quote(text: &str) -> String { + let mut quoted = String::with_capacity(text.len() + 2); + quoted.push('"'); + for ch in text.chars() { + match ch { + '"' => quoted.push_str("\\\""), + '\\' => quoted.push_str("\\\\"), + '\n' => quoted.push_str("\\n"), + '\r' => quoted.push_str("\\r"), + '\t' => quoted.push_str("\\t"), + other => quoted.push(other), + } + } + quoted.push('"'); + quoted +} + +fn parse_steps( + answer: &Value, + callables: &[Callable], + declared: &[(String, String, bool)], +) -> Result, IntakeError> { + let raw = answer["steps"] + .as_array() + .filter(|steps| !steps.is_empty()) + .ok_or_else(|| { + IntakeError::Invalid( + "the reply has no `steps` — return at least one step with an `id` and a \ + `run` script, an `ask` instruction or a `use` workflow id" + .to_string(), + ) + })?; + + let mut problems = Vec::new(); + let mut steps: Vec = Vec::new(); + for (index, step) in raw.iter().enumerate() { + let id = sanitize_id(step["id"].as_str().unwrap_or_default()); + if id.is_empty() { + problems.push(format!("step {index} has no usable `id`")); + continue; + } + if id == "start" || steps.iter().any(|existing| existing.id == id) { + problems.push(format!("step id `{id}` is taken — ids must be unique")); + continue; + } + let run = step["run"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()); + let ask = step["ask"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()); + let called = step["use"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()); + let given = [run.is_some(), ask.is_some(), called.is_some()] + .iter() + .filter(|given| **given) + .count(); + if given > 1 { + problems.push(format!( + "step `{id}` has more than one of `run`, `ask` and `use` — a step does \ + exactly one thing, so split it" + )); + continue; + } + let action = match (run, ask, called) { + (Some(script), _, _) => Action::Run(script.to_string()), + (_, Some(prompt), _) => Action::Ask { + prompt: prompt.to_string(), + worker: step["worker"] + .as_str() + .map(str::trim) + .filter(|s| !s.is_empty()) + .map(ToString::to_string), + }, + (_, _, Some(workflow_id)) => { + match call(&id, workflow_id, &step["with"], callables, declared, &steps) { + Ok(action) => action, + Err(problem) => { + problems.push(problem); + continue; + } + } + } + (None, None, None) => { + problems.push(format!( + "step `{id}` has none of a `run` script, an `ask` instruction or a \ + `use` workflow id" + )); + continue; + } + }; + let mut reads = Vec::new(); + if let Some(raw_reads) = step["reads"].as_array() { + for read in raw_reads { + let read = sanitize_id(read.as_str().unwrap_or_default()); + if steps.iter().any(|existing| existing.id == read) { + reads.push(read); + } else { + problems.push(format!( + "step `{id}` reads `{read}`, which is not an EARLIER step id" + )); + } + } + } + if matches!(action, Action::Run(_)) && !reads.is_empty() { + problems.push(format!( + "step `{id}`: `reads` only works on ask steps — a run script sees nothing" + )); + } + if matches!(action, Action::Use { .. }) && !reads.is_empty() { + problems.push(format!( + "step `{id}`: `reads` only works on ask steps — a called workflow takes \ + what you pass it, so put `\"@step.\"` in `with` instead" + )); + } + steps.push(Step { id, action, reads }); + } + if !problems.is_empty() { + return Err(IntakeError::Invalid(problems.join("; "))); + } + Ok(steps) +} + +/// Build one `use` step, refusing everything that could only fail later. +/// +/// Three checks, and each stands for a run this saves: an id nobody offered +/// is a hallucination that the resolver would turn into a mid-run capability +/// error; a required input left unfilled fails the child's own declaration +/// check after the parent has already spent its earlier steps; and a `with` +/// key the child never declared is a value the model believes it is passing +/// and the child will never see. +fn call( + id: &str, + workflow_id: &str, + with: &Value, + callables: &[Callable], + declared: &[(String, String, bool)], + earlier: &[Step], +) -> Result { + let Some(callable) = callables.iter().find(|c| c.id == workflow_id) else { + let offered: Vec<&str> = callables.iter().map(|c| c.id.as_str()).collect(); + return Err(if offered.is_empty() { + format!( + "step `{id}` uses `{workflow_id}`, but this host has no saved workflows to \ + call — write the step yourself" + ) + } else { + format!( + "step `{id}` uses `{workflow_id}`, which is not one of the workflows you \ + can call ({})", + offered.join(", ") + ) + }); + }; + + let given = match with { + Value::Null => Map::new(), + Value::Object(fields) => fields.clone(), + _ => { + return Err(format!( + "step `{id}`: `with` must be an object mapping `{workflow_id}`'s input \ + names to values" + )); + } + }; + + for (name, _) in callable.inputs.iter().filter(|(_, required)| *required) { + // Present-but-empty is not supplied. `"repo": null` and `"repo": ""` + // pass a `contains_key` check and are then forwarded unchanged, so the + // child fails its own declaration check mid-run — the exact failure + // this refusal exists to move to intake. `gated` in `author.rs` already + // reads unfilled the same way; the two must not disagree. + let filled = given + .get(name) + .is_some_and(|value| !value.is_null() && value.as_str() != Some("")); + if !filled { + return Err(format!( + "step `{id}`: `{workflow_id}` requires the input `{name}` and `with` does \ + not supply it" + )); + } + } + let mut forwarded = Map::new(); + for (name, value) in given { + if !callable.inputs.iter().any(|(input, _)| input == &name) { + return Err(format!( + "step `{id}`: `{workflow_id}` declares no input `{name}` — it takes {}", + if callable.inputs.is_empty() { + "none".to_string() + } else { + callable + .inputs + .iter() + .map(|(input, _)| input.as_str()) + .collect::>() + .join(", ") + } + )); + } + forwarded.insert( + name, + forward(&value, declared, earlier).map_err(|why| format!("step `{id}`: {why}"))?, + ); + } + Ok(Action::Use { + workflow_id: workflow_id.to_string(), + with: forwarded, + }) +} + +/// Turn one `with` value into what the child should receive. +/// +/// `@input.x` and `@step.y` become the engine expressions that read them; a +/// plain value is passed as itself. The sigil exists so a model can wire a +/// child's input to live data without writing jq — the same bargain the rest +/// of this surface makes. +fn forward( + value: &Value, + declared: &[(String, String, bool)], + earlier: &[Step], +) -> Result { + let Some(reference) = value.as_str().and_then(|text| text.strip_prefix('@')) else { + return Ok(value.clone()); + }; + if let Some(name) = reference.strip_prefix("input.") { + let name = sanitize_id(name); + if !declared.iter().any(|(declared, _, _)| declared == &name) { + return Err(format!( + "`@input.{name}` names an input you did not declare — add it to `declared`" + )); + } + return Ok(Value::String(format!("=.run.inputs{}", jq_field(&name)))); + } + if let Some(step) = reference.strip_prefix("step.") { + let step = sanitize_id(step); + let Some(action) = kind_of(&step, earlier) else { + return Err(format!( + "`@step.{step}` names a step that is not an EARLIER step of this plan" + )); + }; + return Ok(Value::String(format!( + "={}", + output_of(&step, Some(action)) + ))); + } + Err(format!( + "`{reference}` is not a reference this understands — write `@input.`, \ + `@step.`, or a plain value" + )) +} + +fn parse_declared(answer: &Value) -> Vec<(String, String, bool)> { + answer["declared"] + .as_array() + .map(|declared| { + declared + .iter() + .filter_map(|input| { + let name = sanitize_id(input["name"].as_str().unwrap_or_default()); + if name.is_empty() { + return None; + } + Some(( + name, + input["description"] + .as_str() + .unwrap_or_default() + .to_string(), + input["required"].as_bool().unwrap_or(false), + )) + }) + .collect() + }) + .unwrap_or_default() +} + +/// A graph name from the recipe: the first ask step's opening words, or the +/// step ids — something a shelf listing can show, not an id. +fn graph_name(why: &str, steps: &[Step]) -> String { + let head: String = why.split_whitespace().take(6).collect::>().join(" "); + if !head.is_empty() { + return head; + } + steps + .iter() + .map(|step| step.id.as_str()) + .collect::>() + .join(" → ") +} + +/// Identifiers the engine and jq both accept: lowercase, alnum and `_`. +fn sanitize_id(raw: &str) -> String { + let mut id: String = raw + .trim() + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_lowercase() + } else { + '_' + } + }) + .collect(); + while id.starts_with('_') { + id.remove(0); + } + while id.ends_with('_') { + id.pop(); + } + id +} + diff --git a/crates/tinyflows-adaptive/src/intake/recipe_tests.rs b/crates/tinyflows-adaptive/src/intake/recipe_tests.rs index aba79e3f..a8ceab20 100644 --- a/crates/tinyflows-adaptive/src/intake/recipe_tests.rs +++ b/crates/tinyflows-adaptive/src/intake/recipe_tests.rs @@ -283,494 +283,5 @@ fn a_step_id_starting_with_a_digit_still_compiles_as_jq() { use super::Callable; -fn audit() -> Callable { - Callable { - id: "pr-audit-review".to_string(), - name: "PR audit review".to_string(), - description: "reviews a pull request and posts the verdict".to_string(), - inputs: vec![("repo".to_string(), true), ("depth".to_string(), false)], - } -} - -fn compose_recipe() -> serde_json::Value { - json!({ - "why": "audit the PR, then summarise what it found", - "declared": [ - { "name": "repo", "description": "owner/name", "required": true } - ], - "inputs": { "repo": "acme/thing" }, - "steps": [ - { "id": "audit", "use": "pr-audit-review", "with": { "repo": "@input.repo" } }, - { "id": "summary", "ask": "Summarise the audit in three bullets.", - "reads": ["audit"] } - ] - }) -} - -#[test] -fn a_use_step_lowers_to_a_sub_workflow_node_that_references_the_callee() { - let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); - assert!( - validate_all(&graph).is_empty(), - "{:?}", - validate_all(&graph) - ); - let node = graph - .nodes - .iter() - .find(|node| node.id == "audit") - .expect("the use step became a node"); - assert_eq!(node.kind, NodeKind::SubWorkflow); - // By reference, never inlined: the callee keeps its own identity, its own - // scores, and whatever it becomes next. - assert_eq!(node.config["workflow_id"], json!("pr-audit-review")); - assert!( - node.config.get("workflow").is_none(), - "an inlined child would fork the callee at authoring time" - ); - // `@input.repo` became the expression that reads the parent's run input. - assert_eq!( - node.config["inputs"]["repo"], - json!("=.run.inputs[\"repo\"]") - ); -} - -#[test] -fn the_lowered_sub_workflow_config_satisfies_the_engines_own_contract() { - // The same drift guard the shell lowering has, for the same reason: the - // `use` step's whole value is that the engine already knows how to run a - // child, and it knows it by reading `workflow_id` and `inputs`. A rename on - // either side would surface as a capability error mid-run, attributed to - // the work rather than to the plan. - let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); - let config = &graph - .nodes - .iter() - .find(|node| node.id == "audit") - .expect("the use step became a node") - .config; - let contract = tinyflows::catalog::all_contracts() - .iter() - .find(|contract| contract.kind == "sub_workflow") - .expect("the engine has a sub_workflow contract") - .clone(); - let fields: Vec<&str> = contract - .config_fields - .iter() - .map(|field| field.name.as_str()) - .collect(); - for key in ["workflow_id", "inputs"] { - assert!( - fields.contains(&key), - "the engine's sub_workflow contract no longer declares `{key}`: {fields:?}" - ); - assert!( - config.get(key).is_some(), - "a lowered use step must fill config.{key}: {config}" - ); - } - // Required fields are the engine's own list; filling one is not enough if - // it grows another. - for field in contract.config_fields.iter().filter(|field| field.required) { - assert!( - config.get(&field.name).is_some(), - "the lowering fills none of the engine's required sub_workflow field \ - `{}`: {config}", - field.name - ); - } -} - -#[test] -fn a_step_reference_in_with_reads_the_earlier_step_the_way_its_kind_produces() { - let recipe = json!({ - "why": "fetch the diff, then hand it to a saved reviewer", - "declared": [], - "steps": [ - { "id": "diff", "run": "git diff" }, - { "id": "review", "use": "reviewer", "with": { "patch": "@step.diff" } } - ] - }); - let callable = Callable { - id: "reviewer".to_string(), - name: String::new(), - description: "reviews a patch".to_string(), - inputs: vec![("patch".to_string(), true)], - }; - let (graph, _, _) = lower(&recipe, &[callable]).expect("lowers"); - let node = graph - .nodes - .iter() - .find(|node| node.id == "review") - .expect("the use step became a node"); - // A run step's output is its stdout, and the reference knows that without - // the model having to. - assert_eq!( - node.config["inputs"]["patch"], - json!("=.nodes[\"diff\"].item.json.stdout") - ); -} - -#[test] -fn a_required_input_present_but_empty_is_refused_the_way_an_absent_one_is() { - // `contains_key` accepts `null` and `""`, which `forward` then passes - // through unchanged — so the child fails its OWN declaration check - // mid-run, which is the failure this refusal exists to move to intake. - // `gated` in `author.rs` already reads unfilled this way; the two checks - // disagreeing is what let the value through. - for empty in [json!(null), json!("")] { - let recipe = json!({ - "why": "audit the PR", - "declared": [], - "steps": [ - { "id": "audit", "use": "pr-audit-review", "with": { "repo": empty } } - ] - }); - let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); - assert!( - err.contains("requires the input `repo`"), - "an empty value is not a supplied value ({empty}): {err}" - ); - } -} - -#[test] -fn a_use_step_naming_a_workflow_nobody_offered_is_refused_at_intake() { - // Not deferred to the resolver: a hallucinated id would surface as a - // capability error mid-run, after the earlier steps had already been paid - // for, and be attributed to the work rather than to the plan. - let err = lower(&compose_recipe(), &[]) - .expect_err("refused") - .to_string(); - assert!(err.contains("no saved workflows to call"), "{err}"); - - let other = Callable { - id: "something-else".to_string(), - ..audit() - }; - let err = lower(&compose_recipe(), &[other]) - .expect_err("refused") - .to_string(); - assert!( - err.contains("something-else"), - "names what IS callable: {err}" - ); -} - -#[test] -fn a_use_step_that_omits_a_required_input_is_refused_with_the_name() { - let recipe = json!({ - "why": "call it with nothing", - "steps": [{ "id": "audit", "use": "pr-audit-review", "with": {} }] - }); - let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); - assert!(err.contains("requires the input `repo`"), "{err}"); -} - -#[test] -fn a_with_key_the_callee_never_declared_is_refused_rather_than_dropped() { - // Silently dropping it would leave the model believing it passed a value - // the child will never see — the worst kind of pass, because the run - // completes. - let recipe = json!({ - "why": "wrong input name", - "declared": [{ "name": "repo", "description": "owner/name", "required": true }], - "steps": [{ - "id": "audit", "use": "pr-audit-review", - "with": { "repo": "@input.repo", "reponame": "acme/thing" } - }] - }); - let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); - assert!(err.contains("declares no input `reponame`"), "{err}"); - assert!(err.contains("repo, depth"), "says what it does take: {err}"); -} - -#[test] -fn an_input_reference_to_something_undeclared_is_refused() { - let recipe = json!({ - "why": "reference an input that does not exist", - "declared": [], - "steps": [{ - "id": "audit", "use": "pr-audit-review", "with": { "repo": "@input.repo" } - }] - }); - let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); - assert!(err.contains("did not declare"), "{err}"); -} - -#[test] -fn reads_on_a_use_step_points_at_with_instead() { - let recipe = json!({ - "why": "reads does not apply", - "declared": [{ "name": "repo", "description": "owner/name", "required": true }], - "steps": [ - { "id": "diff", "run": "git diff" }, - { "id": "audit", "use": "pr-audit-review", "reads": ["diff"], - "with": { "repo": "@input.repo" } } - ] - }); - let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); - assert!(err.contains("`with`"), "{err}"); -} - -#[test] -fn a_step_reading_a_use_step_gets_the_childs_answer_not_its_run_state() { - // The defect this projection exists for: a `sub_workflow` node emits the - // child's ENTIRE final run state, so a naive read hands the next agent the - // child's bookkeeping with the deliverable buried in it. - let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); - let summary = graph - .nodes - .iter() - .find(|node| node.id == "summary") - .expect("the ask step"); - let prompt = summary.config["prompt"] - .as_str() - .expect("a generated prompt expression"); - - // Run the generated expression against a real child run state, through the - // engine's own evaluator — the only thing that proves the projection is - // valid jq and picks the right leaves. - let state = json!({ - "run": { "inputs": { "repo": "acme/thing" } }, - "inputs": { "repo": "acme/thing" }, - "nodes": { "audit": { "item": child_run_state(), "items": [child_run_state()] } } - }); - let rendered = tinyflows::expr::resolve(&json!(prompt), &state); - let rendered = rendered.as_str().expect("resolves to a string"); - - assert!( - rendered.contains("Requesting changes."), - "the child's deliverable must reach the reader: {rendered}" - ); - assert!( - rendered.contains("3 files changed"), - "and so must every other leaf it produced: {rendered}" - ); - assert!( - rendered.contains("## verdict"), - "each labelled with the child step it came from: {rendered}" - ); - assert!( - !rendered.contains("trigger"), - "but not the child's own bookkeeping: {rendered}" - ); -} - -#[test] -fn a_child_that_produced_nothing_readable_says_so_rather_than_erroring() { - // The projection walks a state this graph did not choose the shape of, so - // every hop is written defensively; a jq error here would fail the parent - // node instead of reporting the step that produced nothing. - let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); - let prompt = graph - .nodes - .iter() - .find(|node| node.id == "summary") - .expect("the ask step") - .config["prompt"] - .as_str() - .expect("a generated prompt expression") - .to_string(); - - for state in [ - json!({ "run": {}, "nodes": { "audit": { "item": { "nodes": {} }, "items": [] } } }), - json!({ "run": {}, "nodes": { "audit": { "item": null, "items": [] } } }), - json!({ "run": {}, "nodes": {} }), - ] { - let rendered = tinyflows::expr::resolve(&json!(prompt), &state); - let rendered = rendered.as_str().unwrap_or_default(); - assert!( - rendered.contains("(no output)"), - "empty child state {state} rendered: {rendered}" - ); - } -} - -#[test] -fn the_callable_listing_names_the_inputs_a_call_must_fill() { - // A model asked to supply `with` for inputs it was never shown is a model - // guessing — the same defect the chooser had. - let rendered = super::render_callables(&[audit()]); - assert!(rendered.contains("pr-audit-review"), "{rendered}"); - assert!( - rendered.contains("with: repo, depth (optional)"), - "{rendered}" - ); - assert!( - super::render_callables(&[]).is_empty(), - "a cold store offers no `use` list at all, rather than an empty one" - ); -} - -/// A child workflow's final run state, in the shape the engine records it. -/// -/// Raw run-state slots, so items are serialized (`{"json": …}`) — one shape in -/// from the parent's scope projection, which exposes bare payloads. Getting -/// that boundary wrong is precisely what `child_answer` has to survive. -fn child_run_state() -> serde_json::Value { - json!({ - "run": { "trigger": [], "inputs": { "repo": "acme/thing" } }, - "nodes": { - // The trigger slot, verbatim from a real run: its payload is the - // seeded item ARRAY, not an object. Every child has one, and it is - // what made the first spelling of the projection fail — a - // fixture whose slots were all objects passed while the real - // thing resolved the whole prompt to null. - "start": { "items": [{ "json": [{ "json": {} }] }] }, - "fetch_pr": { "items": [{ "json": { - "json": { "exit_code": 0, "stdout": "3 files changed" }, - "text": null, "raw": {} - } }] }, - "verdict": { "items": [{ "json": { - "json": { "text": "Requesting changes.", "worker": "local" }, - "text": "Requesting changes.", - "raw": { "text": "Requesting changes." } - } }] } - } - }) -} - -#[test] -fn an_agents_prose_is_read_from_the_envelopes_text_not_from_inside_its_json() { - // The regression this file exists to prevent, found in this file's own - // output: `item.json.text` reads a `text` field inside the STRUCTURED - // value, which a prose reply has not got, so every `reads` of an agent - // step rendered "(no output)". Evaluated rather than string-matched — - // asserting the path spelling is what let the wrong spelling ship. - let recipe = json!({ - "why": "chain of agents", - "steps": [ - { "id": "draft", "ask": "Draft it." }, - { "id": "polish", "ask": "Polish the draft.", "reads": ["draft"] } - ] - }); - let (graph, _, _) = lower(&recipe, &[]).expect("lowers"); - let prompt = graph.nodes[2].config["prompt"].as_str().expect("prompt"); - - let scope = json!({ - "run": {}, "inputs": null, "item": null, "items": [], - "nodes": { "draft": { "item": { - "json": "The draft, in prose.", - "text": "The draft, in prose.", - "raw": "The draft, in prose." - } } } - }); - let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); - let rendered = rendered.as_str().expect("resolves to a string"); - assert!( - rendered.contains("The draft, in prose."), - "an upstream agent's reply must reach the next step: {rendered}" - ); - assert!( - !rendered.contains("(no output)"), - "it rendered the missing-value marker instead: {rendered}" - ); -} - -#[test] -fn a_scripts_stdout_is_read_from_inside_its_json_because_that_is_where_it_is() { - // The asymmetry that made the agent path look right: a shell node's - // structured value genuinely holds `{exit_code, stdout}`, so this one IS - // nested. Pinned by evaluation so the two never get "harmonised". - let (graph, _, _) = lower(&review_recipe(), &[]).expect("lowers"); - let prompt = graph.nodes[2].config["prompt"].as_str().expect("prompt"); - let scope = json!({ - "run": { "inputs": { "repo": "acme/thing" } }, - "inputs": { "repo": "acme/thing" }, "item": null, "items": [], - "nodes": { "fetch": { "item": { - "json": { "exit_code": 0, "stdout": "#41 flaky test" }, - "text": null, "raw": {} - } } } - }); - let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); - let rendered = rendered.as_str().expect("resolves to a string"); - assert!(rendered.contains("#41 flaky test"), "{rendered}"); - assert!( - rendered.contains("acme/thing"), - "declared inputs too: {rendered}" - ); -} - -#[test] -fn an_errand_lowers_to_one_agent_turn_that_validates() { - let graph = super::errand("how much disk is this directory using").expect("lowers"); - - assert!( - validate_all(&graph).is_empty(), - "the engine must accept it: {:?}", - validate_all(&graph) - ); - // A trigger and exactly one agent node. Anything more means the errand - // grew a procedure, which is the one thing it is defined as not having. - assert_eq!(graph.nodes.len(), 2, "{:?}", graph.nodes); - assert_eq!(graph.nodes[0].kind, NodeKind::Trigger); - assert_eq!(graph.nodes[1].kind, NodeKind::Agent); - assert!( - graph.inputs.is_empty(), - "an errand declares nothing: it is answered from the goal alone" - ); -} - -#[test] -fn an_errands_prompt_actually_carries_the_goal() { - // Evaluated, not string-matched. The `item.json.text` defect shipped past a - // reviewer *and* a test because both read the expression instead of running - // it — a prompt that resolves to nothing looks fine as source. - let graph = super::errand(" how much disk is this directory using ").expect("lowers"); - let prompt = graph.nodes[1].config["prompt"] - .as_str() - .expect("the agent node carries a prompt"); - let scope = json!({ - "run": { "inputs": {} }, "inputs": {}, - "item": null, "items": [], "nodes": {} - }); - let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); - let rendered = rendered.as_str().expect("resolves to a string"); - assert_eq!( - rendered, "how much disk is this directory using", - "the goal, trimmed, and nothing else bolted on" - ); -} - -#[test] -fn an_errand_is_the_same_lowering_an_authored_ask_gets() { - // Why `errand` goes through `lower` rather than building two nodes by hand: - // a second definition of what an `ask` compiles to would drift silently the - // first time the envelope path changed. - let errand = super::errand("say something").expect("lowers"); - let (authored, _, _) = lower( - &json!({ - "why": "one turn of work, no procedure in it", - "declared": [], "inputs": {}, - "steps": [{ "id": "errand", "ask": "say something" }] - }), - &[], - ) - .expect("lowers"); - assert_eq!(errand.nodes[1].config, authored.nodes[1].config); -} - -#[test] -fn every_control_character_in_a_goal_survives_as_a_valid_jq_literal() { - let scope = json!({ "run": { "inputs": {} }, "inputs": {}, "item": null, - "items": [], "nodes": {} }); - let mut broke = Vec::new(); - for code in 0u32..0x20 { - let ch = char::from_u32(code).expect("control char"); - let goal = format!("disk{ch}usage"); - let Ok(graph) = super::errand(&goal) else { - broke.push(format!("U+{code:04X}: lowering refused it")); - continue; - }; - let prompt = graph.nodes[1].config["prompt"].as_str().expect("prompt"); - let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); - if rendered.as_str().is_none() { - broke.push(format!("U+{code:04X}: {rendered:?}")); - } - } - assert!( - broke.is_empty(), - "control characters that broke the prompt: {broke:?}" - ); -} +include!("recipe_tests/recipe_part_01_tests.rs"); +include!("recipe_tests/recipe_part_02_tests.rs"); diff --git a/crates/tinyflows-adaptive/src/intake/recipe_tests/recipe_part_01_tests.rs b/crates/tinyflows-adaptive/src/intake/recipe_tests/recipe_part_01_tests.rs new file mode 100644 index 00000000..763512ac --- /dev/null +++ b/crates/tinyflows-adaptive/src/intake/recipe_tests/recipe_part_01_tests.rs @@ -0,0 +1,319 @@ +fn audit() -> Callable { + Callable { + id: "pr-audit-review".to_string(), + name: "PR audit review".to_string(), + description: "reviews a pull request and posts the verdict".to_string(), + inputs: vec![("repo".to_string(), true), ("depth".to_string(), false)], + } +} + +fn compose_recipe() -> serde_json::Value { + json!({ + "why": "audit the PR, then summarise what it found", + "declared": [ + { "name": "repo", "description": "owner/name", "required": true } + ], + "inputs": { "repo": "acme/thing" }, + "steps": [ + { "id": "audit", "use": "pr-audit-review", "with": { "repo": "@input.repo" } }, + { "id": "summary", "ask": "Summarise the audit in three bullets.", + "reads": ["audit"] } + ] + }) +} + +#[test] +fn a_use_step_lowers_to_a_sub_workflow_node_that_references_the_callee() { + let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); + assert!( + validate_all(&graph).is_empty(), + "{:?}", + validate_all(&graph) + ); + let node = graph + .nodes + .iter() + .find(|node| node.id == "audit") + .expect("the use step became a node"); + assert_eq!(node.kind, NodeKind::SubWorkflow); + // By reference, never inlined: the callee keeps its own identity, its own + // scores, and whatever it becomes next. + assert_eq!(node.config["workflow_id"], json!("pr-audit-review")); + assert!( + node.config.get("workflow").is_none(), + "an inlined child would fork the callee at authoring time" + ); + // `@input.repo` became the expression that reads the parent's run input. + assert_eq!( + node.config["inputs"]["repo"], + json!("=.run.inputs[\"repo\"]") + ); +} + +#[test] +fn the_lowered_sub_workflow_config_satisfies_the_engines_own_contract() { + // The same drift guard the shell lowering has, for the same reason: the + // `use` step's whole value is that the engine already knows how to run a + // child, and it knows it by reading `workflow_id` and `inputs`. A rename on + // either side would surface as a capability error mid-run, attributed to + // the work rather than to the plan. + let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); + let config = &graph + .nodes + .iter() + .find(|node| node.id == "audit") + .expect("the use step became a node") + .config; + let contract = tinyflows::catalog::all_contracts() + .iter() + .find(|contract| contract.kind == "sub_workflow") + .expect("the engine has a sub_workflow contract") + .clone(); + let fields: Vec<&str> = contract + .config_fields + .iter() + .map(|field| field.name.as_str()) + .collect(); + for key in ["workflow_id", "inputs"] { + assert!( + fields.contains(&key), + "the engine's sub_workflow contract no longer declares `{key}`: {fields:?}" + ); + assert!( + config.get(key).is_some(), + "a lowered use step must fill config.{key}: {config}" + ); + } + // Required fields are the engine's own list; filling one is not enough if + // it grows another. + for field in contract.config_fields.iter().filter(|field| field.required) { + assert!( + config.get(&field.name).is_some(), + "the lowering fills none of the engine's required sub_workflow field \ + `{}`: {config}", + field.name + ); + } +} + +#[test] +fn a_step_reference_in_with_reads_the_earlier_step_the_way_its_kind_produces() { + let recipe = json!({ + "why": "fetch the diff, then hand it to a saved reviewer", + "declared": [], + "steps": [ + { "id": "diff", "run": "git diff" }, + { "id": "review", "use": "reviewer", "with": { "patch": "@step.diff" } } + ] + }); + let callable = Callable { + id: "reviewer".to_string(), + name: String::new(), + description: "reviews a patch".to_string(), + inputs: vec![("patch".to_string(), true)], + }; + let (graph, _, _) = lower(&recipe, &[callable]).expect("lowers"); + let node = graph + .nodes + .iter() + .find(|node| node.id == "review") + .expect("the use step became a node"); + // A run step's output is its stdout, and the reference knows that without + // the model having to. + assert_eq!( + node.config["inputs"]["patch"], + json!("=.nodes[\"diff\"].item.json.stdout") + ); +} + +#[test] +fn a_required_input_present_but_empty_is_refused_the_way_an_absent_one_is() { + // `contains_key` accepts `null` and `""`, which `forward` then passes + // through unchanged — so the child fails its OWN declaration check + // mid-run, which is the failure this refusal exists to move to intake. + // `gated` in `author.rs` already reads unfilled this way; the two checks + // disagreeing is what let the value through. + for empty in [json!(null), json!("")] { + let recipe = json!({ + "why": "audit the PR", + "declared": [], + "steps": [ + { "id": "audit", "use": "pr-audit-review", "with": { "repo": empty } } + ] + }); + let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); + assert!( + err.contains("requires the input `repo`"), + "an empty value is not a supplied value ({empty}): {err}" + ); + } +} + +#[test] +fn a_use_step_naming_a_workflow_nobody_offered_is_refused_at_intake() { + // Not deferred to the resolver: a hallucinated id would surface as a + // capability error mid-run, after the earlier steps had already been paid + // for, and be attributed to the work rather than to the plan. + let err = lower(&compose_recipe(), &[]) + .expect_err("refused") + .to_string(); + assert!(err.contains("no saved workflows to call"), "{err}"); + + let other = Callable { + id: "something-else".to_string(), + ..audit() + }; + let err = lower(&compose_recipe(), &[other]) + .expect_err("refused") + .to_string(); + assert!( + err.contains("something-else"), + "names what IS callable: {err}" + ); +} + +#[test] +fn a_use_step_that_omits_a_required_input_is_refused_with_the_name() { + let recipe = json!({ + "why": "call it with nothing", + "steps": [{ "id": "audit", "use": "pr-audit-review", "with": {} }] + }); + let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); + assert!(err.contains("requires the input `repo`"), "{err}"); +} + +#[test] +fn a_with_key_the_callee_never_declared_is_refused_rather_than_dropped() { + // Silently dropping it would leave the model believing it passed a value + // the child will never see — the worst kind of pass, because the run + // completes. + let recipe = json!({ + "why": "wrong input name", + "declared": [{ "name": "repo", "description": "owner/name", "required": true }], + "steps": [{ + "id": "audit", "use": "pr-audit-review", + "with": { "repo": "@input.repo", "reponame": "acme/thing" } + }] + }); + let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); + assert!(err.contains("declares no input `reponame`"), "{err}"); + assert!(err.contains("repo, depth"), "says what it does take: {err}"); +} + +#[test] +fn an_input_reference_to_something_undeclared_is_refused() { + let recipe = json!({ + "why": "reference an input that does not exist", + "declared": [], + "steps": [{ + "id": "audit", "use": "pr-audit-review", "with": { "repo": "@input.repo" } + }] + }); + let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); + assert!(err.contains("did not declare"), "{err}"); +} + +#[test] +fn reads_on_a_use_step_points_at_with_instead() { + let recipe = json!({ + "why": "reads does not apply", + "declared": [{ "name": "repo", "description": "owner/name", "required": true }], + "steps": [ + { "id": "diff", "run": "git diff" }, + { "id": "audit", "use": "pr-audit-review", "reads": ["diff"], + "with": { "repo": "@input.repo" } } + ] + }); + let err = lower(&recipe, &[audit()]).expect_err("refused").to_string(); + assert!(err.contains("`with`"), "{err}"); +} + +#[test] +fn a_step_reading_a_use_step_gets_the_childs_answer_not_its_run_state() { + // The defect this projection exists for: a `sub_workflow` node emits the + // child's ENTIRE final run state, so a naive read hands the next agent the + // child's bookkeeping with the deliverable buried in it. + let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); + let summary = graph + .nodes + .iter() + .find(|node| node.id == "summary") + .expect("the ask step"); + let prompt = summary.config["prompt"] + .as_str() + .expect("a generated prompt expression"); + + // Run the generated expression against a real child run state, through the + // engine's own evaluator — the only thing that proves the projection is + // valid jq and picks the right leaves. + let state = json!({ + "run": { "inputs": { "repo": "acme/thing" } }, + "inputs": { "repo": "acme/thing" }, + "nodes": { "audit": { "item": child_run_state(), "items": [child_run_state()] } } + }); + let rendered = tinyflows::expr::resolve(&json!(prompt), &state); + let rendered = rendered.as_str().expect("resolves to a string"); + + assert!( + rendered.contains("Requesting changes."), + "the child's deliverable must reach the reader: {rendered}" + ); + assert!( + rendered.contains("3 files changed"), + "and so must every other leaf it produced: {rendered}" + ); + assert!( + rendered.contains("## verdict"), + "each labelled with the child step it came from: {rendered}" + ); + assert!( + !rendered.contains("trigger"), + "but not the child's own bookkeeping: {rendered}" + ); +} + +#[test] +fn a_child_that_produced_nothing_readable_says_so_rather_than_erroring() { + // The projection walks a state this graph did not choose the shape of, so + // every hop is written defensively; a jq error here would fail the parent + // node instead of reporting the step that produced nothing. + let (graph, _, _) = lower(&compose_recipe(), &[audit()]).expect("lowers"); + let prompt = graph + .nodes + .iter() + .find(|node| node.id == "summary") + .expect("the ask step") + .config["prompt"] + .as_str() + .expect("a generated prompt expression") + .to_string(); + + for state in [ + json!({ "run": {}, "nodes": { "audit": { "item": { "nodes": {} }, "items": [] } } }), + json!({ "run": {}, "nodes": { "audit": { "item": null, "items": [] } } }), + json!({ "run": {}, "nodes": {} }), + ] { + let rendered = tinyflows::expr::resolve(&json!(prompt), &state); + let rendered = rendered.as_str().unwrap_or_default(); + assert!( + rendered.contains("(no output)"), + "empty child state {state} rendered: {rendered}" + ); + } +} + +#[test] +fn the_callable_listing_names_the_inputs_a_call_must_fill() { + // A model asked to supply `with` for inputs it was never shown is a model + // guessing — the same defect the chooser had. + let rendered = super::render_callables(&[audit()]); + assert!(rendered.contains("pr-audit-review"), "{rendered}"); + assert!( + rendered.contains("with: repo, depth (optional)"), + "{rendered}" + ); + assert!( + super::render_callables(&[]).is_empty(), + "a cold store offers no `use` list at all, rather than an empty one" + ); +} diff --git a/crates/tinyflows-adaptive/src/intake/recipe_tests/recipe_part_02_tests.rs b/crates/tinyflows-adaptive/src/intake/recipe_tests/recipe_part_02_tests.rs new file mode 100644 index 00000000..91e1230a --- /dev/null +++ b/crates/tinyflows-adaptive/src/intake/recipe_tests/recipe_part_02_tests.rs @@ -0,0 +1,171 @@ +/// A child workflow's final run state, in the shape the engine records it. +/// +/// Raw run-state slots, so items are serialized (`{"json": …}`) — one shape in +/// from the parent's scope projection, which exposes bare payloads. Getting +/// that boundary wrong is precisely what `child_answer` has to survive. +fn child_run_state() -> serde_json::Value { + json!({ + "run": { "trigger": [], "inputs": { "repo": "acme/thing" } }, + "nodes": { + // The trigger slot, verbatim from a real run: its payload is the + // seeded item ARRAY, not an object. Every child has one, and it is + // what made the first spelling of the projection fail — a + // fixture whose slots were all objects passed while the real + // thing resolved the whole prompt to null. + "start": { "items": [{ "json": [{ "json": {} }] }] }, + "fetch_pr": { "items": [{ "json": { + "json": { "exit_code": 0, "stdout": "3 files changed" }, + "text": null, "raw": {} + } }] }, + "verdict": { "items": [{ "json": { + "json": { "text": "Requesting changes.", "worker": "local" }, + "text": "Requesting changes.", + "raw": { "text": "Requesting changes." } + } }] } + } + }) +} + +#[test] +fn an_agents_prose_is_read_from_the_envelopes_text_not_from_inside_its_json() { + // The regression this file exists to prevent, found in this file's own + // output: `item.json.text` reads a `text` field inside the STRUCTURED + // value, which a prose reply has not got, so every `reads` of an agent + // step rendered "(no output)". Evaluated rather than string-matched — + // asserting the path spelling is what let the wrong spelling ship. + let recipe = json!({ + "why": "chain of agents", + "steps": [ + { "id": "draft", "ask": "Draft it." }, + { "id": "polish", "ask": "Polish the draft.", "reads": ["draft"] } + ] + }); + let (graph, _, _) = lower(&recipe, &[]).expect("lowers"); + let prompt = graph.nodes[2].config["prompt"].as_str().expect("prompt"); + + let scope = json!({ + "run": {}, "inputs": null, "item": null, "items": [], + "nodes": { "draft": { "item": { + "json": "The draft, in prose.", + "text": "The draft, in prose.", + "raw": "The draft, in prose." + } } } + }); + let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); + let rendered = rendered.as_str().expect("resolves to a string"); + assert!( + rendered.contains("The draft, in prose."), + "an upstream agent's reply must reach the next step: {rendered}" + ); + assert!( + !rendered.contains("(no output)"), + "it rendered the missing-value marker instead: {rendered}" + ); +} + +#[test] +fn a_scripts_stdout_is_read_from_inside_its_json_because_that_is_where_it_is() { + // The asymmetry that made the agent path look right: a shell node's + // structured value genuinely holds `{exit_code, stdout}`, so this one IS + // nested. Pinned by evaluation so the two never get "harmonised". + let (graph, _, _) = lower(&review_recipe(), &[]).expect("lowers"); + let prompt = graph.nodes[2].config["prompt"].as_str().expect("prompt"); + let scope = json!({ + "run": { "inputs": { "repo": "acme/thing" } }, + "inputs": { "repo": "acme/thing" }, "item": null, "items": [], + "nodes": { "fetch": { "item": { + "json": { "exit_code": 0, "stdout": "#41 flaky test" }, + "text": null, "raw": {} + } } } + }); + let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); + let rendered = rendered.as_str().expect("resolves to a string"); + assert!(rendered.contains("#41 flaky test"), "{rendered}"); + assert!( + rendered.contains("acme/thing"), + "declared inputs too: {rendered}" + ); +} + +#[test] +fn an_errand_lowers_to_one_agent_turn_that_validates() { + let graph = super::errand("how much disk is this directory using").expect("lowers"); + + assert!( + validate_all(&graph).is_empty(), + "the engine must accept it: {:?}", + validate_all(&graph) + ); + // A trigger and exactly one agent node. Anything more means the errand + // grew a procedure, which is the one thing it is defined as not having. + assert_eq!(graph.nodes.len(), 2, "{:?}", graph.nodes); + assert_eq!(graph.nodes[0].kind, NodeKind::Trigger); + assert_eq!(graph.nodes[1].kind, NodeKind::Agent); + assert!( + graph.inputs.is_empty(), + "an errand declares nothing: it is answered from the goal alone" + ); +} + +#[test] +fn an_errands_prompt_actually_carries_the_goal() { + // Evaluated, not string-matched. The `item.json.text` defect shipped past a + // reviewer *and* a test because both read the expression instead of running + // it — a prompt that resolves to nothing looks fine as source. + let graph = super::errand(" how much disk is this directory using ").expect("lowers"); + let prompt = graph.nodes[1].config["prompt"] + .as_str() + .expect("the agent node carries a prompt"); + let scope = json!({ + "run": { "inputs": {} }, "inputs": {}, + "item": null, "items": [], "nodes": {} + }); + let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); + let rendered = rendered.as_str().expect("resolves to a string"); + assert_eq!( + rendered, "how much disk is this directory using", + "the goal, trimmed, and nothing else bolted on" + ); +} + +#[test] +fn an_errand_is_the_same_lowering_an_authored_ask_gets() { + // Why `errand` goes through `lower` rather than building two nodes by hand: + // a second definition of what an `ask` compiles to would drift silently the + // first time the envelope path changed. + let errand = super::errand("say something").expect("lowers"); + let (authored, _, _) = lower( + &json!({ + "why": "one turn of work, no procedure in it", + "declared": [], "inputs": {}, + "steps": [{ "id": "errand", "ask": "say something" }] + }), + &[], + ) + .expect("lowers"); + assert_eq!(errand.nodes[1].config, authored.nodes[1].config); +} + +#[test] +fn every_control_character_in_a_goal_survives_as_a_valid_jq_literal() { + let scope = json!({ "run": { "inputs": {} }, "inputs": {}, "item": null, + "items": [], "nodes": {} }); + let mut broke = Vec::new(); + for code in 0u32..0x20 { + let ch = char::from_u32(code).expect("control char"); + let goal = format!("disk{ch}usage"); + let Ok(graph) = super::errand(&goal) else { + broke.push(format!("U+{code:04X}: lowering refused it")); + continue; + }; + let prompt = graph.nodes[1].config["prompt"].as_str().expect("prompt"); + let rendered = tinyflows::expr::resolve(&json!(prompt), &scope); + if rendered.as_str().is_none() { + broke.push(format!("U+{code:04X}: {rendered:?}")); + } + } + assert!( + broke.is_empty(), + "control characters that broke the prompt: {broke:?}" + ); +} diff --git a/crates/tinyflows-adaptive/src/ledger/conformance.rs b/crates/tinyflows-adaptive/src/ledger/conformance.rs index b72689f1..da4e36d8 100644 --- a/crates/tinyflows-adaptive/src/ledger/conformance.rs +++ b/crates/tinyflows-adaptive/src/ledger/conformance.rs @@ -198,591 +198,7 @@ async fn workflow_scores_accumulate(store: &dyn Ledger) { ); } -/// Run every tenant-isolation case. -/// -/// Separate from [`run_all`] because it needs three handles onto **one** -/// store — global, and two tenants — and how a backend makes a scoped handle -/// is its own business (`for_tenant` on both that ship). A backend that does -/// not support scoping simply does not call this. -/// -/// # Panics -/// On any isolation failure. Each one is a leak of one tenant's knowledge into -/// another's prompt, so none of them is a soft assertion. -pub async fn run_tenants(global: &dyn Ledger, a: &dyn Ledger, b: &dyn Ledger) { - assert_eq!(global.scope(), None, "the global handle must be unscoped"); - assert!(a.scope().is_some() && b.scope().is_some(), "both scoped"); - assert_ne!(a.scope(), b.scope(), "two different tenants"); - - a_tenants_lesson_is_invisible_to_another(a, b).await; - a_global_lesson_is_visible_to_every_tenant(global, a, b).await; - promote_stamps_the_handle_not_the_argument(a).await; - workflow_scores_do_not_bleed_between_tenants(a, b).await; - a_tenant_writing_does_not_move_the_global_score(global, a).await; - an_episode_id_alone_does_not_reach_another_tenants_attempts(a, b).await; - naming_another_tenants_lesson_id_does_not_move_its_score(global, a, b).await; -} - -async fn naming_another_tenants_lesson_id_does_not_move_its_score( - global: &dyn Ledger, - a: &dyn Ledger, - b: &dyn Ledger, -) { - // The ids reaching `score_lesson` come from model output (corroboration), - // so this is a hole a prompt injection walks through if the backend - // updates by id alone. - let private = a - .promote(&lesson("a private class of situation"), &[]) - .await - .expect("promote"); - b.score_lesson(&private, true) - .await - .expect("no-op, not error"); - let untouched = a - .lessons(None) - .await - .expect("lessons") - .into_iter() - .find(|l| l.id == private) - .expect("still there"); - assert_eq!( - (untouched.applied, untouched.helped), - (0, 0), - "tenant {:?} moved tenant {:?}'s score by naming its id", - b.scope(), - a.scope() - ); - - // A global lesson is visible to every tenant, so scoring it is legitimate. - let shared = global - .promote(&lesson("a class anyone can hit"), &[]) - .await - .expect("promote"); - b.score_lesson(&shared, true).await.expect("score"); - let moved = b - .lessons(None) - .await - .expect("lessons") - .into_iter() - .find(|l| l.id == shared) - .expect("visible"); - assert_eq!((moved.applied, moved.helped), (1, 1)); -} - -async fn an_episode_id_alone_does_not_reach_another_tenants_attempts( - a: &dyn Ledger, - b: &dyn Ledger, -) { - // An episode id is opaque and a service may hand one straight through from - // a request path. Being keyed by episode is not isolation — guessing an id - // would be enough — so the rows carry the bucket too. - a.append(&row("ep-secret", 1, "authored:aaa")) - .await - .expect("append"); - assert_eq!(a.rows("ep-secret").await.expect("rows").len(), 1); - assert!( - b.rows("ep-secret").await.expect("rows").is_empty(), - "tenant {:?} read tenant {:?}'s attempts by knowing the episode id", - b.scope(), - a.scope() - ); -} - -async fn a_tenants_lesson_is_invisible_to_another(a: &dyn Ledger, b: &dyn Ledger) { - let mut mine = lesson("a private class of task"); - mine.claim = "names an internal repository path".into(); - let id = a.promote(&mine, &[]).await.expect("promote"); - - let seen_by_a = a.lessons(None).await.expect("lessons"); - assert!( - seen_by_a.iter().any(|l| l.id == id), - "a tenant must see its own lesson" - ); - - let seen_by_b = b.lessons(None).await.expect("lessons"); - assert!( - !seen_by_b.iter().any(|l| l.id == id), - "tenant {:?} can read tenant {:?}'s lesson — this is the leak the scope exists to stop", - b.scope(), - a.scope() - ); -} - -async fn a_global_lesson_is_visible_to_every_tenant( - global: &dyn Ledger, - a: &dyn Ledger, - b: &dyn Ledger, -) { - let id = global - .promote(&lesson("a class of task anyone can hit"), &[]) - .await - .expect("promote"); - for tenant in [a, b] { - let seen = tenant.lessons(None).await.expect("lessons"); - assert!( - seen.iter().any(|l| l.id == id), - "tenant {:?} cannot see a global lesson", - tenant.scope() - ); - } -} - -async fn promote_stamps_the_handle_not_the_argument(a: &dyn Ledger) { - // A caller — or a model whose answer was deserialized straight into a - // `Lesson` — must not be able to publish into another bucket by asking. - let mut forged = lesson("a class of task claiming to be someone else's"); - forged.scope_key = Some("some-other-tenant".to_string()); - let id = a.promote(&forged, &[]).await.expect("promote"); - - let stored = a - .lessons(None) - .await - .expect("lessons") - .into_iter() - .find(|l| l.id == id) - .expect("stored"); - assert_eq!( - stored.scope_key.as_deref(), - a.scope(), - "promote must stamp the handle's scope, whatever the argument said" - ); -} - -async fn workflow_scores_do_not_bleed_between_tenants(a: &dyn Ledger, b: &dyn Ledger) { - let id = "wf-shared-id"; - a.score_workflow(id, true).await.expect("score"); - a.score_workflow(id, true).await.expect("score"); - b.score_workflow(id, false).await.expect("score"); - - let for_a = a.workflow_score(id).await.expect("score"); - let for_b = b.workflow_score(id).await.expect("score"); - assert_eq!( - (for_a.applied, for_a.helped), - (2, 2), - "tenant a's own record" - ); - assert_eq!( - (for_b.applied, for_b.helped), - (1, 0), - "tenant b's own record" - ); -} - -async fn a_tenant_writing_does_not_move_the_global_score(global: &dyn Ledger, a: &dyn Ledger) { - let id = "wf-tenant-only"; - a.score_workflow(id, true).await.expect("score"); - let seen = global.workflow_score(id).await.expect("score"); - assert_eq!( - (seen.applied, seen.helped), - (0, 0), - "the global bucket is its own bucket, not a union of every tenant's" - ); -} - -/// Run every lineage case. Part of [`run_all`]'s contract for any backend that -/// stores variant links, which is both that ship. -/// -/// # Panics -/// On any lineage failure. -pub async fn run_lineage(store: &dyn Ledger) { - an_unlinked_workflow_is_a_family_of_one(store).await; - lineage_reads_the_same_from_any_member(store).await; - linking_the_same_pair_twice_is_a_no_op(store).await; - a_variant_of_a_variant_stays_in_one_family(store).await; - a_cycle_is_truncated_rather_than_hung(store).await; -} - -async fn an_unlinked_workflow_is_a_family_of_one(store: &dyn Ledger) { - let family = store.lineage("wf-lonely").await.expect("lineage"); - assert_eq!(family, vec!["wf-lonely".to_string()]); -} - -async fn lineage_reads_the_same_from_any_member(store: &dyn Ledger) { - store - .link_variant("wf-a", "wf-a-fix-1") - .await - .expect("link"); - store - .link_variant("wf-a", "wf-a-fix-2") - .await - .expect("link"); - - let from_root = store.lineage("wf-a").await.expect("lineage"); - let from_leaf = store.lineage("wf-a-fix-2").await.expect("lineage"); - assert_eq!( - from_root, from_leaf, - "the champion must not depend on which member was asked" - ); - assert_eq!( - from_root[0], "wf-a", - "root first — the fallback relies on it" - ); - assert_eq!(from_root.len(), 3); -} - -async fn linking_the_same_pair_twice_is_a_no_op(store: &dyn Ledger) { - // A repair converging on an existing variant id will re-link. It must not - // duplicate the family member. - store - .link_variant("wf-b", "wf-b-fix-1") - .await - .expect("link"); - store - .link_variant("wf-b", "wf-b-fix-1") - .await - .expect("link"); - assert_eq!(store.lineage("wf-b").await.expect("lineage").len(), 2); -} - -async fn a_variant_of_a_variant_stays_in_one_family(store: &dyn Ledger) { - // `repair` takes whatever ran as the parent, and what ran may itself be a - // variant. Two generations are still one family, or the grandchild would be - // compared against nothing. - store - .link_variant("wf-c", "wf-c-fix-1") - .await - .expect("link"); - store - .link_variant("wf-c-fix-1", "wf-c-fix-2") - .await - .expect("link"); - - let family = store.lineage("wf-c-fix-2").await.expect("lineage"); - assert_eq!(family[0], "wf-c"); - assert_eq!(family.len(), 3, "{family:?}"); -} - -async fn a_cycle_is_truncated_rather_than_hung(store: &dyn Ledger) { - // Nothing should write this, but the ledger is read on the hot path of - // every attempt and a hang there stops the whole loop. Bounded walks mean - // a corrupt link costs a truncated answer instead. - store.link_variant("wf-y", "wf-x").await.expect("link"); - store.link_variant("wf-x", "wf-y").await.expect("link"); - let family = store.lineage("wf-x").await.expect("lineage"); - assert!(family.len() <= super::MAX_FAMILY, "{family:?}"); -} - -/// Run every episode-checkpoint case. -/// -/// # Panics -/// On any failure. Each is a way a restarted process would lose an episode. -pub async fn run_episodes(store: &dyn Ledger) { - an_unknown_episode_is_absent_not_an_error(store).await; - an_episode_round_trips_everything_the_rows_cannot_hold(store).await; - saving_twice_updates_rather_than_duplicating(store).await; - running_only_filters_to_the_recovery_list(store).await; - a_rows_verdict_survives_as_fields_not_as_prose(store).await; - the_episode_list_is_newest_first_on_every_backend(store).await; -} - -async fn the_episode_list_is_newest_first_on_every_backend(store: &dyn Ledger) { - // `Page` documents newest-first, and paging an unordered list returns - // opposite ends on different backends — `Page::first(1)` must mean the - // same episode everywhere. - for (id, at) in [ - ("ep-ord-old", "2026-02-01T00:00:01Z"), - ("ep-ord-new", "2026-02-01T00:00:03Z"), - ("ep-ord-mid", "2026-02-01T00:00:02Z"), - ] { - let mut e = episode(id, EpisodeStatus::Running, 1, 0); - e.updated_at = at.to_string(); - store.save_episode(&e).await.expect("save"); - } - let ordered: Vec = store - .episodes(false, super::Page::ALL) - .await - .expect("episodes") - .into_iter() - .map(|e| e.id) - .filter(|id| id.starts_with("ep-ord-")) - .collect(); - assert_eq!( - ordered, - ["ep-ord-new", "ep-ord-mid", "ep-ord-old"], - "newest first, on this backend as on every other" - ); -} - -fn episode(id: &str, status: EpisodeStatus, attempt: u32, stalled: u32) -> Episode { - Episode { - id: id.to_string(), - goal: crate::contracts::Goal::new("write the weekly report"), - scope_key: None, - status, - attempt, - stalled, - started_at: "2026-01-01T00:00:00Z".to_string(), - updated_at: "2026-01-01T00:00:05Z".to_string(), - } -} - -async fn an_unknown_episode_is_absent_not_an_error(store: &dyn Ledger) { - assert!( - store - .episode("never-started") - .await - .expect("read") - .is_none() - ); -} - -async fn an_episode_round_trips_everything_the_rows_cannot_hold(store: &dyn Ledger) { - let mut want = episode("ep-round", EpisodeStatus::Running, 3, 2); - want.goal.success_criteria = "cites the actual figures".to_string(); - store.save_episode(&want).await.expect("save"); - - let got = store - .episode("ep-round") - .await - .expect("read") - .expect("saved"); - assert_eq!( - got.goal.text, want.goal.text, - "the goal is unrecoverable from rows" - ); - assert_eq!(got.goal.success_criteria, "cites the actual figures"); - assert_eq!(got.attempt, 3); - assert_eq!(got.stalled, 2, "the stall count cannot be recomputed"); - assert_eq!(got.status, EpisodeStatus::Running); -} - -async fn saving_twice_updates_rather_than_duplicating(store: &dyn Ledger) { - store - .save_episode(&episode("ep-twice", EpisodeStatus::Running, 1, 0)) - .await - .expect("save"); - store - .save_episode(&episode( - "ep-twice", - EpisodeStatus::StoodDown("out of attempts after 12".to_string()), - 12, - 0, - )) - .await - .expect("save"); - - let got = store - .episode("ep-twice") - .await - .expect("read") - .expect("saved"); - assert_eq!(got.attempt, 12); - match got.status { - EpisodeStatus::StoodDown(reason) => assert!(reason.contains("out of attempts")), - other => panic!("expected the second write to win, got {other:?}"), - } - let all = store - .episodes(false, super::Page::ALL) - .await - .expect("episodes"); - assert_eq!( - all.iter().filter(|e| e.id == "ep-twice").count(), - 1, - "one episode, not two" - ); -} - -async fn running_only_filters_to_the_recovery_list(store: &dyn Ledger) { - store - .save_episode(&episode("ep-live", EpisodeStatus::Running, 1, 0)) - .await - .expect("save"); - store - .save_episode(&episode("ep-won", EpisodeStatus::Satisfied, 2, 0)) - .await - .expect("save"); - - let running = store - .episodes(true, super::Page::ALL) - .await - .expect("episodes"); - assert!(running.iter().any(|e| e.id == "ep-live")); - assert!( - !running.iter().any(|e| e.id == "ep-won"), - "a finished episode is not resumed" - ); -} - -async fn a_rows_verdict_survives_as_fields_not_as_prose(store: &dyn Ledger) { - // `satisfied` used to be recoverable only by matching the outcome string, - // and `advanced` not at all — so a restart could not recompute the stall. - let mut won = row("ep-fields", 1, "authored:aaa"); - won.satisfied = true; - won.advanced = true; - store.append(&won).await.expect("append"); - - let back = &store.rows("ep-fields").await.expect("rows")[0]; - assert!(back.satisfied); - assert!(back.advanced); -} - -/// Run every transcript and paging case. -/// -/// # Panics -/// On any failure. -pub async fn run_transcripts(store: &dyn Ledger) { - an_attempt_with_no_transcript_is_empty_not_an_error(store).await; - a_transcript_round_trips_in_order(store).await; - a_looped_node_keeps_every_iteration(store).await; - saving_a_transcript_twice_replaces_rather_than_appends(store).await; - a_page_windows_the_episode_list(store).await; - an_agent_step_keeps_its_harness_transcript(store).await; -} - -/// An `agent` step's harness transcript survives the ledger. -/// -/// `Ran::steps` is the archival record — "every node activation, at full record -/// fidelity" — so a backend that persists the step but drops what the harness -/// did inside it satisfies the type and loses the point. -async fn an_agent_step_keeps_its_harness_transcript(store: &dyn Ledger) { - use tinyflows::transcript::TranscriptEntry; - - let entries = vec![ - TranscriptEntry::bounded(1, "agent_thinking", "memoise the chain"), - TranscriptEntry::bounded(2, "tool_call", "shell: python3 solve.py"), - TranscriptEntry::bounded(3, "tool_result", "837799"), - ]; - let mut solve = step("solve", 1); - solve.transcript = entries.clone(); - - store - .save_steps("ldg_transcript", &[solve, step("check", 2)]) - .await - .expect("save"); - - let back = store.steps("ldg_transcript").await.expect("steps"); - assert_eq!(back.len(), 2); - assert_eq!( - back[0].transcript, entries, - "the agent node's transcript round-trips whole and in order" - ); - assert!( - back[1].transcript.is_empty(), - "a step that recorded none still reads as none, not as the previous step's" - ); -} - -fn step(node_id: &str, n: u64) -> crate::execute::StepRecord { - crate::execute::StepRecord { - node_id: node_id.to_string(), - status: crate::execute::StepOutcome::Success, - output: serde_json::json!({ "i": n }), - duration_ms: n, - null_bindings: Vec::new(), - transcript: Vec::new(), - } -} - -async fn an_attempt_with_no_transcript_is_empty_not_an_error(store: &dyn Ledger) { - assert!(store.steps("ldg_nothing").await.expect("steps").is_empty()); -} - -async fn a_transcript_round_trips_in_order(store: &dyn Ledger) { - let mut errored = step("fetch", 7); - errored.status = crate::execute::StepOutcome::Error; - errored.null_bindings = vec![tinyflows::expr::NullResolution { - location: "args.to".to_string(), - expression: "=nodes.x.item.email".to_string(), - }]; - store - .save_steps("ldg_a", &[step("start", 1), errored]) - .await - .expect("save"); - - let back = store.steps("ldg_a").await.expect("steps"); - assert_eq!(back.len(), 2); - assert_eq!(back[0].node_id, "start", "execution order is the record"); - assert_eq!(back[1].status, crate::execute::StepOutcome::Error); - assert_eq!(back[1].duration_ms, 7); - assert_eq!( - back[1].null_bindings.len(), - 1, - "the nested list survives both a JSON column and a native array" - ); - assert_eq!(back[1].output, serde_json::json!({ "i": 7 })); -} - -async fn a_looped_node_keeps_every_iteration(store: &dyn Ledger) { - // The reason this is a record per step rather than one blob per attempt. - let steps: Vec<_> = (0..12).map(|n| step("body", n)).collect(); - store.save_steps("ldg_loop", &steps).await.expect("save"); - - let back = store.steps("ldg_loop").await.expect("steps"); - assert_eq!(back.len(), 12); - assert_eq!( - back.iter().map(|s| s.duration_ms).collect::>(), - (0..12).collect::>(), - "iterations in order, not deduplicated by node id" - ); -} - -async fn saving_a_transcript_twice_replaces_rather_than_appends(store: &dyn Ledger) { - // A retried write must not double the record. - store - .save_steps("ldg_twice", &[step("a", 1), step("b", 2)]) - .await - .expect("save"); - store - .save_steps("ldg_twice", &[step("a", 1), step("b", 2)]) - .await - .expect("save"); - assert_eq!(store.steps("ldg_twice").await.expect("steps").len(), 2); - - // And a SHORTER re-save must not leave the old tail behind — an upsert - // keyed by sequence replaces only the sequences present, and the stitched - // result would read as one transcript mixing two attempts. - store - .save_steps("ldg_twice", &[step("a", 9)]) - .await - .expect("save"); - let back = store.steps("ldg_twice").await.expect("steps"); - assert_eq!(back.len(), 1, "{back:?}"); - assert_eq!( - back[0].duration_ms, 9, - "and it is the new save, not the old" - ); -} - -async fn a_page_windows_the_episode_list(store: &dyn Ledger) { - for n in 0..5 { - store - .save_episode(&episode( - &format!("ep-page-{n}"), - EpisodeStatus::Running, - 1, - 0, - )) - .await - .expect("save"); - } - let all = store - .episodes(false, super::Page::ALL) - .await - .expect("episodes"); - assert!(all.len() >= 5); - - let first_two = store - .episodes(false, super::Page::first(2)) - .await - .expect("episodes"); - assert_eq!(first_two.len(), 2); - assert_eq!( - first_two.iter().map(|e| &e.id).collect::>(), - all[..2].iter().map(|e| &e.id).collect::>(), - "the same order, windowed" - ); - - let past_the_end = store - .episodes( - false, - super::Page { - limit: 10, - offset: all.len() + 5, - }, - ) - .await - .expect("episodes"); - assert!( - past_the_end.is_empty(), - "an offset past the end is empty, not a panic" - ); -} +include!("conformance/tenants.rs"); +include!("conformance/lineage.rs"); +include!("conformance/episodes.rs"); +include!("conformance/transcripts.rs"); diff --git a/crates/tinyflows-adaptive/src/ledger/conformance/episodes.rs b/crates/tinyflows-adaptive/src/ledger/conformance/episodes.rs new file mode 100644 index 00000000..8f9fff8f --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/conformance/episodes.rs @@ -0,0 +1,154 @@ +/// Run every episode-checkpoint case. +/// +/// # Panics +/// On any failure. Each is a way a restarted process would lose an episode. +pub async fn run_episodes(store: &dyn Ledger) { + an_unknown_episode_is_absent_not_an_error(store).await; + an_episode_round_trips_everything_the_rows_cannot_hold(store).await; + saving_twice_updates_rather_than_duplicating(store).await; + running_only_filters_to_the_recovery_list(store).await; + a_rows_verdict_survives_as_fields_not_as_prose(store).await; + the_episode_list_is_newest_first_on_every_backend(store).await; +} + +async fn the_episode_list_is_newest_first_on_every_backend(store: &dyn Ledger) { + // `Page` documents newest-first, and paging an unordered list returns + // opposite ends on different backends — `Page::first(1)` must mean the + // same episode everywhere. + for (id, at) in [ + ("ep-ord-old", "2026-02-01T00:00:01Z"), + ("ep-ord-new", "2026-02-01T00:00:03Z"), + ("ep-ord-mid", "2026-02-01T00:00:02Z"), + ] { + let mut e = episode(id, EpisodeStatus::Running, 1, 0); + e.updated_at = at.to_string(); + store.save_episode(&e).await.expect("save"); + } + let ordered: Vec = store + .episodes(false, super::Page::ALL) + .await + .expect("episodes") + .into_iter() + .map(|e| e.id) + .filter(|id| id.starts_with("ep-ord-")) + .collect(); + assert_eq!( + ordered, + ["ep-ord-new", "ep-ord-mid", "ep-ord-old"], + "newest first, on this backend as on every other" + ); +} + +fn episode(id: &str, status: EpisodeStatus, attempt: u32, stalled: u32) -> Episode { + Episode { + id: id.to_string(), + goal: crate::contracts::Goal::new("write the weekly report"), + scope_key: None, + status, + attempt, + stalled, + started_at: "2026-01-01T00:00:00Z".to_string(), + updated_at: "2026-01-01T00:00:05Z".to_string(), + } +} + +async fn an_unknown_episode_is_absent_not_an_error(store: &dyn Ledger) { + assert!( + store + .episode("never-started") + .await + .expect("read") + .is_none() + ); +} + +async fn an_episode_round_trips_everything_the_rows_cannot_hold(store: &dyn Ledger) { + let mut want = episode("ep-round", EpisodeStatus::Running, 3, 2); + want.goal.success_criteria = "cites the actual figures".to_string(); + store.save_episode(&want).await.expect("save"); + + let got = store + .episode("ep-round") + .await + .expect("read") + .expect("saved"); + assert_eq!( + got.goal.text, want.goal.text, + "the goal is unrecoverable from rows" + ); + assert_eq!(got.goal.success_criteria, "cites the actual figures"); + assert_eq!(got.attempt, 3); + assert_eq!(got.stalled, 2, "the stall count cannot be recomputed"); + assert_eq!(got.status, EpisodeStatus::Running); +} + +async fn saving_twice_updates_rather_than_duplicating(store: &dyn Ledger) { + store + .save_episode(&episode("ep-twice", EpisodeStatus::Running, 1, 0)) + .await + .expect("save"); + store + .save_episode(&episode( + "ep-twice", + EpisodeStatus::StoodDown("out of attempts after 12".to_string()), + 12, + 0, + )) + .await + .expect("save"); + + let got = store + .episode("ep-twice") + .await + .expect("read") + .expect("saved"); + assert_eq!(got.attempt, 12); + match got.status { + EpisodeStatus::StoodDown(reason) => assert!(reason.contains("out of attempts")), + other => panic!("expected the second write to win, got {other:?}"), + } + let all = store + .episodes(false, super::Page::ALL) + .await + .expect("episodes"); + assert_eq!( + all.iter().filter(|e| e.id == "ep-twice").count(), + 1, + "one episode, not two" + ); +} + +async fn running_only_filters_to_the_recovery_list(store: &dyn Ledger) { + store + .save_episode(&episode("ep-live", EpisodeStatus::Running, 1, 0)) + .await + .expect("save"); + store + .save_episode(&episode("ep-won", EpisodeStatus::Satisfied, 2, 0)) + .await + .expect("save"); + + let running = store + .episodes(true, super::Page::ALL) + .await + .expect("episodes"); + assert!(running.iter().any(|e| e.id == "ep-live")); + assert!( + !running.iter().any(|e| e.id == "ep-won"), + "a finished episode is not resumed" + ); +} + +async fn a_rows_verdict_survives_as_fields_not_as_prose(store: &dyn Ledger) { + // `satisfied` used to be recoverable only by matching the outcome string, + // and `advanced` not at all — so a restart could not recompute the stall. + let mut won = row("ep-fields", 1, "authored:aaa"); + won.satisfied = true; + won.advanced = true; + store.append(&won).await.expect("append"); + + let back = &store.rows("ep-fields").await.expect("rows")[0]; + assert!(back.satisfied); + assert!(back.advanced); +} + diff --git a/crates/tinyflows-adaptive/src/ledger/conformance/lineage.rs b/crates/tinyflows-adaptive/src/ledger/conformance/lineage.rs new file mode 100644 index 00000000..fffce54d --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/conformance/lineage.rs @@ -0,0 +1,82 @@ +/// Run every lineage case. Part of [`run_all`]'s contract for any backend that +/// stores variant links. Both shipped backends do. +/// +/// # Panics +/// On any lineage failure. +pub async fn run_lineage(store: &dyn Ledger) { + an_unlinked_workflow_is_a_family_of_one(store).await; + lineage_reads_the_same_from_any_member(store).await; + linking_the_same_pair_twice_is_a_no_op(store).await; + a_variant_of_a_variant_stays_in_one_family(store).await; + a_cycle_is_truncated_rather_than_hung(store).await; +} + +async fn an_unlinked_workflow_is_a_family_of_one(store: &dyn Ledger) { + let family = store.lineage("wf-lonely").await.expect("lineage"); + assert_eq!(family, vec!["wf-lonely".to_string()]); +} + +async fn lineage_reads_the_same_from_any_member(store: &dyn Ledger) { + store + .link_variant("wf-a", "wf-a-fix-1") + .await + .expect("link"); + store + .link_variant("wf-a", "wf-a-fix-2") + .await + .expect("link"); + + let from_root = store.lineage("wf-a").await.expect("lineage"); + let from_leaf = store.lineage("wf-a-fix-2").await.expect("lineage"); + assert_eq!( + from_root, from_leaf, + "the champion must not depend on which member was asked" + ); + assert_eq!( + from_root[0], "wf-a", + "root first — the fallback relies on it" + ); + assert_eq!(from_root.len(), 3); +} + +async fn linking_the_same_pair_twice_is_a_no_op(store: &dyn Ledger) { + // A repair converging on an existing variant id will re-link. It must not + // duplicate the family member. + store + .link_variant("wf-b", "wf-b-fix-1") + .await + .expect("link"); + store + .link_variant("wf-b", "wf-b-fix-1") + .await + .expect("link"); + assert_eq!(store.lineage("wf-b").await.expect("lineage").len(), 2); +} + +async fn a_variant_of_a_variant_stays_in_one_family(store: &dyn Ledger) { + // `repair` takes whatever ran as the parent, and what ran may itself be a + // variant. Two generations are still one family, or the grandchild would be + // compared against nothing. + store + .link_variant("wf-c", "wf-c-fix-1") + .await + .expect("link"); + store + .link_variant("wf-c-fix-1", "wf-c-fix-2") + .await + .expect("link"); + + let family = store.lineage("wf-c-fix-2").await.expect("lineage"); + assert_eq!(family[0], "wf-c"); + assert_eq!(family.len(), 3, "{family:?}"); +} + +async fn a_cycle_is_truncated_rather_than_hung(store: &dyn Ledger) { + // Nothing should write this, but the ledger is read on the hot path of + // every attempt and a hang there stops the whole loop. Bounded walks mean + // a corrupt link costs a truncated answer instead. + store.link_variant("wf-y", "wf-x").await.expect("link"); + store.link_variant("wf-x", "wf-y").await.expect("link"); + let family = store.lineage("wf-x").await.expect("lineage"); + assert!(family.len() <= super::MAX_FAMILY, "{family:?}"); +} diff --git a/crates/tinyflows-adaptive/src/ledger/conformance/tenants.rs b/crates/tinyflows-adaptive/src/ledger/conformance/tenants.rs new file mode 100644 index 00000000..bf1ad730 --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/conformance/tenants.rs @@ -0,0 +1,226 @@ +/// Run every tenant-isolation case. +/// +/// Separate from [`run_all`] because it needs three handles onto **one** +/// store — global, and two tenants — and how a backend makes a scoped handle +/// is its own business (`for_tenant` on both that ship). A backend that does +/// not support scoping simply does not call this. +/// +/// # Panics +/// On any isolation failure. Each one is a leak of one tenant's knowledge into +/// another's prompt, so none of them is a soft assertion. +pub async fn run_tenants(global: &dyn Ledger, a: &dyn Ledger, b: &dyn Ledger) { + assert_eq!(global.scope(), None, "the global handle must be unscoped"); + assert!(a.scope().is_some() && b.scope().is_some(), "both scoped"); + assert_ne!(a.scope(), b.scope(), "two different tenants"); + + a_tenants_lesson_is_invisible_to_another(a, b).await; + a_global_lesson_is_visible_to_every_tenant(global, a, b).await; + a_global_lessons_evidence_is_visible_to_every_tenant(global, a, b).await; + promote_stamps_the_handle_not_the_argument(a).await; + workflow_scores_do_not_bleed_between_tenants(a, b).await; + a_tenant_writing_does_not_move_the_global_score(global, a).await; + an_episode_id_alone_does_not_reach_another_tenants_attempts(a, b).await; + identical_episode_ids_are_stored_once_per_tenant(a, b).await; + naming_another_tenants_lesson_id_does_not_move_its_score(global, a, b).await; +} + +async fn a_global_lessons_evidence_is_visible_to_every_tenant( + global: &dyn Ledger, + a: &dyn Ledger, + b: &dyn Ledger, +) { + let row_id = global + .append(&row("ep-global-evidence", 1, "authored:global")) + .await + .expect("append global evidence"); + let lesson_id = global + .promote(&lesson("a globally useful class of task"), &[row_id]) + .await + .expect("promote global lesson"); + for tenant in [a, b] { + let evidence = tenant.evidence(&lesson_id).await.expect("global evidence"); + assert_eq!(evidence.len(), 1, "tenant {:?}", tenant.scope()); + assert_eq!(evidence[0].episode, "ep-global-evidence"); + } +} + +async fn identical_episode_ids_are_stored_once_per_tenant(a: &dyn Ledger, b: &dyn Ledger) { + let mut for_a = episode("ep-shared-id", EpisodeStatus::Running, 1, 0); + for_a.goal.text = "tenant a goal".to_string(); + let mut for_b = episode("ep-shared-id", EpisodeStatus::Running, 2, 0); + for_b.goal.text = "tenant b goal".to_string(); + + a.save_episode(&for_a).await.expect("save tenant a"); + b.save_episode(&for_b).await.expect("save tenant b"); + + let got_a = a + .episode("ep-shared-id") + .await + .expect("read tenant a") + .expect("tenant a episode"); + let got_b = b + .episode("ep-shared-id") + .await + .expect("read tenant b") + .expect("tenant b episode"); + assert_eq!(got_a.goal.text, "tenant a goal"); + assert_eq!(got_a.attempt, 1); + assert_eq!(got_b.goal.text, "tenant b goal"); + assert_eq!(got_b.attempt, 2); +} + +async fn naming_another_tenants_lesson_id_does_not_move_its_score( + global: &dyn Ledger, + a: &dyn Ledger, + b: &dyn Ledger, +) { + // The ids reaching `score_lesson` come from model output (corroboration), + // so this is a hole a prompt injection walks through if the backend + // updates by id alone. + let private = a + .promote(&lesson("a private class of situation"), &[]) + .await + .expect("promote"); + b.score_lesson(&private, true) + .await + .expect("no-op, not error"); + let untouched = a + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == private) + .expect("still there"); + assert_eq!( + (untouched.applied, untouched.helped), + (0, 0), + "tenant {:?} moved tenant {:?}'s score by naming its id", + b.scope(), + a.scope() + ); + + // A global lesson is visible to every tenant, so scoring it is legitimate. + let shared = global + .promote(&lesson("a class anyone can hit"), &[]) + .await + .expect("promote"); + b.score_lesson(&shared, true).await.expect("score"); + let moved = b + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == shared) + .expect("visible"); + assert_eq!((moved.applied, moved.helped), (1, 1)); +} + +async fn an_episode_id_alone_does_not_reach_another_tenants_attempts( + a: &dyn Ledger, + b: &dyn Ledger, +) { + // An episode id is opaque and a service may hand one straight through from + // a request path. Being keyed by episode is not isolation — guessing an id + // would be enough — so the rows carry the bucket too. + a.append(&row("ep-secret", 1, "authored:aaa")) + .await + .expect("append"); + assert_eq!(a.rows("ep-secret").await.expect("rows").len(), 1); + assert!( + b.rows("ep-secret").await.expect("rows").is_empty(), + "tenant {:?} read tenant {:?}'s attempts by knowing the episode id", + b.scope(), + a.scope() + ); +} + +async fn a_tenants_lesson_is_invisible_to_another(a: &dyn Ledger, b: &dyn Ledger) { + let mut mine = lesson("a private class of task"); + mine.claim = "names an internal repository path".into(); + let id = a.promote(&mine, &[]).await.expect("promote"); + + let seen_by_a = a.lessons(None).await.expect("lessons"); + assert!( + seen_by_a.iter().any(|l| l.id == id), + "a tenant must see its own lesson" + ); + + let seen_by_b = b.lessons(None).await.expect("lessons"); + assert!( + !seen_by_b.iter().any(|l| l.id == id), + "tenant {:?} can read tenant {:?}'s lesson — this is the leak the scope exists to stop", + b.scope(), + a.scope() + ); +} + +async fn a_global_lesson_is_visible_to_every_tenant( + global: &dyn Ledger, + a: &dyn Ledger, + b: &dyn Ledger, +) { + let id = global + .promote(&lesson("a class of task anyone can hit"), &[]) + .await + .expect("promote"); + for tenant in [a, b] { + let seen = tenant.lessons(None).await.expect("lessons"); + assert!( + seen.iter().any(|l| l.id == id), + "tenant {:?} cannot see a global lesson", + tenant.scope() + ); + } +} + +async fn promote_stamps_the_handle_not_the_argument(a: &dyn Ledger) { + // A caller — or a model whose answer was deserialized straight into a + // `Lesson` — must not be able to publish into another bucket by asking. + let mut forged = lesson("a class of task claiming to be someone else's"); + forged.scope_key = Some("some-other-tenant".to_string()); + let id = a.promote(&forged, &[]).await.expect("promote"); + + let stored = a + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == id) + .expect("stored"); + assert_eq!( + stored.scope_key.as_deref(), + a.scope(), + "promote must stamp the handle's scope, whatever the argument said" + ); +} + +async fn workflow_scores_do_not_bleed_between_tenants(a: &dyn Ledger, b: &dyn Ledger) { + let id = "wf-shared-id"; + a.score_workflow(id, true).await.expect("score"); + a.score_workflow(id, true).await.expect("score"); + b.score_workflow(id, false).await.expect("score"); + + let for_a = a.workflow_score(id).await.expect("score"); + let for_b = b.workflow_score(id).await.expect("score"); + assert_eq!( + (for_a.applied, for_a.helped), + (2, 2), + "tenant a's own record" + ); + assert_eq!( + (for_b.applied, for_b.helped), + (1, 0), + "tenant b's own record" + ); +} + +async fn a_tenant_writing_does_not_move_the_global_score(global: &dyn Ledger, a: &dyn Ledger) { + let id = "wf-tenant-only"; + a.score_workflow(id, true).await.expect("score"); + let seen = global.workflow_score(id).await.expect("score"); + assert_eq!( + (seen.applied, seen.helped), + (0, 0), + "the global bucket is its own bucket, not a union of every tenant's" + ); +} diff --git a/crates/tinyflows-adaptive/src/ledger/conformance/transcripts.rs b/crates/tinyflows-adaptive/src/ledger/conformance/transcripts.rs new file mode 100644 index 00000000..629cd292 --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/conformance/transcripts.rs @@ -0,0 +1,171 @@ +/// Run every transcript and paging case. +/// +/// # Panics +/// On any failure. +pub async fn run_transcripts(store: &dyn Ledger) { + an_attempt_with_no_transcript_is_empty_not_an_error(store).await; + a_transcript_round_trips_in_order(store).await; + a_looped_node_keeps_every_iteration(store).await; + saving_a_transcript_twice_replaces_rather_than_appends(store).await; + a_page_windows_the_episode_list(store).await; + an_agent_step_keeps_its_harness_transcript(store).await; +} + +/// An `agent` step's harness transcript survives the ledger. +/// +/// `Ran::steps` is the archival record — "every node activation, at full record +/// fidelity" — so a backend that persists the step but drops what the harness +/// did inside it satisfies the type and loses the point. +async fn an_agent_step_keeps_its_harness_transcript(store: &dyn Ledger) { + use tinyflows::transcript::TranscriptEntry; + + let entries = vec![ + TranscriptEntry::bounded(1, "agent_thinking", "memoise the chain"), + TranscriptEntry::bounded(2, "tool_call", "shell: python3 solve.py"), + TranscriptEntry::bounded(3, "tool_result", "837799"), + ]; + let mut solve = step("solve", 1); + solve.transcript = entries.clone(); + + store + .save_steps("ldg_transcript", &[solve, step("check", 2)]) + .await + .expect("save"); + + let back = store.steps("ldg_transcript").await.expect("steps"); + assert_eq!(back.len(), 2); + assert_eq!( + back[0].transcript, entries, + "the agent node's transcript round-trips whole and in order" + ); + assert!( + back[1].transcript.is_empty(), + "a step that recorded none still reads as none, not as the previous step's" + ); +} + +fn step(node_id: &str, n: u64) -> crate::execute::StepRecord { + crate::execute::StepRecord { + node_id: node_id.to_string(), + status: crate::execute::StepOutcome::Success, + output: serde_json::json!({ "i": n }), + duration_ms: n, + null_bindings: Vec::new(), + transcript: Vec::new(), + } +} + +async fn an_attempt_with_no_transcript_is_empty_not_an_error(store: &dyn Ledger) { + assert!(store.steps("ldg_nothing").await.expect("steps").is_empty()); +} + +async fn a_transcript_round_trips_in_order(store: &dyn Ledger) { + let mut errored = step("fetch", 7); + errored.status = crate::execute::StepOutcome::Error; + errored.null_bindings = vec![tinyflows::expr::NullResolution { + location: "args.to".to_string(), + expression: "=nodes.x.item.email".to_string(), + }]; + store + .save_steps("ldg_a", &[step("start", 1), errored]) + .await + .expect("save"); + + let back = store.steps("ldg_a").await.expect("steps"); + assert_eq!(back.len(), 2); + assert_eq!(back[0].node_id, "start", "execution order is the record"); + assert_eq!(back[1].status, crate::execute::StepOutcome::Error); + assert_eq!(back[1].duration_ms, 7); + assert_eq!( + back[1].null_bindings.len(), + 1, + "the nested list survives both a JSON column and a native array" + ); + assert_eq!(back[1].output, serde_json::json!({ "i": 7 })); +} + +async fn a_looped_node_keeps_every_iteration(store: &dyn Ledger) { + // The reason this is a record per step rather than one blob per attempt. + let steps: Vec<_> = (0..12).map(|n| step("body", n)).collect(); + store.save_steps("ldg_loop", &steps).await.expect("save"); + + let back = store.steps("ldg_loop").await.expect("steps"); + assert_eq!(back.len(), 12); + assert_eq!( + back.iter().map(|s| s.duration_ms).collect::>(), + (0..12).collect::>(), + "iterations in order, not deduplicated by node id" + ); +} + +async fn saving_a_transcript_twice_replaces_rather_than_appends(store: &dyn Ledger) { + // A retried write must not double the record. + store + .save_steps("ldg_twice", &[step("a", 1), step("b", 2)]) + .await + .expect("save"); + store + .save_steps("ldg_twice", &[step("a", 1), step("b", 2)]) + .await + .expect("save"); + assert_eq!(store.steps("ldg_twice").await.expect("steps").len(), 2); + + // And a SHORTER re-save must not leave the old tail behind — an upsert + // keyed by sequence replaces only the sequences present, and the stitched + // result would read as one transcript mixing two attempts. + store + .save_steps("ldg_twice", &[step("a", 9)]) + .await + .expect("save"); + let back = store.steps("ldg_twice").await.expect("steps"); + assert_eq!(back.len(), 1, "{back:?}"); + assert_eq!( + back[0].duration_ms, 9, + "and it is the new save, not the old" + ); +} + +async fn a_page_windows_the_episode_list(store: &dyn Ledger) { + for n in 0..5 { + store + .save_episode(&episode( + &format!("ep-page-{n}"), + EpisodeStatus::Running, + 1, + 0, + )) + .await + .expect("save"); + } + let all = store + .episodes(false, super::Page::ALL) + .await + .expect("episodes"); + assert!(all.len() >= 5); + + let first_two = store + .episodes(false, super::Page::first(2)) + .await + .expect("episodes"); + assert_eq!(first_two.len(), 2); + assert_eq!( + first_two.iter().map(|e| &e.id).collect::>(), + all[..2].iter().map(|e| &e.id).collect::>(), + "the same order, windowed" + ); + + let past_the_end = store + .episodes( + false, + super::Page { + limit: 10, + offset: all.len() + 5, + }, + ) + .await + .expect("episodes"); + assert!( + past_the_end.is_empty(), + "an offset past the end is empty, not a panic" + ); +} diff --git a/crates/tinyflows-adaptive/src/ledger/memory.rs b/crates/tinyflows-adaptive/src/ledger/memory.rs index 3e992451..58a7add3 100644 --- a/crates/tinyflows-adaptive/src/ledger/memory.rs +++ b/crates/tinyflows-adaptive/src/ledger/memory.rs @@ -171,11 +171,13 @@ impl Ledger for MemoryLedger { .filter(|(lesson, _)| lesson == lesson_id) .map(|(_, row)| row.as_str()) .collect(); - let bucket = self.bucket(); Ok(inner .rows .iter() - .filter(|(scope, r)| scope == &bucket && cited.contains(&r.id.as_str())) + .filter(|(scope, r)| { + self.visible((!scope.is_empty()).then_some(scope.as_str())) + && cited.contains(&r.id.as_str()) + }) .map(|(_, r)| r.clone()) .collect()) } @@ -254,7 +256,11 @@ impl Ledger for MemoryLedger { ..episode.clone() }; let mut inner = self.guard(); - match inner.episodes.iter_mut().find(|e| e.id == episode.id) { + match inner + .episodes + .iter_mut() + .find(|e| e.id == episode.id && e.scope_key.as_deref() == self.scope.as_deref()) + { Some(existing) => { // `started_at` and the scope are facts about creation, not // progress, so an update leaves them alone — matching mongo's diff --git a/crates/tinyflows-adaptive/src/ledger/mod.rs b/crates/tinyflows-adaptive/src/ledger/mod.rs index 8ad38e36..5a4459e3 100644 --- a/crates/tinyflows-adaptive/src/ledger/mod.rs +++ b/crates/tinyflows-adaptive/src/ledger/mod.rs @@ -493,63 +493,5 @@ pub trait Ledger: Send + Sync { } #[cfg(test)] -mod signature_tests { - use super::{LedgerRow, signatures}; - - fn row(attempt: u32, sig: &str) -> LedgerRow { - LedgerRow { - id: format!("r{attempt}"), - episode: "ep".into(), - attempt, - approach_sig: sig.into(), - approach_desc: String::new(), - workflow_id: None, - outcome: String::new(), - cause: String::new(), - cost_usd: 0.0, - at: "2026-01-01T00:00:00Z".into(), - satisfied: false, - advanced: false, - } - } - - #[test] - fn an_approach_tried_twice_appears_once() { - let got = signatures(&[ - row(1, "selected:weekly"), - row(2, "authored:aaa"), - row(3, "selected:weekly"), - ]); - assert_eq!(got, vec!["selected:weekly", "authored:aaa"]); - } - - #[test] - fn first_seen_order_is_kept() { - // It is rendered into a prompt, and a list that reshuffles between - // attempts is one a planner cannot be reasoned about against. - let got = signatures(&[row(1, "c"), row(2, "a"), row(3, "b")]); - assert_eq!(got, vec!["c", "a", "b"]); - } - - #[test] - fn no_rows_is_an_empty_list_rather_than_a_surprise() { - assert!(signatures(&[]).is_empty()); - } - - #[tokio::test] - async fn the_trait_method_agrees_with_the_function_it_now_calls() { - // `tried` is this over a fresh read. If the two ever disagree, one - // caller's exclusion list is not the other's. - use super::Ledger; - let ledger = super::memory::MemoryLedger::new(); - for (attempt, sig) in [ - (1u32, "selected:weekly"), - (2, "authored:aaa"), - (3, "selected:weekly"), - ] { - ledger.append(&row(attempt, sig)).await.expect("append"); - } - let rows = ledger.rows("ep").await.expect("rows"); - assert_eq!(ledger.tried("ep").await.expect("tried"), signatures(&rows)); - } -} +#[path = "signature_tests.rs"] +mod signature_tests; diff --git a/crates/tinyflows-adaptive/src/ledger/mongo.rs b/crates/tinyflows-adaptive/src/ledger/mongo.rs index df02b589..c9e4cc38 100644 --- a/crates/tinyflows-adaptive/src/ledger/mongo.rs +++ b/crates/tinyflows-adaptive/src/ledger/mongo.rs @@ -7,7 +7,7 @@ //! here loses increments under exactly the load a hosted deployment has. use async_trait::async_trait; -use mongodb::bson::{Document, doc}; +use mongodb::bson::{Bson, Document, doc}; use mongodb::options::{IndexOptions, ReturnDocument}; use mongodb::{Client, Collection, Database, IndexModel}; @@ -92,6 +92,15 @@ impl MongoLedger { self.scope.as_deref().unwrap_or_default() } + /// Match the current episode bucket, including pre-tenancy global rows + /// whose `scope_key` field is absent. + fn episode_scope_filter(&self) -> Document { + match &self.scope { + Some(scope) => doc! { "scope_key": scope }, + None => doc! { "scope_key": { "$in": ["", Bson::Null] } }, + } + } + async fn ensure_indexes(&self) -> Result<()> { // Ordered by `seq`, never by timestamp: two attempts finishing in the // same second would otherwise read back in an arbitrary order, which @@ -204,7 +213,10 @@ fn read_row(doc: &Document) -> LedgerRow { fn read_episode(doc: &Document) -> Result { let scope = text(doc, "scope_key"); Ok(Episode { - id: text(doc, "_id"), + id: doc + .get_str("id") + .map(str::to_string) + .unwrap_or_else(|_| text(doc, "_id")), goal: serde_json::from_str(&text(doc, "goal")) .map_err(|e| LedgerError::Corrupt(e.to_string()))?, scope_key: (!scope.is_empty()).then_some(scope), @@ -217,355 +229,7 @@ fn read_episode(doc: &Document) -> Result { }) } -#[async_trait] -impl Ledger for MongoLedger { - fn scope(&self) -> Option<&str> { - self.scope.as_deref() - } - - async fn append(&self, row: &LedgerRow) -> Result { - let seq = self.next_seq(ROWS).await?; - let id = format!("ldg_{seq:08}"); - self.rows() - .insert_one(doc! { - "_id": &id, - "episode": &row.episode, - "attempt": i64::from(row.attempt), - "approach_sig": &row.approach_sig, - "approach_desc": &row.approach_desc, - "workflow_id": row.workflow_id.clone(), - "outcome": &row.outcome, - "cause": &row.cause, - "cost_usd": row.cost_usd, - "at": &row.at, - "satisfied": row.satisfied, - "advanced": row.advanced, - "scope_key": self.bucket(), - "seq": seq, - }) - .await?; - Ok(id) - } - - async fn rows(&self, episode: &str) -> Result> { - let mut cursor = self - .rows() - .find(doc! { "episode": episode, "scope_key": self.bucket() }) - .sort(doc! { "seq": 1 }) - .await?; - let mut out = Vec::new(); - while cursor.advance().await? { - out.push(read_row(&cursor.deserialize_current()?)); - } - Ok(out) - } - - async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { - let seq = self.next_seq(LESSONS).await?; - let id = format!("les_{seq:08}"); - self.lessons_c() - .insert_one(doc! { - "_id": &id, - "kind": kind_str(lesson.kind), - "trigger": &lesson.trigger, - "mechanism": &lesson.mechanism, - "claim": &lesson.claim, - "applied": i64::from(lesson.applied), - "helped": i64::from(lesson.helped), - // The handle's, never the argument's. - "scope_key": self.bucket(), - "seq": seq, - }) - .await?; - for row_id in cites { - // Upsert on the pair so re-promoting the same citation is a no-op - // rather than a duplicate edge. - self.evidence() - .update_one( - doc! { "lesson_id": &id, "row_id": row_id }, - doc! { "$setOnInsert": { "lesson_id": &id, "row_id": row_id } }, - ) - .upsert(true) - .await?; - } - Ok(id) - } - - async fn lessons(&self, kind: Option) -> Result> { - // This bucket plus global. An unscoped handle's bucket is global, so - // the two halves coincide and it sees exactly what it wrote. `null` is - // in the set because `$in` only matches a *missing* field when the - // array contains null — and a lesson written before scoping existed - // has no field at all; those read as global, which is what they were. - let mine = doc! { "$in": [self.bucket(), "", mongodb::bson::Bson::Null] }; - let filter = match kind { - Some(want) => doc! { "kind": kind_str(want), "scope_key": mine }, - None => doc! { "scope_key": mine }, - }; - let mut cursor = self - .lessons_c() - .find(filter) - .sort(doc! { "seq": 1 }) - .await?; - let mut out = Vec::new(); - while cursor.advance().await? { - let d = cursor.deserialize_current()?; - out.push(Lesson { - id: text(&d, "_id"), - kind: LessonKind::parse(&text(&d, "kind")), - trigger: text(&d, "trigger"), - mechanism: text(&d, "mechanism"), - claim: text(&d, "claim"), - applied: as_u32(&d, "applied"), - helped: as_u32(&d, "helped"), - scope_key: Some(text(&d, "scope_key")).filter(|s| !s.is_empty()), - }); - } - Ok(out) - } - - async fn evidence(&self, lesson_id: &str) -> Result> { - let mut cursor = self - .evidence() - .find(doc! { "lesson_id": lesson_id }) - .await?; - let mut ids = Vec::new(); - while cursor.advance().await? { - ids.push(text(&cursor.deserialize_current()?, "row_id")); - } - if ids.is_empty() { - return Ok(Vec::new()); - } - let mut found = self - .rows() - .find(doc! { "_id": { "$in": ids }, "scope_key": self.bucket() }) - .sort(doc! { "seq": 1 }) - .await?; - let mut out = Vec::new(); - while found.advance().await? { - out.push(read_row(&found.deserialize_current()?)); - } - Ok(out) - } - - async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { - // Constrained to what this handle can see — the id arrives from model - // output, and naming another tenant's lesson must not move its score. - self.lessons_c() - .update_one( - doc! { "_id": lesson_id, - "scope_key": { "$in": [self.bucket(), "", mongodb::bson::Bson::Null] } }, - doc! { "$inc": { "applied": 1_i64, "helped": i64::from(helped) } }, - ) - .await?; - Ok(()) - } - - async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { - // `$inc` on an upsert, not read-modify-write: several loops may finish - // the same workflow at once, and a lost increment is a promotion gate - // reading the wrong evidence. - self.scores() - .update_one( - doc! { "workflow_id": workflow_id, "scope_key": self.bucket() }, - doc! { "$inc": { "applied": 1_i64, "helped": i64::from(helped) } }, - ) - .upsert(true) - .await?; - Ok(()) - } - - async fn workflow_score(&self, workflow_id: &str) -> Result { - let found = self - .scores() - .find_one(doc! { "workflow_id": workflow_id, "scope_key": self.bucket() }) - .await?; - Ok(found.map_or_else(Score::default, |d| Score { - applied: as_u32(&d, "applied"), - helped: as_u32(&d, "helped"), - })) - } - - async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { - self.variants() - .update_one( - doc! { "scope_key": self.bucket(), "variant": variant }, - doc! { "$setOnInsert": { - "scope_key": self.bucket(), "variant": variant, "parent": parent - } }, - ) - .upsert(true) - .await?; - Ok(()) - } - - async fn parent_of(&self, id: &str) -> Result> { - let found = self - .variants() - .find_one(doc! { "scope_key": self.bucket(), "variant": id }) - .await?; - Ok(found.map(|d| text(&d, "parent")).filter(|p| !p.is_empty())) - } - - async fn save_episode(&self, episode: &Episode) -> Result<()> { - let goal = serde_json::to_string(&episode.goal) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?; - let status = serde_json::to_string(&episode.status) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?; - self.episodes_c() - .update_one( - doc! { "_id": &episode.id }, - doc! { - "$set": { - "goal": goal, - "status": status, - "attempt": i64::from(episode.attempt), - "stalled": i64::from(episode.stalled), - "updated_at": &episode.updated_at, - }, - // Set once: the handle's scope and the first timestamp are - // facts about the episode's creation, not its progress. - "$setOnInsert": { - "scope_key": self.bucket(), - "started_at": &episode.started_at, - }, - }, - ) - .upsert(true) - .await?; - Ok(()) - } - - async fn episode(&self, id: &str) -> Result> { - let found = self - .episodes_c() - .find_one(doc! { "_id": id, "scope_key": self.bucket() }) - .await?; - found.as_ref().map(read_episode).transpose() - } - - async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()> { - // Replace, not overlay: a shorter re-save must not leave the old tail - // behind it, or `steps()` returns two attempts stitched together. - self.steps_c() - .delete_many(doc! { "scope_key": self.bucket(), "row_id": row_id }) - .await?; - // A document per step. One per attempt would exceed the 16 MB cap on a - // looped graph, and would do it only in production. - for (seq, step) in steps.iter().enumerate() { - let seq = i64::try_from(seq).unwrap_or(i64::MAX); - self.steps_c() - .update_one( - doc! { "scope_key": self.bucket(), "row_id": row_id, "seq": seq }, - doc! { "$set": { - "node_id": &step.node_id, - "status": serde_json::to_string(&step.status) - .map_err(|e| LedgerError::Corrupt(e.to_string()))? - .trim_matches('"'), - "output": serde_json::to_string(&step.output) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - "duration_ms": i64::try_from(step.duration_ms).unwrap_or(i64::MAX), - "null_bindings": serde_json::to_string(&step.null_bindings) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - "transcript": serde_json::to_string(&step.transcript) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - } }, - ) - .upsert(true) - .await?; - } - Ok(()) - } - - async fn steps(&self, row_id: &str) -> Result> { - let mut cursor = self - .steps_c() - .find(doc! { "scope_key": self.bucket(), "row_id": row_id }) - .sort(doc! { "seq": 1 }) - .await?; - let mut out = Vec::new(); - while cursor.advance().await? { - let d = cursor.deserialize_current()?; - out.push(crate::execute::StepRecord { - node_id: text(&d, "node_id"), - status: if text(&d, "status") == "error" { - crate::execute::StepOutcome::Error - } else { - crate::execute::StepOutcome::Success - }, - output: serde_json::from_str(&text(&d, "output")) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - duration_ms: u64::from(as_u32(&d, "duration_ms")), - null_bindings: serde_json::from_str(&text(&d, "null_bindings")) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - // A document written before this field existed has no - // `transcript` key; `text` yields "" for it, which is not valid - // JSON. Absent means "recorded none", so it reads as empty - // rather than corrupting the whole attempt's steps. - transcript: match text(&d, "transcript").as_str() { - "" => Vec::new(), - raw => serde_json::from_str(raw) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - }, - }); - } - Ok(out) - } - - async fn episodes(&self, running_only: bool, page: super::Page) -> Result> { - let mut cursor = self - .episodes_c() - .find(doc! { "scope_key": self.bucket() }) - .sort(doc! { "updated_at": -1, "_id": 1 }) - .await?; - let mut out = Vec::new(); - while cursor.advance().await? { - let episode = read_episode(&cursor.deserialize_current()?)?; - if !running_only || episode.status == EpisodeStatus::Running { - out.push(episode); - } - } - Ok(page.apply(out)) - } - - async fn children_of(&self, id: &str) -> Result> { - let mut cursor = self - .variants() - .find(doc! { "scope_key": self.bucket(), "parent": id }) - .sort(doc! { "variant": 1 }) - .await?; - let mut out = Vec::new(); - while cursor.advance().await? { - out.push(text(&cursor.deserialize_current()?, "variant")); - } - Ok(out) - } -} - +include!("mongo/ledger_impl.rs"); #[cfg(test)] -mod tests { - use super::*; - use crate::ledger::conformance; - - /// Runs the same suite the sqlite backend passes, against a real server. - /// - /// Ignored by default: it needs one. Point `ADAPTIVE_MONGO_URI` at a - /// throwaway database and run with `--ignored`. Skipping silently when the - /// variable is absent would let this rot unnoticed, so the case is - /// `#[ignore]` and visible in the run summary instead. - #[tokio::test] - #[ignore = "needs a MongoDB server; set ADAPTIVE_MONGO_URI"] - async fn passes_the_conformance_suite() { - let uri = std::env::var("ADAPTIVE_MONGO_URI").expect("ADAPTIVE_MONGO_URI"); - let name = format!("adaptive_conformance_{}", std::process::id()); - let store = MongoLedger::connect(&uri, &name).await.expect("connect"); - conformance::run_all(&store).await; - conformance::run_tenants( - &store, - &store.for_tenant("user-a"), - &store.for_tenant("user-b"), - ) - .await; - store.db.drop().await.expect("drop the throwaway database"); - } -} +#[path = "mongo_tests.rs"] +mod tests; diff --git a/crates/tinyflows-adaptive/src/ledger/mongo/ledger_impl.rs b/crates/tinyflows-adaptive/src/ledger/mongo/ledger_impl.rs new file mode 100644 index 00000000..ef670313 --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/mongo/ledger_impl.rs @@ -0,0 +1,336 @@ +#[async_trait] +impl Ledger for MongoLedger { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn append(&self, row: &LedgerRow) -> Result { + let seq = self.next_seq(ROWS).await?; + let id = format!("ldg_{seq:08}"); + self.rows() + .insert_one(doc! { + "_id": &id, + "episode": &row.episode, + "attempt": i64::from(row.attempt), + "approach_sig": &row.approach_sig, + "approach_desc": &row.approach_desc, + "workflow_id": row.workflow_id.clone(), + "outcome": &row.outcome, + "cause": &row.cause, + "cost_usd": row.cost_usd, + "at": &row.at, + "satisfied": row.satisfied, + "advanced": row.advanced, + "scope_key": self.bucket(), + "seq": seq, + }) + .await?; + Ok(id) + } + + async fn rows(&self, episode: &str) -> Result> { + let mut cursor = self + .rows() + .find(doc! { "episode": episode, "scope_key": self.bucket() }) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + out.push(read_row(&cursor.deserialize_current()?)); + } + Ok(out) + } + + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { + let seq = self.next_seq(LESSONS).await?; + let id = format!("les_{seq:08}"); + self.lessons_c() + .insert_one(doc! { + "_id": &id, + "kind": kind_str(lesson.kind), + "trigger": &lesson.trigger, + "mechanism": &lesson.mechanism, + "claim": &lesson.claim, + "applied": i64::from(lesson.applied), + "helped": i64::from(lesson.helped), + // The handle's, never the argument's. + "scope_key": self.bucket(), + "seq": seq, + }) + .await?; + for row_id in cites { + // Upsert on the pair so re-promoting the same citation is a no-op + // rather than a duplicate edge. + self.evidence() + .update_one( + doc! { "lesson_id": &id, "row_id": row_id }, + doc! { "$setOnInsert": { "lesson_id": &id, "row_id": row_id } }, + ) + .upsert(true) + .await?; + } + Ok(id) + } + + async fn lessons(&self, kind: Option) -> Result> { + // This bucket plus global. An unscoped handle's bucket is global, so + // the two halves coincide and it sees exactly what it wrote. `null` is + // in the set because `$in` only matches a *missing* field when the + // array contains null — and a lesson written before scoping existed + // has no field at all; those read as global, which is what they were. + let mine = doc! { "$in": [self.bucket(), "", mongodb::bson::Bson::Null] }; + let filter = match kind { + Some(want) => doc! { "kind": kind_str(want), "scope_key": mine }, + None => doc! { "scope_key": mine }, + }; + let mut cursor = self + .lessons_c() + .find(filter) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + let d = cursor.deserialize_current()?; + out.push(Lesson { + id: text(&d, "_id"), + kind: LessonKind::parse(&text(&d, "kind")), + trigger: text(&d, "trigger"), + mechanism: text(&d, "mechanism"), + claim: text(&d, "claim"), + applied: as_u32(&d, "applied"), + helped: as_u32(&d, "helped"), + scope_key: Some(text(&d, "scope_key")).filter(|s| !s.is_empty()), + }); + } + Ok(out) + } + + async fn evidence(&self, lesson_id: &str) -> Result> { + let mut cursor = self + .evidence() + .find(doc! { "lesson_id": lesson_id }) + .await?; + let mut ids = Vec::new(); + while cursor.advance().await? { + ids.push(text(&cursor.deserialize_current()?, "row_id")); + } + if ids.is_empty() { + return Ok(Vec::new()); + } + let mut found = self + .rows() + .find(doc! { + "_id": { "$in": ids }, + "scope_key": { "$in": [self.bucket(), "", mongodb::bson::Bson::Null] }, + }) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while found.advance().await? { + out.push(read_row(&found.deserialize_current()?)); + } + Ok(out) + } + + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { + // Constrained to what this handle can see — the id arrives from model + // output, and naming another tenant's lesson must not move its score. + self.lessons_c() + .update_one( + doc! { "_id": lesson_id, + "scope_key": { "$in": [self.bucket(), "", mongodb::bson::Bson::Null] } }, + doc! { "$inc": { "applied": 1_i64, "helped": i64::from(helped) } }, + ) + .await?; + Ok(()) + } + + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { + // `$inc` on an upsert, not read-modify-write: several loops may finish + // the same workflow at once, and a lost increment is a promotion gate + // reading the wrong evidence. + self.scores() + .update_one( + doc! { "workflow_id": workflow_id, "scope_key": self.bucket() }, + doc! { "$inc": { "applied": 1_i64, "helped": i64::from(helped) } }, + ) + .upsert(true) + .await?; + Ok(()) + } + + async fn workflow_score(&self, workflow_id: &str) -> Result { + let found = self + .scores() + .find_one(doc! { "workflow_id": workflow_id, "scope_key": self.bucket() }) + .await?; + Ok(found.map_or_else(Score::default, |d| Score { + applied: as_u32(&d, "applied"), + helped: as_u32(&d, "helped"), + })) + } + + async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { + self.variants() + .update_one( + doc! { "scope_key": self.bucket(), "variant": variant }, + doc! { "$setOnInsert": { + "scope_key": self.bucket(), "variant": variant, "parent": parent + } }, + ) + .upsert(true) + .await?; + Ok(()) + } + + async fn parent_of(&self, id: &str) -> Result> { + let found = self + .variants() + .find_one(doc! { "scope_key": self.bucket(), "variant": id }) + .await?; + Ok(found.map(|d| text(&d, "parent")).filter(|p| !p.is_empty())) + } + + async fn save_episode(&self, episode: &Episode) -> Result<()> { + let goal = serde_json::to_string(&episode.goal) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?; + let status = serde_json::to_string(&episode.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?; + let mut filter = self.episode_scope_filter(); + filter.insert( + "$or", + vec![doc! { "id": &episode.id }, doc! { "_id": &episode.id }], + ); + self.episodes_c() + .update_one( + filter, + doc! { + "$set": { + "id": &episode.id, + "goal": goal, + "status": status, + "attempt": i64::from(episode.attempt), + "stalled": i64::from(episode.stalled), + "updated_at": &episode.updated_at, + }, + // Set once: the handle's scope and the first timestamp are + // facts about the episode's creation, not its progress. + "$setOnInsert": { + "_id": { "scope_key": self.bucket(), "id": &episode.id }, + "scope_key": self.bucket(), + "started_at": &episode.started_at, + }, + }, + ) + .upsert(true) + .await?; + Ok(()) + } + + async fn episode(&self, id: &str) -> Result> { + let mut filter = self.episode_scope_filter(); + filter.insert("$or", vec![doc! { "id": id }, doc! { "_id": id }]); + let found = self + .episodes_c() + .find_one(filter) + .await?; + found.as_ref().map(read_episode).transpose() + } + + async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()> { + // Replace, not overlay: a shorter re-save must not leave the old tail + // behind it, or `steps()` returns two attempts stitched together. + self.steps_c() + .delete_many(doc! { "scope_key": self.bucket(), "row_id": row_id }) + .await?; + // A document per step. One per attempt would exceed the 16 MB cap on a + // looped graph, and would do it only in production. + for (seq, step) in steps.iter().enumerate() { + let seq = i64::try_from(seq).unwrap_or(i64::MAX); + self.steps_c() + .update_one( + doc! { "scope_key": self.bucket(), "row_id": row_id, "seq": seq }, + doc! { "$set": { + "node_id": &step.node_id, + "status": serde_json::to_string(&step.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))? + .trim_matches('"'), + "output": serde_json::to_string(&step.output) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + "duration_ms": i64::try_from(step.duration_ms).unwrap_or(i64::MAX), + "null_bindings": serde_json::to_string(&step.null_bindings) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + "transcript": serde_json::to_string(&step.transcript) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + } }, + ) + .upsert(true) + .await?; + } + Ok(()) + } + + async fn steps(&self, row_id: &str) -> Result> { + let mut cursor = self + .steps_c() + .find(doc! { "scope_key": self.bucket(), "row_id": row_id }) + .sort(doc! { "seq": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + let d = cursor.deserialize_current()?; + out.push(crate::execute::StepRecord { + node_id: text(&d, "node_id"), + status: if text(&d, "status") == "error" { + crate::execute::StepOutcome::Error + } else { + crate::execute::StepOutcome::Success + }, + output: serde_json::from_str(&text(&d, "output")) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + duration_ms: u64::from(as_u32(&d, "duration_ms")), + null_bindings: serde_json::from_str(&text(&d, "null_bindings")) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + // A document written before this field existed has no + // `transcript` key; `text` yields "" for it, which is not valid + // JSON. Absent means "recorded none", so it reads as empty + // rather than corrupting the whole attempt's steps. + transcript: match text(&d, "transcript").as_str() { + "" => Vec::new(), + raw => serde_json::from_str(raw) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + }, + }); + } + Ok(out) + } + + async fn episodes(&self, running_only: bool, page: super::Page) -> Result> { + let mut cursor = self + .episodes_c() + .find(self.episode_scope_filter()) + .sort(doc! { "updated_at": -1, "_id": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + let episode = read_episode(&cursor.deserialize_current()?)?; + if !running_only || episode.status == EpisodeStatus::Running { + out.push(episode); + } + } + Ok(page.apply(out)) + } + + async fn children_of(&self, id: &str) -> Result> { + let mut cursor = self + .variants() + .find(doc! { "scope_key": self.bucket(), "parent": id }) + .sort(doc! { "variant": 1 }) + .await?; + let mut out = Vec::new(); + while cursor.advance().await? { + out.push(text(&cursor.deserialize_current()?, "variant")); + } + Ok(out) + } +} diff --git a/crates/tinyflows-adaptive/src/ledger/mongo_tests.rs b/crates/tinyflows-adaptive/src/ledger/mongo_tests.rs new file mode 100644 index 00000000..69561f72 --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/mongo_tests.rs @@ -0,0 +1,86 @@ +use super::*; +use crate::ledger::conformance; + +/// Runs the same suite the sqlite backend passes, against a real server. +/// +/// Ignored by default: it needs one. Point `ADAPTIVE_MONGO_URI` at a +/// throwaway database and run with `--ignored`. Skipping silently when the +/// variable is absent would let this rot unnoticed, so the case is +/// `#[ignore]` and visible in the run summary instead. +#[tokio::test] +#[ignore = "needs a MongoDB server; set ADAPTIVE_MONGO_URI"] +async fn passes_the_conformance_suite() { + let uri = std::env::var("ADAPTIVE_MONGO_URI").expect("ADAPTIVE_MONGO_URI"); + let name = format!("adaptive_conformance_{}", std::process::id()); + let store = MongoLedger::connect(&uri, &name).await.expect("connect"); + conformance::run_all(&store).await; + conformance::run_tenants( + &store, + &store.for_tenant("user-a"), + &store.for_tenant("user-b"), + ) + .await; + store.db.drop().await.expect("drop the throwaway database"); +} + +#[tokio::test] +#[ignore = "needs a MongoDB server; set ADAPTIVE_MONGO_URI"] +async fn a_legacy_global_episode_without_a_scope_key_remains_readable_and_updatable() { + let uri = std::env::var("ADAPTIVE_MONGO_URI").expect("ADAPTIVE_MONGO_URI"); + let name = format!("adaptive_legacy_episode_{}", std::process::id()); + let store = MongoLedger::connect(&uri, &name).await.expect("connect"); + let goal = crate::contracts::Goal::new("preserve the legacy episode"); + store + .episodes_c() + .insert_one(doc! { + "_id": "legacy-global", + "goal": serde_json::to_string(&goal).expect("serialize goal"), + "status": serde_json::to_string(&EpisodeStatus::Running).expect("serialize status"), + "attempt": 1_i64, + "stalled": 0_i64, + "started_at": "2026-01-01T00:00:00Z", + "updated_at": "2026-01-01T00:00:05Z", + }) + .await + .expect("insert legacy episode"); + + let mut episode = store + .episode("legacy-global") + .await + .expect("read legacy episode") + .expect("legacy episode exists"); + assert_eq!(episode.scope_key, None); + assert_eq!( + store + .episodes(false, crate::ledger::Page::first(10)) + .await + .expect("list episodes") + .len(), + 1 + ); + + episode.attempt = 2; + store.save_episode(&episode).await.expect("update episode"); + assert_eq!( + store + .episode("legacy-global") + .await + .expect("read updated episode") + .expect("updated episode exists") + .attempt, + 2 + ); + assert_eq!( + store + .episodes_c() + .count_documents(doc! { "$or": [ + { "id": "legacy-global" }, + { "_id": "legacy-global" }, + ] }) + .await + .expect("count matching episodes"), + 1 + ); + + store.db.drop().await.expect("drop the throwaway database"); +} diff --git a/crates/tinyflows-adaptive/src/ledger/signature_tests.rs b/crates/tinyflows-adaptive/src/ledger/signature_tests.rs new file mode 100644 index 00000000..6fd460a3 --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/signature_tests.rs @@ -0,0 +1,58 @@ +use super::{LedgerRow, signatures}; + +fn row(attempt: u32, sig: &str) -> LedgerRow { + LedgerRow { + id: format!("r{attempt}"), + episode: "ep".into(), + attempt, + approach_sig: sig.into(), + approach_desc: String::new(), + workflow_id: None, + outcome: String::new(), + cause: String::new(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + } +} + +#[test] +fn an_approach_tried_twice_appears_once() { + let got = signatures(&[ + row(1, "selected:weekly"), + row(2, "authored:aaa"), + row(3, "selected:weekly"), + ]); + assert_eq!(got, vec!["selected:weekly", "authored:aaa"]); +} + +#[test] +fn first_seen_order_is_kept() { + // It is rendered into a prompt, and a list that reshuffles between + // attempts is one a planner cannot be reasoned about against. + let got = signatures(&[row(1, "c"), row(2, "a"), row(3, "b")]); + assert_eq!(got, vec!["c", "a", "b"]); +} + +#[test] +fn no_rows_is_an_empty_list_rather_than_a_surprise() { + assert!(signatures(&[]).is_empty()); +} + +#[tokio::test] +async fn the_trait_method_agrees_with_the_function_it_now_calls() { + // `tried` is this over a fresh read. If the two ever disagree, one + // caller's exclusion list is not the other's. + use super::Ledger; + let ledger = super::memory::MemoryLedger::new(); + for (attempt, sig) in [ + (1u32, "selected:weekly"), + (2, "authored:aaa"), + (3, "selected:weekly"), + ] { + ledger.append(&row(attempt, sig)).await.expect("append"); + } + let rows = ledger.rows("ep").await.expect("rows"); + assert_eq!(ledger.tried("ep").await.expect("tried"), signatures(&rows)); +} diff --git a/crates/tinyflows-adaptive/src/ledger/sqlite.rs b/crates/tinyflows-adaptive/src/ledger/sqlite.rs index b9e7669a..21b3b152 100644 --- a/crates/tinyflows-adaptive/src/ledger/sqlite.rs +++ b/crates/tinyflows-adaptive/src/ledger/sqlite.rs @@ -85,14 +85,15 @@ const DDL: &[&str] = &[ )", "CREATE INDEX IF NOT EXISTS ix_variants_parent ON variants(scope_key, parent)", "CREATE TABLE IF NOT EXISTS episodes ( - id TEXT PRIMARY KEY, + id TEXT NOT NULL, scope_key TEXT NOT NULL DEFAULT '', goal TEXT NOT NULL, status TEXT NOT NULL, attempt INTEGER NOT NULL DEFAULT 0, stalled INTEGER NOT NULL DEFAULT 0, started_at TEXT NOT NULL, - updated_at TEXT NOT NULL + updated_at TEXT NOT NULL, + PRIMARY KEY (scope_key, id) )", "CREATE INDEX IF NOT EXISTS ix_episodes_scope ON episodes(scope_key, updated_at)", // One row per step, never one blob per attempt: a looped node produces a @@ -322,6 +323,7 @@ impl SqliteLedger { for statement in MIGRATIONS { let _ = conn.execute(statement, []); } + migrate_episode_identity(&conn)?; Ok(Self { conn: std::sync::Arc::new(Mutex::new(conn)), scope: None, @@ -358,6 +360,40 @@ impl SqliteLedger { } } +/// Rebuild the pre-tenancy episodes table whose primary key was only `id`. +fn migrate_episode_identity(conn: &Connection) -> Result<()> { + let scoped_primary_key: i64 = conn.query_row( + "SELECT pk FROM pragma_table_info('episodes') WHERE name = 'scope_key'", + [], + |row| row.get(0), + )?; + if scoped_primary_key != 0 { + return Ok(()); + } + conn.execute_batch( + "BEGIN IMMEDIATE; + CREATE TABLE episodes_scoped ( + id TEXT NOT NULL, + scope_key TEXT NOT NULL DEFAULT '', + goal TEXT NOT NULL, + status TEXT NOT NULL, + attempt INTEGER NOT NULL DEFAULT 0, + stalled INTEGER NOT NULL DEFAULT 0, + started_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (scope_key, id) + ); + INSERT INTO episodes_scoped + SELECT id, scope_key, goal, status, attempt, stalled, started_at, updated_at + FROM episodes; + DROP TABLE episodes; + ALTER TABLE episodes_scoped RENAME TO episodes; + CREATE INDEX ix_episodes_scope ON episodes(scope_key, updated_at); + COMMIT;", + )?; + Ok(()) +} + fn next_seq(conn: &Connection, table: &str) -> Result { let current: Option = conn .query_row(&format!("SELECT MAX(seq) FROM {table}"), [], |r| r.get(0)) @@ -419,529 +455,7 @@ fn read_episode(r: &rusqlite::Row<'_>) -> rusqlite::Result> { })()) } -#[async_trait] -impl Ledger for SqliteLedger { - fn scope(&self) -> Option<&str> { - self.scope.as_deref() - } - - async fn append(&self, row: &LedgerRow) -> Result { - let conn = self.guard()?; - let seq = next_seq(&conn, "ledger_rows")?; - let id = new_id("ldg", seq); - conn.execute( - "INSERT INTO ledger_rows(id, episode, attempt, approach_sig, approach_desc, - workflow_id, outcome, cause, cost_usd, at, - satisfied, advanced, scope_key, seq) - VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14)", - params![ - id, - row.episode, - i64::from(row.attempt), - row.approach_sig, - row.approach_desc, - row.workflow_id, - row.outcome, - row.cause, - row.cost_usd, - row.at, - i64::from(row.satisfied), - i64::from(row.advanced), - self.bucket(), - seq, - ], - )?; - Ok(id) - } - - async fn rows(&self, episode: &str) -> Result> { - let conn = self.guard()?; - // Scoped as well as keyed by episode. An episode id is opaque and a - // service may hand one straight through from a request path, so this - // must not be the one read where guessing an id is enough. - let mut stmt = conn.prepare( - "SELECT * FROM ledger_rows WHERE episode = ?1 AND scope_key = ?2 ORDER BY seq", - )?; - let found = stmt - .query_map(params![episode, self.bucket()], read_row)? - .collect::>>()?; - Ok(found) - } - - async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { - let conn = self.guard()?; - let seq = next_seq(&conn, "lessons")?; - let id = new_id("les", seq); - conn.execute( - "INSERT INTO lessons(id, kind, trigger, mechanism, claim, applied, helped, - scope_key, seq) - VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9)", - params![ - id, - serde_json::to_string(&lesson.kind) - .map_err(|e| LedgerError::Corrupt(e.to_string()))? - .trim_matches('"'), - lesson.trigger, - lesson.mechanism, - lesson.claim, - i64::from(lesson.applied), - i64::from(lesson.helped), - // The handle's, never the argument's. - self.bucket(), - seq, - ], - )?; - for row_id in cites { - conn.execute( - "INSERT OR IGNORE INTO lesson_evidence(lesson_id, row_id) VALUES(?1,?2)", - params![id, row_id], - )?; - } - Ok(id) - } - - async fn lessons(&self, kind: Option) -> Result> { - let conn = self.guard()?; - // This bucket plus global. An unscoped handle's bucket is global, so - // the two halves coincide and it sees exactly what it wrote. - let mut stmt = conn - .prepare("SELECT * FROM lessons WHERE scope_key = ?1 OR scope_key = '' ORDER BY seq")?; - let all = stmt - .query_map([self.bucket()], |r| { - let scope: String = r.get("scope_key")?; - Ok(Lesson { - id: r.get("id")?, - kind: LessonKind::parse(&r.get::<_, String>("kind")?), - trigger: r.get("trigger")?, - mechanism: r.get("mechanism")?, - claim: r.get("claim")?, - applied: r.get::<_, i64>("applied")?.try_into().unwrap_or(0), - helped: r.get::<_, i64>("helped")?.try_into().unwrap_or(0), - scope_key: (!scope.is_empty()).then_some(scope), - }) - })? - .collect::>>()?; - Ok(match kind { - Some(want) => all.into_iter().filter(|l| l.kind == want).collect(), - None => all, - }) - } - - async fn evidence(&self, lesson_id: &str) -> Result> { - let conn = self.guard()?; - let mut stmt = conn.prepare( - "SELECT r.* FROM ledger_rows r - JOIN lesson_evidence e ON e.row_id = r.id - WHERE e.lesson_id = ?1 AND r.scope_key = ?2 ORDER BY r.seq", - )?; - let found = stmt - .query_map(params![lesson_id, self.bucket()], read_row)? - .collect::>>()?; - Ok(found) - } - - async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { - let conn = self.guard()?; - conn.execute( - // Constrained to what this handle can see — the id arrives from - // model output, and naming another tenant's lesson must not move - // its score. - "UPDATE lessons SET applied = applied + 1, helped = helped + ?2 - WHERE id = ?1 AND (scope_key = ?3 OR scope_key = '')", - params![lesson_id, i64::from(helped), self.bucket()], - )?; - Ok(()) - } - - async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { - let conn = self.guard()?; - // Upsert: the first run of a workflow is the common case and must not - // need a separate registration step. - conn.execute( - "INSERT INTO workflow_scores(scope_key, workflow_id, applied, helped) - VALUES(?1, ?2, 1, ?3) - ON CONFLICT(scope_key, workflow_id) DO UPDATE SET - applied = applied + 1, - helped = helped + ?3", - params![self.bucket(), workflow_id, i64::from(helped)], - )?; - Ok(()) - } - - async fn workflow_score(&self, workflow_id: &str) -> Result { - let conn = self.guard()?; - let found = conn - .query_row( - "SELECT applied, helped FROM workflow_scores - WHERE scope_key = ?1 AND workflow_id = ?2", - params![self.bucket(), workflow_id], - |r| { - Ok(Score { - applied: r.get::<_, i64>(0)?.try_into().unwrap_or(0), - helped: r.get::<_, i64>(1)?.try_into().unwrap_or(0), - }) - }, - ) - .optional()?; - Ok(found.unwrap_or_default()) - } - - async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { - let conn = self.guard()?; - conn.execute( - "INSERT OR IGNORE INTO variants(scope_key, variant, parent) VALUES(?1,?2,?3)", - params![self.bucket(), variant, parent], - )?; - Ok(()) - } - - async fn parent_of(&self, id: &str) -> Result> { - let conn = self.guard()?; - let found = conn - .query_row( - "SELECT parent FROM variants WHERE scope_key = ?1 AND variant = ?2", - params![self.bucket(), id], - |r| r.get(0), - ) - .optional()?; - Ok(found) - } - - async fn save_episode(&self, episode: &Episode) -> Result<()> { - let conn = self.guard()?; - conn.execute( - "INSERT INTO episodes(id, scope_key, goal, status, attempt, stalled, - started_at, updated_at) - VALUES(?1,?2,?3,?4,?5,?6,?7,?8) - ON CONFLICT(id) DO UPDATE SET - goal = ?3, status = ?4, attempt = ?5, stalled = ?6, updated_at = ?8", - params![ - episode.id, - self.bucket(), - serde_json::to_string(&episode.goal) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - serde_json::to_string(&episode.status) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - i64::from(episode.attempt), - i64::from(episode.stalled), - episode.started_at, - episode.updated_at, - ], - )?; - Ok(()) - } - - async fn episode(&self, id: &str) -> Result> { - let conn = self.guard()?; - let found = conn - .query_row( - "SELECT * FROM episodes WHERE id = ?1 AND scope_key = ?2", - params![id, self.bucket()], - read_episode, - ) - .optional()?; - found.transpose() - } - - async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()> { - let conn = self.guard()?; - // Replace, not overlay: `INSERT OR REPLACE` only touches the sequence - // numbers present in `steps`, so a shorter re-save would leave the old - // tail behind and `steps()` would stitch two attempts together. - conn.execute( - "DELETE FROM attempt_steps WHERE scope_key = ?1 AND row_id = ?2", - params![self.bucket(), row_id], - )?; - for (seq, step) in steps.iter().enumerate() { - conn.execute( - "INSERT OR REPLACE INTO attempt_steps(scope_key, row_id, seq, node_id, status, - output, duration_ms, null_bindings, - transcript) - VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9)", - params![ - self.bucket(), - row_id, - i64::try_from(seq).unwrap_or(i64::MAX), - step.node_id, - serde_json::to_string(&step.status) - .map_err(|e| LedgerError::Corrupt(e.to_string()))? - .trim_matches('"'), - serde_json::to_string(&step.output) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - i64::try_from(step.duration_ms).unwrap_or(i64::MAX), - serde_json::to_string(&step.null_bindings) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - serde_json::to_string(&step.transcript) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - ], - )?; - } - Ok(()) - } - - async fn steps(&self, row_id: &str) -> Result> { - let conn = self.guard()?; - let mut stmt = conn.prepare( - "SELECT node_id, status, output, duration_ms, null_bindings, transcript - FROM attempt_steps - WHERE scope_key = ?1 AND row_id = ?2 ORDER BY seq", - )?; - let found = stmt - .query_map(params![self.bucket(), row_id], |r| { - Ok(( - r.get::<_, String>(0)?, - r.get::<_, String>(1)?, - r.get::<_, String>(2)?, - r.get::<_, i64>(3)?, - r.get::<_, String>(4)?, - r.get::<_, String>(5)?, - )) - })? - .collect::>>()?; - - found - .into_iter() - .map( - |(node_id, status, output, duration_ms, bindings, transcript)| { - Ok(crate::execute::StepRecord { - node_id, - status: if status == "error" { - crate::execute::StepOutcome::Error - } else { - crate::execute::StepOutcome::Success - }, - output: serde_json::from_str(&output) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - duration_ms: u64::try_from(duration_ms).unwrap_or(0), - null_bindings: serde_json::from_str(&bindings) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - transcript: serde_json::from_str(&transcript) - .map_err(|e| LedgerError::Corrupt(e.to_string()))?, - }) - }, - ) - .collect() - } - - async fn episodes(&self, running_only: bool, page: super::Page) -> Result> { - let conn = self.guard()?; - let mut stmt = conn - .prepare("SELECT * FROM episodes WHERE scope_key = ?1 ORDER BY updated_at DESC, id")?; - let all = stmt - .query_map([self.bucket()], read_episode)? - .collect::>>()?; - let kept: Result> = all - .into_iter() - .filter(|e| { - !running_only || e.as_ref().is_ok_and(|e| e.status == EpisodeStatus::Running) - }) - .collect(); - Ok(page.apply(kept?)) - } - - async fn children_of(&self, id: &str) -> Result> { - let conn = self.guard()?; - let mut stmt = conn.prepare( - "SELECT variant FROM variants WHERE scope_key = ?1 AND parent = ?2 ORDER BY variant", - )?; - let found = stmt - .query_map(params![self.bucket(), id], |r| r.get(0))? - .collect::>>()?; - Ok(found) - } -} - +include!("sqlite/ledger_impl.rs"); #[cfg(test)] -mod tests { - use super::*; - use crate::ledger::conformance; - - #[tokio::test] - async fn passes_the_conformance_suite() { - let store = SqliteLedger::in_memory().expect("open in-memory ledger"); - conformance::run_all(&store).await; - } - - #[tokio::test] - async fn passes_the_tenant_isolation_suite() { - let store = SqliteLedger::in_memory().expect("open in-memory ledger"); - let a = store.for_tenant("user-a"); - let b = store.for_tenant("user-b"); - conformance::run_tenants(&store, &a, &b).await; - } - - #[tokio::test] - async fn a_scoped_handle_shares_the_connection_rather_than_the_file() { - // Two handles for the SAME tenant must see each other's writes — that - // is what "shares" means. Probing it across scopes would now fail by - // design, because rows carry the bucket that wrote them. - let store = SqliteLedger::in_memory().expect("open in-memory ledger"); - let one = store.for_tenant("user-a"); - let two = store.for_tenant("user-a"); - one.append(&conformance::row("ep-shared", 1, "authored")) - .await - .expect("append"); - assert_eq!(two.rows("ep-shared").await.expect("rows").len(), 1); - assert!( - store.rows("ep-shared").await.expect("rows").is_empty(), - "and the global bucket is its own, not a union" - ); - } - - #[tokio::test] - async fn opening_a_path_creates_the_directory_holding_it() { - // A first run against `/var/lib/whatever/ledger.db` must not fail - // because nobody made the folder. - let root = std::env::temp_dir().join(format!("adaptive-mkdir-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&root); - let path = root.join("deep").join("nested").join("ledger.db"); - - let store = SqliteLedger::open(&path).expect("open"); - store - .append(&conformance::row("ep-mkdir", 1, "authored")) - .await - .expect("append"); - assert!(path.exists(), "{}", path.display()); - let _ = std::fs::remove_dir_all(&root); - } - - #[test] - fn the_environment_moves_the_ledger_without_a_rebuild() { - let fallback = std::path::Path::new("/srv/app/ledger.db"); - assert_eq!( - chosen_path(Some("/mnt/data/ledger.db"), fallback), - std::path::PathBuf::from("/mnt/data/ledger.db") - ); - } - - #[test] - fn an_unset_environment_falls_back_to_the_path_in_the_code() { - let fallback = std::path::Path::new("/srv/app/ledger.db"); - assert_eq!(chosen_path(None, fallback), fallback); - } - - #[test] - fn a_blank_variable_reads_as_unset_rather_than_as_an_empty_path() { - // What a shell leaves behind when a value was meant to be interpolated - // and was not. Opening "" fails in a way that names nothing useful. - let fallback = std::path::Path::new("/srv/app/ledger.db"); - assert_eq!(chosen_path(Some(""), fallback), fallback); - assert_eq!(chosen_path(Some(" "), fallback), fallback); - } - - fn fake_env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option + use<> { - let owned: Vec<(String, String)> = pairs - .iter() - .map(|(k, v)| ((*k).to_string(), (*v).to_string())) - .collect(); - move |key: &str| owned.iter().find(|(k, _)| k == key).map(|(_, v)| v.clone()) - } - - #[test] - fn each_platform_uses_its_own_documented_directory() { - let home = fake_env(&[("HOME", "/home/ada")]); - assert_eq!( - data_dir(Platform::Xdg, &home), - Some("/home/ada/.local/share".into()) - ); - assert_eq!( - data_dir(Platform::MacOs, &fake_env(&[("HOME", "/Users/ada")])), - Some("/Users/ada/Library/Application Support".into()) - ); - assert_eq!( - data_dir( - Platform::Windows, - &fake_env(&[("LOCALAPPDATA", "C:\\Users\\ada\\AppData\\Local")]) - ), - Some("C:\\Users\\ada\\AppData\\Local".into()) - ); - } - - #[test] - fn xdg_data_home_wins_over_the_spec_s_own_fallback() { - let env = fake_env(&[("XDG_DATA_HOME", "/data"), ("HOME", "/home/ada")]); - assert_eq!(data_dir(Platform::Xdg, &env), Some("/data".into())); - } - - #[test] - fn windows_uses_the_local_profile_not_the_roaming_one() { - // A roaming profile syncs between machines, and a SQLite file copied - // mid-write between two that both think they own it is a corrupted - // database. Setting only APPDATA must therefore find nothing. - let roaming = fake_env(&[("APPDATA", "C:\\Users\\ada\\AppData\\Roaming")]); - assert_eq!(data_dir(Platform::Windows, &roaming), None); - } - - #[test] - fn the_conventional_path_is_namespaced_by_project_and_named_for_the_crate() { - let env = fake_env(&[("HOME", "/home/ada")]); - assert_eq!( - default_path(Platform::Xdg, &env).expect("path"), - std::path::PathBuf::from("/home/ada/.local/share/tinyflows/adaptive.db") - ); - } - - #[test] - fn the_variable_still_wins_over_the_convention() { - let env = fake_env(&[(DB_PATH_VAR, "/mnt/data/ledger.db"), ("HOME", "/home/ada")]); - assert_eq!( - default_path(Platform::Xdg, &env).expect("path"), - std::path::PathBuf::from("/mnt/data/ledger.db") - ); - } - - #[test] - fn nowhere_conventional_is_an_error_that_says_what_to_set() { - // A daemon under a user with no home. Guessing would put a database - // somewhere nobody looks, and losing it silently is the failure this - // whole crate is written to avoid. - let err = default_path(Platform::Xdg, &fake_env(&[])).expect_err("no home"); - assert!(err.to_string().contains(DB_PATH_VAR), "{err}"); - } - - #[test] - fn a_configured_path_is_trimmed() { - let fallback = std::path::Path::new("/srv/app/ledger.db"); - assert_eq!( - chosen_path(Some(" /mnt/data/ledger.db\n"), fallback), - std::path::PathBuf::from("/mnt/data/ledger.db") - ); - } - - #[tokio::test] - async fn a_reopened_ledger_still_has_its_rows() { - // The whole point of the sqlite backend over the in-memory one. - let dir = std::env::temp_dir().join(format!("adaptive-ledger-{}", std::process::id())); - std::fs::create_dir_all(&dir).expect("temp dir"); - let path = dir.join("ledger.db"); - let _ = std::fs::remove_file(&path); - - { - let store = SqliteLedger::open(&path).expect("open"); - store - .append(&conformance::row("ep", 1, "authored")) - .await - .expect("append"); - } - let reopened = SqliteLedger::open(&path).expect("reopen"); - assert_eq!(reopened.rows("ep").await.expect("rows").len(), 1); - - let _ = std::fs::remove_dir_all(&dir); - } - - #[tokio::test] - async fn insertion_order_survives_a_timestamp_tie() { - // Two attempts finishing in the same second is common; ordering by `at` - // would make the exclusion list arbitrary. - let store = SqliteLedger::in_memory().expect("open"); - for sig in ["first", "second", "third"] { - let mut r = conformance::row("tie", 1, sig); - r.at = "2026-01-01T00:00:00Z".to_string(); - store.append(&r).await.expect("append"); - } - assert_eq!( - store.tried("tie").await.expect("tried"), - vec!["first", "second", "third"] - ); - } -} +#[path = "sqlite_tests.rs"] +mod tests; diff --git a/crates/tinyflows-adaptive/src/ledger/sqlite/ledger_impl.rs b/crates/tinyflows-adaptive/src/ledger/sqlite/ledger_impl.rs new file mode 100644 index 00000000..836b7e9d --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/sqlite/ledger_impl.rs @@ -0,0 +1,332 @@ +#[async_trait] +impl Ledger for SqliteLedger { + fn scope(&self) -> Option<&str> { + self.scope.as_deref() + } + + async fn append(&self, row: &LedgerRow) -> Result { + let conn = self.guard()?; + let seq = next_seq(&conn, "ledger_rows")?; + let id = new_id("ldg", seq); + conn.execute( + "INSERT INTO ledger_rows(id, episode, attempt, approach_sig, approach_desc, + workflow_id, outcome, cause, cost_usd, at, + satisfied, advanced, scope_key, seq) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14)", + params![ + id, + row.episode, + i64::from(row.attempt), + row.approach_sig, + row.approach_desc, + row.workflow_id, + row.outcome, + row.cause, + row.cost_usd, + row.at, + i64::from(row.satisfied), + i64::from(row.advanced), + self.bucket(), + seq, + ], + )?; + Ok(id) + } + + async fn rows(&self, episode: &str) -> Result> { + let conn = self.guard()?; + // Scoped as well as keyed by episode. An episode id is opaque and a + // service may hand one straight through from a request path, so this + // must not be the one read where guessing an id is enough. + let mut stmt = conn.prepare( + "SELECT * FROM ledger_rows WHERE episode = ?1 AND scope_key = ?2 ORDER BY seq", + )?; + let found = stmt + .query_map(params![episode, self.bucket()], read_row)? + .collect::>>()?; + Ok(found) + } + + async fn promote(&self, lesson: &Lesson, cites: &[String]) -> Result { + let conn = self.guard()?; + let seq = next_seq(&conn, "lessons")?; + let id = new_id("les", seq); + conn.execute( + "INSERT INTO lessons(id, kind, trigger, mechanism, claim, applied, helped, + scope_key, seq) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9)", + params![ + id, + serde_json::to_string(&lesson.kind) + .map_err(|e| LedgerError::Corrupt(e.to_string()))? + .trim_matches('"'), + lesson.trigger, + lesson.mechanism, + lesson.claim, + i64::from(lesson.applied), + i64::from(lesson.helped), + // The handle's, never the argument's. + self.bucket(), + seq, + ], + )?; + for row_id in cites { + conn.execute( + "INSERT OR IGNORE INTO lesson_evidence(lesson_id, row_id) VALUES(?1,?2)", + params![id, row_id], + )?; + } + Ok(id) + } + + async fn lessons(&self, kind: Option) -> Result> { + let conn = self.guard()?; + // This bucket plus global. An unscoped handle's bucket is global, so + // the two halves coincide and it sees exactly what it wrote. + let mut stmt = conn + .prepare("SELECT * FROM lessons WHERE scope_key = ?1 OR scope_key = '' ORDER BY seq")?; + let all = stmt + .query_map([self.bucket()], |r| { + let scope: String = r.get("scope_key")?; + Ok(Lesson { + id: r.get("id")?, + kind: LessonKind::parse(&r.get::<_, String>("kind")?), + trigger: r.get("trigger")?, + mechanism: r.get("mechanism")?, + claim: r.get("claim")?, + applied: r.get::<_, i64>("applied")?.try_into().unwrap_or(0), + helped: r.get::<_, i64>("helped")?.try_into().unwrap_or(0), + scope_key: (!scope.is_empty()).then_some(scope), + }) + })? + .collect::>>()?; + Ok(match kind { + Some(want) => all.into_iter().filter(|l| l.kind == want).collect(), + None => all, + }) + } + + async fn evidence(&self, lesson_id: &str) -> Result> { + let conn = self.guard()?; + let mut stmt = conn.prepare( + "SELECT r.* FROM ledger_rows r + JOIN lesson_evidence e ON e.row_id = r.id + WHERE e.lesson_id = ?1 AND (r.scope_key = ?2 OR r.scope_key = '') + ORDER BY r.seq", + )?; + let found = stmt + .query_map(params![lesson_id, self.bucket()], read_row)? + .collect::>>()?; + Ok(found) + } + + async fn score_lesson(&self, lesson_id: &str, helped: bool) -> Result<()> { + let conn = self.guard()?; + conn.execute( + // Constrained to what this handle can see — the id arrives from + // model output, and naming another tenant's lesson must not move + // its score. + "UPDATE lessons SET applied = applied + 1, helped = helped + ?2 + WHERE id = ?1 AND (scope_key = ?3 OR scope_key = '')", + params![lesson_id, i64::from(helped), self.bucket()], + )?; + Ok(()) + } + + async fn score_workflow(&self, workflow_id: &str, helped: bool) -> Result<()> { + let conn = self.guard()?; + // Upsert: the first run of a workflow is the common case and must not + // need a separate registration step. + conn.execute( + "INSERT INTO workflow_scores(scope_key, workflow_id, applied, helped) + VALUES(?1, ?2, 1, ?3) + ON CONFLICT(scope_key, workflow_id) DO UPDATE SET + applied = applied + 1, + helped = helped + ?3", + params![self.bucket(), workflow_id, i64::from(helped)], + )?; + Ok(()) + } + + async fn workflow_score(&self, workflow_id: &str) -> Result { + let conn = self.guard()?; + let found = conn + .query_row( + "SELECT applied, helped FROM workflow_scores + WHERE scope_key = ?1 AND workflow_id = ?2", + params![self.bucket(), workflow_id], + |r| { + Ok(Score { + applied: r.get::<_, i64>(0)?.try_into().unwrap_or(0), + helped: r.get::<_, i64>(1)?.try_into().unwrap_or(0), + }) + }, + ) + .optional()?; + Ok(found.unwrap_or_default()) + } + + async fn link_variant(&self, parent: &str, variant: &str) -> Result<()> { + let conn = self.guard()?; + conn.execute( + "INSERT OR IGNORE INTO variants(scope_key, variant, parent) VALUES(?1,?2,?3)", + params![self.bucket(), variant, parent], + )?; + Ok(()) + } + + async fn parent_of(&self, id: &str) -> Result> { + let conn = self.guard()?; + let found = conn + .query_row( + "SELECT parent FROM variants WHERE scope_key = ?1 AND variant = ?2", + params![self.bucket(), id], + |r| r.get(0), + ) + .optional()?; + Ok(found) + } + + async fn save_episode(&self, episode: &Episode) -> Result<()> { + let conn = self.guard()?; + conn.execute( + "INSERT INTO episodes(id, scope_key, goal, status, attempt, stalled, + started_at, updated_at) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8) + ON CONFLICT(scope_key, id) DO UPDATE SET + goal = ?3, status = ?4, attempt = ?5, stalled = ?6, updated_at = ?8", + params![ + episode.id, + self.bucket(), + serde_json::to_string(&episode.goal) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + serde_json::to_string(&episode.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + i64::from(episode.attempt), + i64::from(episode.stalled), + episode.started_at, + episode.updated_at, + ], + )?; + Ok(()) + } + + async fn episode(&self, id: &str) -> Result> { + let conn = self.guard()?; + let found = conn + .query_row( + "SELECT * FROM episodes WHERE id = ?1 AND scope_key = ?2", + params![id, self.bucket()], + read_episode, + ) + .optional()?; + found.transpose() + } + + async fn save_steps(&self, row_id: &str, steps: &[crate::execute::StepRecord]) -> Result<()> { + let conn = self.guard()?; + // Replace, not overlay: `INSERT OR REPLACE` only touches the sequence + // numbers present in `steps`, so a shorter re-save would leave the old + // tail behind and `steps()` would stitch two attempts together. + conn.execute( + "DELETE FROM attempt_steps WHERE scope_key = ?1 AND row_id = ?2", + params![self.bucket(), row_id], + )?; + for (seq, step) in steps.iter().enumerate() { + conn.execute( + "INSERT OR REPLACE INTO attempt_steps(scope_key, row_id, seq, node_id, status, + output, duration_ms, null_bindings, + transcript) + VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9)", + params![ + self.bucket(), + row_id, + i64::try_from(seq).unwrap_or(i64::MAX), + step.node_id, + serde_json::to_string(&step.status) + .map_err(|e| LedgerError::Corrupt(e.to_string()))? + .trim_matches('"'), + serde_json::to_string(&step.output) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + i64::try_from(step.duration_ms).unwrap_or(i64::MAX), + serde_json::to_string(&step.null_bindings) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + serde_json::to_string(&step.transcript) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + ], + )?; + } + Ok(()) + } + + async fn steps(&self, row_id: &str) -> Result> { + let conn = self.guard()?; + let mut stmt = conn.prepare( + "SELECT node_id, status, output, duration_ms, null_bindings, transcript + FROM attempt_steps + WHERE scope_key = ?1 AND row_id = ?2 ORDER BY seq", + )?; + let found = stmt + .query_map(params![self.bucket(), row_id], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, String>(2)?, + r.get::<_, i64>(3)?, + r.get::<_, String>(4)?, + r.get::<_, String>(5)?, + )) + })? + .collect::>>()?; + + found + .into_iter() + .map( + |(node_id, status, output, duration_ms, bindings, transcript)| { + Ok(crate::execute::StepRecord { + node_id, + status: if status == "error" { + crate::execute::StepOutcome::Error + } else { + crate::execute::StepOutcome::Success + }, + output: serde_json::from_str(&output) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + duration_ms: u64::try_from(duration_ms).unwrap_or(0), + null_bindings: serde_json::from_str(&bindings) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + transcript: serde_json::from_str(&transcript) + .map_err(|e| LedgerError::Corrupt(e.to_string()))?, + }) + }, + ) + .collect() + } + + async fn episodes(&self, running_only: bool, page: super::Page) -> Result> { + let conn = self.guard()?; + let mut stmt = conn + .prepare("SELECT * FROM episodes WHERE scope_key = ?1 ORDER BY updated_at DESC, id")?; + let all = stmt + .query_map([self.bucket()], read_episode)? + .collect::>>()?; + let kept: Result> = all + .into_iter() + .filter(|e| { + !running_only || e.as_ref().is_ok_and(|e| e.status == EpisodeStatus::Running) + }) + .collect(); + Ok(page.apply(kept?)) + } + + async fn children_of(&self, id: &str) -> Result> { + let conn = self.guard()?; + let mut stmt = conn.prepare( + "SELECT variant FROM variants WHERE scope_key = ?1 AND parent = ?2 ORDER BY variant", + )?; + let found = stmt + .query_map(params![self.bucket(), id], |r| r.get(0))? + .collect::>>()?; + Ok(found) + } +} diff --git a/crates/tinyflows-adaptive/src/ledger/sqlite_tests.rs b/crates/tinyflows-adaptive/src/ledger/sqlite_tests.rs new file mode 100644 index 00000000..385a95a4 --- /dev/null +++ b/crates/tinyflows-adaptive/src/ledger/sqlite_tests.rs @@ -0,0 +1,223 @@ +use super::*; +use crate::ledger::conformance; + +#[tokio::test] +async fn passes_the_conformance_suite() { + let store = SqliteLedger::in_memory().expect("open in-memory ledger"); + conformance::run_all(&store).await; +} + +#[tokio::test] +async fn passes_the_tenant_isolation_suite() { + let store = SqliteLedger::in_memory().expect("open in-memory ledger"); + let a = store.for_tenant("user-a"); + let b = store.for_tenant("user-b"); + conformance::run_tenants(&store, &a, &b).await; +} + +#[tokio::test] +async fn opening_a_legacy_episode_table_migrates_to_tenant_scoped_identity() { + let root = + std::env::temp_dir().join(format!("adaptive-episode-migration-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).expect("temp dir"); + let path = root.join("ledger.db"); + let connection = Connection::open(&path).expect("legacy database"); + connection + .execute_batch( + "CREATE TABLE episodes ( + id TEXT PRIMARY KEY, + scope_key TEXT NOT NULL DEFAULT '', + goal TEXT NOT NULL, + status TEXT NOT NULL, + attempt INTEGER NOT NULL DEFAULT 0, + stalled INTEGER NOT NULL DEFAULT 0, + started_at TEXT NOT NULL, + updated_at TEXT NOT NULL + );", + ) + .expect("legacy schema"); + drop(connection); + + let store = SqliteLedger::open(&path).expect("migrate legacy schema"); + let a = store.for_tenant("user-a"); + let b = store.for_tenant("user-b"); + conformance::run_tenants(&store, &a, &b).await; + + let _ = std::fs::remove_dir_all(&root); +} + +#[tokio::test] +async fn a_scoped_handle_shares_the_connection_rather_than_the_file() { + // Two handles for the SAME tenant must see each other's writes — that + // is what "shares" means. Probing it across scopes would now fail by + // design, because rows carry the bucket that wrote them. + let store = SqliteLedger::in_memory().expect("open in-memory ledger"); + let one = store.for_tenant("user-a"); + let two = store.for_tenant("user-a"); + one.append(&conformance::row("ep-shared", 1, "authored")) + .await + .expect("append"); + assert_eq!(two.rows("ep-shared").await.expect("rows").len(), 1); + assert!( + store.rows("ep-shared").await.expect("rows").is_empty(), + "and the global bucket is its own, not a union" + ); +} + +#[tokio::test] +async fn opening_a_path_creates_the_directory_holding_it() { + // A first run against `/var/lib/whatever/ledger.db` must not fail + // because nobody made the folder. + let root = std::env::temp_dir().join(format!("adaptive-mkdir-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + let path = root.join("deep").join("nested").join("ledger.db"); + + let store = SqliteLedger::open(&path).expect("open"); + store + .append(&conformance::row("ep-mkdir", 1, "authored")) + .await + .expect("append"); + assert!(path.exists(), "{}", path.display()); + let _ = std::fs::remove_dir_all(&root); +} + +#[test] +fn the_environment_moves_the_ledger_without_a_rebuild() { + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!( + chosen_path(Some("/mnt/data/ledger.db"), fallback), + std::path::PathBuf::from("/mnt/data/ledger.db") + ); +} + +#[test] +fn an_unset_environment_falls_back_to_the_path_in_the_code() { + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!(chosen_path(None, fallback), fallback); +} + +#[test] +fn a_blank_variable_reads_as_unset_rather_than_as_an_empty_path() { + // What a shell leaves behind when a value was meant to be interpolated + // and was not. Opening "" fails in a way that names nothing useful. + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!(chosen_path(Some(""), fallback), fallback); + assert_eq!(chosen_path(Some(" "), fallback), fallback); +} + +fn fake_env(pairs: &[(&str, &str)]) -> impl Fn(&str) -> Option + use<> { + let owned: Vec<(String, String)> = pairs + .iter() + .map(|(k, v)| ((*k).to_string(), (*v).to_string())) + .collect(); + move |key: &str| owned.iter().find(|(k, _)| k == key).map(|(_, v)| v.clone()) +} + +#[test] +fn each_platform_uses_its_own_documented_directory() { + let home = fake_env(&[("HOME", "/home/ada")]); + assert_eq!( + data_dir(Platform::Xdg, &home), + Some("/home/ada/.local/share".into()) + ); + assert_eq!( + data_dir(Platform::MacOs, &fake_env(&[("HOME", "/Users/ada")])), + Some("/Users/ada/Library/Application Support".into()) + ); + assert_eq!( + data_dir( + Platform::Windows, + &fake_env(&[("LOCALAPPDATA", "C:\\Users\\ada\\AppData\\Local")]) + ), + Some("C:\\Users\\ada\\AppData\\Local".into()) + ); +} + +#[test] +fn xdg_data_home_wins_over_the_spec_s_own_fallback() { + let env = fake_env(&[("XDG_DATA_HOME", "/data"), ("HOME", "/home/ada")]); + assert_eq!(data_dir(Platform::Xdg, &env), Some("/data".into())); +} + +#[test] +fn windows_uses_the_local_profile_not_the_roaming_one() { + // A roaming profile syncs between machines, and a SQLite file copied + // mid-write between two that both think they own it is a corrupted + // database. Setting only APPDATA must therefore find nothing. + let roaming = fake_env(&[("APPDATA", "C:\\Users\\ada\\AppData\\Roaming")]); + assert_eq!(data_dir(Platform::Windows, &roaming), None); +} + +#[test] +fn the_conventional_path_is_namespaced_by_project_and_named_for_the_crate() { + let env = fake_env(&[("HOME", "/home/ada")]); + assert_eq!( + default_path(Platform::Xdg, &env).expect("path"), + std::path::PathBuf::from("/home/ada/.local/share/tinyflows/adaptive.db") + ); +} + +#[test] +fn the_variable_still_wins_over_the_convention() { + let env = fake_env(&[(DB_PATH_VAR, "/mnt/data/ledger.db"), ("HOME", "/home/ada")]); + assert_eq!( + default_path(Platform::Xdg, &env).expect("path"), + std::path::PathBuf::from("/mnt/data/ledger.db") + ); +} + +#[test] +fn nowhere_conventional_is_an_error_that_says_what_to_set() { + // A daemon under a user with no home. Guessing would put a database + // somewhere nobody looks, and losing it silently is the failure this + // whole crate is written to avoid. + let err = default_path(Platform::Xdg, &fake_env(&[])).expect_err("no home"); + assert!(err.to_string().contains(DB_PATH_VAR), "{err}"); +} + +#[test] +fn a_configured_path_is_trimmed() { + let fallback = std::path::Path::new("/srv/app/ledger.db"); + assert_eq!( + chosen_path(Some(" /mnt/data/ledger.db\n"), fallback), + std::path::PathBuf::from("/mnt/data/ledger.db") + ); +} + +#[tokio::test] +async fn a_reopened_ledger_still_has_its_rows() { + // The whole point of the sqlite backend over the in-memory one. + let dir = std::env::temp_dir().join(format!("adaptive-ledger-{}", std::process::id())); + std::fs::create_dir_all(&dir).expect("temp dir"); + let path = dir.join("ledger.db"); + let _ = std::fs::remove_file(&path); + + { + let store = SqliteLedger::open(&path).expect("open"); + store + .append(&conformance::row("ep", 1, "authored")) + .await + .expect("append"); + } + let reopened = SqliteLedger::open(&path).expect("reopen"); + assert_eq!(reopened.rows("ep").await.expect("rows").len(), 1); + + let _ = std::fs::remove_dir_all(&dir); +} + +#[tokio::test] +async fn insertion_order_survives_a_timestamp_tie() { + // Two attempts finishing in the same second is common; ordering by `at` + // would make the exclusion list arbitrary. + let store = SqliteLedger::in_memory().expect("open"); + for sig in ["first", "second", "third"] { + let mut r = conformance::row("tie", 1, sig); + r.at = "2026-01-01T00:00:00Z".to_string(); + store.append(&r).await.expect("append"); + } + assert_eq!( + store.tried("tie").await.expect("tried"), + vec!["first", "second", "third"] + ); +} diff --git a/crates/tinyflows-adaptive/tests/closing/closing_part_01_tests.rs b/crates/tinyflows-adaptive/tests/closing/closing_part_01_tests.rs new file mode 100644 index 00000000..511549b2 --- /dev/null +++ b/crates/tinyflows-adaptive/tests/closing/closing_part_01_tests.rs @@ -0,0 +1,237 @@ +/// A plan that calls two saved workflows, with the second one's step failing. +fn composed() -> WorkflowGraph { + use tinyflows::model::{Edge, Node, NodeKind}; + let call = |id: &str, workflow: &str| Node { + id: id.to_string(), + kind: NodeKind::SubWorkflow, + type_version: 1, + name: id.to_string(), + config: json!({ "workflow_id": workflow }), + ports: Vec::new(), + position: None, + }; + WorkflowGraph { + schema_version: 1, + id: None, + name: "composed".into(), + inputs: Vec::new(), + agents: Vec::new(), + nodes: vec![ + Node { + id: "start".into(), + kind: NodeKind::Trigger, + type_version: 1, + name: "start".into(), + config: json!({ "trigger_kind": "manual" }), + ports: Vec::new(), + position: None, + }, + call("write_haiku", "haiku-writer"), + call("write_limerick", "limerick-writer"), + call("never_reached", "epic-writer"), + ], + edges: vec![ + Edge { + from_node: "start".into(), + from_port: "main".into(), + to_node: "write_haiku".into(), + to_port: "main".into(), + }, + Edge { + from_node: "write_haiku".into(), + from_port: "main".into(), + to_node: "write_limerick".into(), + to_port: "main".into(), + }, + Edge { + from_node: "write_limerick".into(), + from_port: "main".into(), + to_node: "never_reached".into(), + to_port: "main".into(), + }, + ], + } +} + +fn step(node: &str, ok: bool) -> tinyflows_adaptive::execute::StepRecord { + use tinyflows_adaptive::execute::{StepOutcome, StepRecord}; + StepRecord { + node_id: node.to_string(), + status: if ok { + StepOutcome::Success + } else { + StepOutcome::Error + }, + output: Value::Null, + duration_ms: 1, + null_bindings: Vec::new(), + transcript: Vec::new(), + } +} + +#[tokio::test] +async fn a_workflow_called_by_a_plan_earns_the_same_record_a_chosen_one_does() { + // Without this a workflow only ever used as a component stays Unproven + // forever: the chooser distrusts it and the promotion gate cannot see it, + // so composition becomes a place procedures go to stop earning a + // reputation. + let llm = Scripted::new(vec![json!({ + "satisfied": true, "blocker": "none", "gap": "", "advanced": true + })]); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis::default(); + let outcome = RunOutcome { + output: json!({}), + pending_approvals: Vec::new(), + cancelled: false, + }; + let mut finished = ran(&outcome, &diagnosis, "wrote the document"); + finished.steps = vec![ + step("write_haiku", true), + // Errored inside a plan that recovered around it. + step("write_limerick", false), + ]; + + close( + &Goal::new("a haiku and a limerick"), + "ep-compose", + 1, + &Approach::Authored { + why: "compose the two writers".into(), + fingerprint: "abc1234".into(), + }, + &composed(), + &finished, + &Budget::default(), + &ledger, + &caps_with(llm), + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closes"); + + let haiku = ledger.workflow_score("haiku-writer").await.expect("scored"); + assert_eq!( + (haiku.applied, haiku.helped), + (1, 1), + "it ran and the attempt was satisfied — the standard a selection is held to" + ); + + let limerick = ledger + .workflow_score("limerick-writer") + .await + .expect("scored"); + assert_eq!( + (limerick.applied, limerick.helped), + (1, 0), + "a child that errored inside a satisfied plan was exercised, not vindicated" + ); + + let never = ledger.workflow_score("epic-writer").await.expect("scored"); + assert_eq!( + (never.applied, never.helped), + (0, 0), + "a call the run never reached is not evidence of anything" + ); +} + +#[tokio::test] +async fn a_called_workflow_earns_nothing_from_an_attempt_that_fell_short() { + // The counters must stay readable as evidence: an unsatisfied episode + // gives a component `applied` and no more, exactly as it would a chosen + // workflow that failed. + let llm = Scripted::new(vec![json!({ + "satisfied": false, "blocker": "goal_not_met", + "gap": "the document is missing the limerick", "advanced": false + })]); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis::default(); + let outcome = RunOutcome { + output: json!({}), + pending_approvals: Vec::new(), + cancelled: false, + }; + let mut finished = ran(&outcome, &diagnosis, "wrote half a document"); + finished.steps = vec![step("write_haiku", true)]; + + close( + &Goal::new("a haiku and a limerick"), + "ep-short", + 1, + &Approach::Authored { + why: "compose the two writers".into(), + fingerprint: "abc1234".into(), + }, + &composed(), + &finished, + &Budget::default(), + &ledger, + &caps_with(llm), + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closes"); + + let haiku = ledger.workflow_score("haiku-writer").await.expect("scored"); + assert_eq!( + (haiku.applied, haiku.helped), + (1, 0), + "the child ran cleanly, but nothing it was part of was satisfied" + ); +} + +#[tokio::test] +async fn every_activation_of_a_looped_call_is_scored_not_just_the_first() { + // A node inside a loop produces one `StepRecord` per iteration. Reading + // only the first record credits the workflow once for work it did three + // times — and, worse, lets an early success hide a later error, so a child + // that failed a pass reads as clean. The counters are the only evidence the + // chooser and the promotion gate have; they have to count what happened. + let llm = Scripted::new(vec![json!({ + "satisfied": true, "blocker": "none", "gap": "", "advanced": true + })]); + let ledger = MemoryLedger::new(); + let diagnosis = Diagnosis::default(); + let outcome = RunOutcome { + output: json!({}), + pending_approvals: Vec::new(), + cancelled: false, + }; + let mut finished = ran(&outcome, &diagnosis, "wrote three haiku"); + // One node, three passes, mixed outcomes — the first one succeeding is + // exactly the arrangement that made the old reading look correct. + finished.steps = vec![ + step("write_haiku", true), + step("write_haiku", false), + step("write_haiku", true), + ]; + + close( + &Goal::new("three haiku"), + "ep-loop", + 1, + &Approach::Authored { + why: "call the writer once per subject".into(), + fingerprint: "abc1234".into(), + }, + &composed(), + &finished, + &Budget::default(), + &ledger, + &caps_with(llm), + None, + "2026-01-01T00:00:00Z", + ) + .await + .expect("closes"); + + let haiku = ledger.workflow_score("haiku-writer").await.expect("scored"); + assert_eq!( + (haiku.applied, haiku.helped), + (3, 2), + "three activations, and the one that errored is not vindicated by the \ + two that did not" + ); +} diff --git a/crates/tinyflows-adaptive/tests/closing.rs b/crates/tinyflows-adaptive/tests/closing_tests.rs similarity index 65% rename from crates/tinyflows-adaptive/tests/closing.rs rename to crates/tinyflows-adaptive/tests/closing_tests.rs index e8bb8d77..5c87dca2 100644 --- a/crates/tinyflows-adaptive/tests/closing.rs +++ b/crates/tinyflows-adaptive/tests/closing_tests.rs @@ -474,240 +474,4 @@ async fn an_episode_with_no_attempts_asks_nothing() { assert_eq!(llm.call_count(), 0); } -/// A plan that calls two saved workflows, with the second one's step failing. -fn composed() -> WorkflowGraph { - use tinyflows::model::{Edge, Node, NodeKind}; - let call = |id: &str, workflow: &str| Node { - id: id.to_string(), - kind: NodeKind::SubWorkflow, - type_version: 1, - name: id.to_string(), - config: json!({ "workflow_id": workflow }), - ports: Vec::new(), - position: None, - }; - WorkflowGraph { - schema_version: 1, - id: None, - name: "composed".into(), - inputs: Vec::new(), - agents: Vec::new(), - nodes: vec![ - Node { - id: "start".into(), - kind: NodeKind::Trigger, - type_version: 1, - name: "start".into(), - config: json!({ "trigger_kind": "manual" }), - ports: Vec::new(), - position: None, - }, - call("write_haiku", "haiku-writer"), - call("write_limerick", "limerick-writer"), - call("never_reached", "epic-writer"), - ], - edges: vec![ - Edge { - from_node: "start".into(), - from_port: "main".into(), - to_node: "write_haiku".into(), - to_port: "main".into(), - }, - Edge { - from_node: "write_haiku".into(), - from_port: "main".into(), - to_node: "write_limerick".into(), - to_port: "main".into(), - }, - Edge { - from_node: "write_limerick".into(), - from_port: "main".into(), - to_node: "never_reached".into(), - to_port: "main".into(), - }, - ], - } -} - -fn step(node: &str, ok: bool) -> tinyflows_adaptive::execute::StepRecord { - use tinyflows_adaptive::execute::{StepOutcome, StepRecord}; - StepRecord { - node_id: node.to_string(), - status: if ok { - StepOutcome::Success - } else { - StepOutcome::Error - }, - output: Value::Null, - duration_ms: 1, - null_bindings: Vec::new(), - transcript: Vec::new(), - } -} - -#[tokio::test] -async fn a_workflow_called_by_a_plan_earns_the_same_record_a_chosen_one_does() { - // Without this a workflow only ever used as a component stays Unproven - // forever: the chooser distrusts it and the promotion gate cannot see it, - // so composition becomes a place procedures go to stop earning a - // reputation. - let llm = Scripted::new(vec![json!({ - "satisfied": true, "blocker": "none", "gap": "", "advanced": true - })]); - let ledger = MemoryLedger::new(); - let diagnosis = Diagnosis::default(); - let outcome = RunOutcome { - output: json!({}), - pending_approvals: Vec::new(), - cancelled: false, - }; - let mut finished = ran(&outcome, &diagnosis, "wrote the document"); - finished.steps = vec![ - step("write_haiku", true), - // Errored inside a plan that recovered around it. - step("write_limerick", false), - ]; - - close( - &Goal::new("a haiku and a limerick"), - "ep-compose", - 1, - &Approach::Authored { - why: "compose the two writers".into(), - fingerprint: "abc1234".into(), - }, - &composed(), - &finished, - &Budget::default(), - &ledger, - &caps_with(llm), - None, - "2026-01-01T00:00:00Z", - ) - .await - .expect("closes"); - - let haiku = ledger.workflow_score("haiku-writer").await.expect("scored"); - assert_eq!( - (haiku.applied, haiku.helped), - (1, 1), - "it ran and the attempt was satisfied — the standard a selection is held to" - ); - - let limerick = ledger - .workflow_score("limerick-writer") - .await - .expect("scored"); - assert_eq!( - (limerick.applied, limerick.helped), - (1, 0), - "a child that errored inside a satisfied plan was exercised, not vindicated" - ); - - let never = ledger.workflow_score("epic-writer").await.expect("scored"); - assert_eq!( - (never.applied, never.helped), - (0, 0), - "a call the run never reached is not evidence of anything" - ); -} - -#[tokio::test] -async fn a_called_workflow_earns_nothing_from_an_attempt_that_fell_short() { - // The counters must stay readable as evidence: an unsatisfied episode - // gives a component `applied` and no more, exactly as it would a chosen - // workflow that failed. - let llm = Scripted::new(vec![json!({ - "satisfied": false, "blocker": "goal_not_met", - "gap": "the document is missing the limerick", "advanced": false - })]); - let ledger = MemoryLedger::new(); - let diagnosis = Diagnosis::default(); - let outcome = RunOutcome { - output: json!({}), - pending_approvals: Vec::new(), - cancelled: false, - }; - let mut finished = ran(&outcome, &diagnosis, "wrote half a document"); - finished.steps = vec![step("write_haiku", true)]; - - close( - &Goal::new("a haiku and a limerick"), - "ep-short", - 1, - &Approach::Authored { - why: "compose the two writers".into(), - fingerprint: "abc1234".into(), - }, - &composed(), - &finished, - &Budget::default(), - &ledger, - &caps_with(llm), - None, - "2026-01-01T00:00:00Z", - ) - .await - .expect("closes"); - - let haiku = ledger.workflow_score("haiku-writer").await.expect("scored"); - assert_eq!( - (haiku.applied, haiku.helped), - (1, 0), - "the child ran cleanly, but nothing it was part of was satisfied" - ); -} - -#[tokio::test] -async fn every_activation_of_a_looped_call_is_scored_not_just_the_first() { - // A node inside a loop produces one `StepRecord` per iteration. Reading - // only the first record credits the workflow once for work it did three - // times — and, worse, lets an early success hide a later error, so a child - // that failed a pass reads as clean. The counters are the only evidence the - // chooser and the promotion gate have; they have to count what happened. - let llm = Scripted::new(vec![json!({ - "satisfied": true, "blocker": "none", "gap": "", "advanced": true - })]); - let ledger = MemoryLedger::new(); - let diagnosis = Diagnosis::default(); - let outcome = RunOutcome { - output: json!({}), - pending_approvals: Vec::new(), - cancelled: false, - }; - let mut finished = ran(&outcome, &diagnosis, "wrote three haiku"); - // One node, three passes, mixed outcomes — the first one succeeding is - // exactly the arrangement that made the old reading look correct. - finished.steps = vec![ - step("write_haiku", true), - step("write_haiku", false), - step("write_haiku", true), - ]; - - close( - &Goal::new("three haiku"), - "ep-loop", - 1, - &Approach::Authored { - why: "call the writer once per subject".into(), - fingerprint: "abc1234".into(), - }, - &composed(), - &finished, - &Budget::default(), - &ledger, - &caps_with(llm), - None, - "2026-01-01T00:00:00Z", - ) - .await - .expect("closes"); - - let haiku = ledger.workflow_score("haiku-writer").await.expect("scored"); - assert_eq!( - (haiku.applied, haiku.helped), - (3, 2), - "three activations, and the one that errored is not vindicated by the \ - two that did not" - ); -} +include!("closing/closing_part_01_tests.rs"); diff --git a/crates/tinyflows-adaptive/tests/driver.rs b/crates/tinyflows-adaptive/tests/driver.rs deleted file mode 100644 index c01df08c..00000000 --- a/crates/tinyflows-adaptive/tests/driver.rs +++ /dev/null @@ -1,1021 +0,0 @@ -//! One instance, many goal runs, and an episode that outlives the process. -//! -//! These test the claim the `driver` module is built on, because it is the one -//! that is expensive to be wrong about: a `Loop` is per **tenant** and a goal -//! run is an **episode id**, so the same instance drives many episodes at once -//! and any instance can pick up an episode any other one started. - -use std::sync::{Arc, Mutex}; - -use async_trait::async_trait; -use serde_json::{Value, json}; -use tinyflows::caps::mock::mock_capabilities; -use tinyflows::caps::{Capabilities, LlmProvider}; -use tinyflows::error::Result as EngineResult; -use tinyflows::store::{FileWorkflowStore, WorkflowStore}; -use tinyflows_adaptive::contracts::Goal; -use tinyflows_adaptive::driver::{Clock, Loop}; -use tinyflows_adaptive::execute::{Local, Unobserved}; -use tinyflows_adaptive::host::HostFacts; -use tinyflows_adaptive::ledger::{EpisodeStatus, Ledger, Page, memory::MemoryLedger}; - -struct Frozen; -impl Clock for Frozen { - fn now(&self) -> String { - "2026-01-01T00:00:00Z".to_string() - } -} - -/// Answers every authoring call the same way, and keeps every request so the -/// tier can be read back off the wire. -struct Always { - reply: Value, - seen: Mutex>, -} - -impl Always { - fn new(reply: Value) -> Arc { - Arc::new(Self { - reply, - seen: Mutex::new(Vec::new()), - }) - } - fn tiers(&self) -> Vec { - self.seen - .lock() - .expect("lock") - .iter() - .map(|r| r["tier"].as_str().unwrap_or("(absent)").to_string()) - .collect() - } -} - -#[async_trait] -impl LlmProvider for Always { - async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { - self.seen.lock().expect("lock").push(request.clone()); - // The tier says which job is asking, so one double can answer them all. - Ok(match request["tier"].as_str().unwrap_or_default() { - "judge" => json!({ - "satisfied": false, "blocker": "goal_not_met", - "gap": "the report has no numbers in it", "advanced": false - }), - "consolidate" => json!({ "lessons": [], "corroborate": [] }), - "select" => json!({ "workflow_id": null, "why": "nothing fits" }), - _ => self.reply.clone(), - }) - } -} - -fn caps_with(llm: Arc) -> Capabilities { - Capabilities { - llm, - ..mock_capabilities() - } -} - -fn store(tag: &str) -> Arc { - let root = std::env::temp_dir().join(format!("adaptive-driver-{}-{tag}", std::process::id())); - let _ = std::fs::remove_dir_all(&root); - std::fs::create_dir_all(root.join("workflows")).expect("temp dir"); - Arc::new(FileWorkflowStore::new( - vec![root.join("workflows")], - root.join("runs"), - )) -} - -fn authoring() -> Arc { - Always::new(json!({ - "why": "nothing stored fits", - "inputs": {}, - "steps": [{ "id": "attempt", "run": "echo attempt-done" }], - })) -} - -#[tokio::test] -async fn one_instance_drives_two_goal_runs_with_independent_counters() { - // The claim the split rests on: the instance holds no per-episode state, so - // two episodes interleaved through it cannot contaminate each other. - let llm = authoring(); - let caps = caps_with(llm); - let ledger = MemoryLedger::new(); - let store = store("two"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - let engine = Loop { - ledger: &ledger, - store: &store, - caps: &caps, - facts: &HostFacts::unknown(), - runner: &runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - }; - - let goal = Goal::new("write the weekly report"); - engine.attempt("ep-a", &goal).await.expect("a1"); - engine.attempt("ep-b", &goal).await.expect("b1"); - engine.attempt("ep-a", &goal).await.expect("a2"); - - let a = ledger.episode("ep-a").await.expect("read").expect("exists"); - let b = ledger.episode("ep-b").await.expect("read").expect("exists"); - assert_eq!(a.attempt, 2); - assert_eq!(b.attempt, 1, "b is untouched by a's two passes"); - assert_eq!(a.stalled, 2, "neither of a's attempts advanced"); - assert_eq!(b.stalled, 1); - - assert_eq!(ledger.rows("ep-a").await.expect("rows").len(), 2); - assert_eq!(ledger.rows("ep-b").await.expect("rows").len(), 1); -} - -#[tokio::test] -async fn a_second_instance_picks_up_an_episode_the_first_one_started() { - // Kill the process mid-episode. Everything the loop needs is in the ledger, - // so a fresh instance continues the numbering rather than starting over - // with a trail that says it has already tried twice. - let ledger = MemoryLedger::new(); - let store = store("resume"); - let goal = Goal::new("write the weekly report"); - - { - let caps = caps_with(authoring()); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - let first = Loop { - ledger: &ledger, - store: &store, - caps: &caps, - facts: &HostFacts::unknown(), - runner: &runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - }; - first.attempt("ep-resume", &goal).await.expect("1"); - first.attempt("ep-resume", &goal).await.expect("2"); - } // the instance goes away, as a deploy would take it - - let unfinished = ledger.episodes(true, Page::ALL).await.expect("episodes"); - assert_eq!(unfinished.len(), 1, "the recovery list a boot reads"); - let recovered = &unfinished[0]; - assert_eq!(recovered.id, "ep-resume"); - assert_eq!(recovered.goal.text, "write the weekly report"); - assert_eq!(recovered.stalled, 2); - - let caps = caps_with(authoring()); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - let second = Loop { - ledger: &ledger, - store: &store, - caps: &caps, - facts: &HostFacts::unknown(), - runner: &runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - }; - let closed = second - .attempt(&recovered.id, &recovered.goal) - .await - .expect("3"); - - assert_eq!( - ledger - .episode("ep-resume") - .await - .expect("read") - .expect("exists") - .attempt, - 3, - "it continued rather than restarting at one" - ); - assert_eq!( - closed.stalled, 3, - "the stall count survived the process that was counting it" - ); -} - -#[tokio::test] -async fn every_inference_request_says_which_job_is_asking() { - // Without this a host cannot route judging and selecting to different - // models, which is the whole point of the tier. - let llm = authoring(); - let caps = caps_with(llm.clone()); - let ledger = MemoryLedger::new(); - let store = store("tiers"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - let engine = Loop { - ledger: &ledger, - store: &store, - caps: &caps, - facts: &HostFacts::unknown(), - runner: &runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - }; - - engine - .attempt("ep-tiers", &Goal::new("write the weekly report")) - .await - .expect("attempt"); - - let tiers = llm.tiers(); - assert!(!tiers.iter().any(|t| t == "(absent)"), "{tiers:?}"); - assert!(tiers.contains(&"author".to_string()), "{tiers:?}"); - assert!(tiers.contains(&"judge".to_string()), "{tiers:?}"); -} - -#[tokio::test] -async fn a_run_drives_to_a_stand_down_and_consolidates_once() { - // The judge never says satisfied and nothing advances, so the stall rule - // ends it. `run` must stop on its own rather than needing a bound of its - // own alongside the one `close` already applies. - let llm = authoring(); - let caps = caps_with(llm.clone()); - let ledger = MemoryLedger::new(); - let store = store("drive"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - let engine = Loop { - ledger: &ledger, - store: &store, - caps: &caps, - facts: &HostFacts::unknown(), - runner: &runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - }; - - let finished = engine - .run("ep-drive", &Goal::new("write the weekly report")) - .await - .expect("run"); - - match &finished.status { - EpisodeStatus::StoodDown(reason) => assert!(reason.contains("no progress"), "{reason}"), - other => panic!("expected a stand-down, got {other:?}"), - } - assert!(finished.attempts >= 2, "{finished:?}"); - assert!(finished.lessons.is_empty(), "nothing generalised"); - - // Consolidation is per episode, not per attempt. - assert_eq!( - llm.tiers().iter().filter(|t| *t == "consolidate").count(), - 1 - ); - - let record = ledger - .episode("ep-drive") - .await - .expect("read") - .expect("exists"); - assert!(matches!(record.status, EpisodeStatus::StoodDown(_))); - assert_ne!( - record.status, - EpisodeStatus::Running, - "a finished episode must leave the recovery list" - ); -} - -// --------------------------------------------------------------------------- -// The loop acquires a skill: authored, worked, kept, then selected. -// --------------------------------------------------------------------------- - -/// A graph parameterised by a declared input, which is what the authoring -/// prompt asks for and what makes a procedure worth keeping. -fn parameterised() -> Value { - json!({ - "why": "review", - "declared": [{ "name": "repo", "description": "the repository", "required": true }], - "inputs": { "repo": "acme/thing" }, - "steps": [{ - "id": "review", - "ask": "Review the open pull requests and summarise them directly." - }], - }) -} - -/// Authors `graph`, judges every run satisfied, and answers the naming call. -struct Succeeds { - authored: Value, - reusable: bool, - seen: Mutex>, -} - -#[async_trait] -impl LlmProvider for Succeeds { - async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { - self.seen.lock().expect("lock").push(request.clone()); - Ok(match request["tier"].as_str().unwrap_or_default() { - "judge" => json!({ "satisfied": true, "gap": "" }), - "consolidate" => json!({ "lessons": [], "corroborate": [] }), - "select" => json!({ "workflow_id": null, "why": "nothing fits yet" }), - "generalise" => json!({ - "name": "Review a repository's pull requests", - "description": "Reviews the open pull requests on a repository. Takes the repository as an input.", - "reusable": self.reusable, - }), - _ => self.authored.clone(), - }) - } -} - -fn succeeding(authored: Value, reusable: bool) -> Arc { - Arc::new(Succeeds { - authored, - reusable, - seen: Mutex::new(Vec::new()), - }) -} - -fn engine_over<'a>( - ledger: &'a dyn Ledger, - store: &'a Arc, - caps: &'a Capabilities, - runner: &'a Local<'a>, - facts: &'a HostFacts, -) -> Loop<'a> { - Loop { - ledger, - store, - caps, - facts, - runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - } -} - -#[tokio::test] -async fn a_graph_that_was_authored_and_worked_becomes_a_stored_procedure() { - // The headline claim: "selects a stored workflow or authors one" is only - // half true if authoring never becomes stored, because then the catalogue - // holds exactly what a person put there and the loop never acquires a skill. - let llm = succeeding(parameterised(), true); - let caps = Capabilities { - llm: llm.clone(), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let store = store("keep"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - - assert!(store.list().expect("list").is_empty(), "a cold store"); - - let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-learn", &Goal::new("review the PRs on acme/thing")) - .await - .expect("run"); - assert_eq!(finished.status, EpisodeStatus::Satisfied); - - let listed = store.list().expect("list"); - assert_eq!(listed.len(), 1, "the procedure was filed: {listed:?}"); - assert!(listed[0].id.starts_with("learned-"), "{}", listed[0].id); - assert!( - listed[0].description.contains("a repository"), - "described as a class, not as the goal: {}", - listed[0].description - ); - - // Scored from the run that earned it — entering at 0/0 would be - // indistinguishable from a procedure nobody has ever run. - let score = ledger.workflow_score(&listed[0].id).await.expect("score"); - assert_eq!((score.applied, score.helped), (1, 1)); -} - -#[tokio::test] -async fn a_graph_that_pasted_its_inputs_is_not_kept() { - // Same run, same success — but the goal's specifics are welded into a - // step, so it matches one task and never another. No model is asked. - // - // The paste sits in a `run` script, not an ask: the intake gate refuses - // ask-pastes outright now, and this test is about the layer BEHIND it — - // keep's own refusal, which still guards every path intake cannot see. - let mut baked = parameterised(); - baked["steps"] = json!([ - { "id": "review", "run": "gh pr list -R acme/thing" }, - { "id": "report", "ask": "Summarise the review output.", "reads": ["review"] } - ]); - - let llm = succeeding(baked, true); - let caps = Capabilities { - llm: llm.clone(), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let store = store("baked"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - - let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-baked", &Goal::new("review the PRs on acme/thing")) - .await - .expect("run"); - - assert_eq!(finished.status, EpisodeStatus::Satisfied, "it still worked"); - assert!( - store.list().expect("list").is_empty(), - "but it is a one-off" - ); - - let tiers: Vec = llm - .seen - .lock() - .expect("lock") - .iter() - .map(|r| r["tier"].as_str().unwrap_or_default().to_string()) - .collect(); - assert!( - !tiers.iter().any(|t| t == "generalise"), - "the mechanical gate settled it without paying for an opinion: {tiers:?}" - ); -} - -#[tokio::test] -async fn the_model_can_still_refuse_a_graph_the_gate_let_through() { - // Parameterised and reusable-looking, but only meaningful for the one goal - // it was written for. The gate cannot see that; a reader can. - let llm = succeeding(parameterised(), false); - let caps = Capabilities { - llm: llm.clone(), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let store = store("refused"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - - engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-refused", &Goal::new("review the PRs on acme/thing")) - .await - .expect("run"); - - assert!(store.list().expect("list").is_empty()); -} - -#[tokio::test] -async fn the_next_episode_selects_what_the_last_one_learned() { - // The whole point, end to end. Episode one finds a cold store and authors; - // episode two finds the procedure episode one filed. - let llm = succeeding(parameterised(), true); - let caps = Capabilities { - llm: llm.clone(), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let store = store("acquire"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - - engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-first", &Goal::new("review the PRs on acme/thing")) - .await - .expect("first"); - - let learned = store.list().expect("list")[0].id.clone(); - llm.seen.lock().expect("lock").clear(); - - engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-second", &Goal::new("review the PRs on other/repo")) - .await - .expect("second"); - - // The selector was offered it, with the evidence from episode one. - let offered = llm.seen.lock().expect("lock")[0]["messages"][1]["content"] - .as_str() - .unwrap_or_default() - .to_string(); - assert!(offered.contains(&learned), "{offered}"); - assert!( - offered.contains("run 1×, satisfied 1×"), - "carrying what it earned: {offered}" - ); -} - -// --------------------------------------------------------------------------- -// The two stores are independent: any backend beside any other. -// --------------------------------------------------------------------------- - -#[tokio::test] -async fn a_ledger_and_a_vault_of_different_kinds_drive_the_same_loop() { - // `Ledger` and `Vault` are separate traits with separate handles, so the - // host mixes them freely — sqlite ledger beside a Mongo vault, or either - // beside memory. Nothing in the loop knows which it got. - use tinyflows_adaptive::ledger::sqlite::SqliteLedger; - use tinyflows_adaptive::workflows::{Snapshot, Vault, memory::MemoryVault}; - - let dir = std::env::temp_dir().join(format!("adaptive-mixed-{}", std::process::id())); - let _ = std::fs::remove_dir_all(&dir); - - // Durable ledger, ephemeral vault. A deliberately silly pairing, chosen - // because if this compiles and runs then every sensible one does. - let ledger = SqliteLedger::open(dir.join("ledger.db")).expect("ledger"); - let vault = MemoryVault::new(); - - let llm = succeeding(parameterised(), true); - let caps = Capabilities { - llm: llm.clone(), - ..mock_capabilities() - }; - let policy: Arc = { - #[derive(Debug, Default)] - struct Permissive; - impl tinyflows::store::HostPolicy for Permissive {} - Arc::new(Permissive) - }; - let snapshot = Snapshot::load(&vault, policy).await.expect("snapshot"); - let store: Arc = Arc::new(snapshot.clone()); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - - let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-mixed", &Goal::new("review the PRs on acme/thing")) - .await - .expect("run"); - assert_eq!(finished.status, EpisodeStatus::Satisfied); - - // The episode landed in sqlite; the learned procedure is waiting in the - // snapshot for a flush into the vault. - assert!(ledger.episode("ep-mixed").await.expect("read").is_some()); - assert_eq!(snapshot.pending(), 1, "the procedure it learned"); - snapshot.flush(&vault).await.expect("flush"); - assert_eq!(vault.load().await.expect("load").len(), 1); - - let _ = std::fs::remove_dir_all(&dir); -} - -#[tokio::test] -async fn a_lesson_put_in_front_of_a_planner_is_scored_against_what_happened() { - // `applied` is the denominator of a lesson's help rate, and nothing was - // moving it: `score_lesson` had one caller, the corroboration loop, which - // moves both counters together. So every lesson read 0/0 or n/n, the rate - // carried no information, and every ordering built on it was inert. - use tinyflows_adaptive::ledger::{Lesson, LessonKind}; - - let llm = succeeding(parameterised(), true); - let caps = Capabilities { - llm: llm.clone(), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let id = ledger - .promote( - &Lesson { - id: String::new(), - kind: LessonKind::Strategy, - trigger: "a report that must cite figures".into(), - mechanism: String::new(), - claim: "read them from the source".into(), - applied: 0, - helped: 0, - scope_key: None, - }, - &[], - ) - .await - .expect("promote"); - - let store = store("scored"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-scored", &Goal::new("review the PRs on acme/thing")) - .await - .expect("run"); - - let back = ledger - .lessons(None) - .await - .expect("lessons") - .into_iter() - .find(|l| l.id == id) - .expect("still there"); - assert_eq!(back.applied, 1, "it was shown to the planner"); - assert_eq!(back.helped, 1, "and the episode was satisfied"); -} - -#[tokio::test] -async fn a_lesson_shown_before_a_failure_moves_only_its_denominator() { - use tinyflows_adaptive::ledger::{Lesson, LessonKind}; - - let llm = authoring(); // its judge always says not-satisfied - let caps = caps_with(llm); - let ledger = MemoryLedger::new(); - let id = ledger - .promote( - &Lesson { - id: String::new(), - kind: LessonKind::Strategy, - trigger: "a class of task".into(), - mechanism: String::new(), - claim: "does not actually help".into(), - applied: 0, - helped: 0, - scope_key: None, - }, - &[], - ) - .await - .expect("promote"); - - let store = store("unhelpful"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-unhelpful", &Goal::new("write the weekly report")) - .await - .expect("run"); - - let back = ledger - .lessons(None) - .await - .expect("lessons") - .into_iter() - .find(|l| l.id == id) - .expect("still there"); - assert!( - back.applied >= 2, - "shown on every attempt: {}", - back.applied - ); - assert_eq!(back.helped, 0, "and it never helped"); -} - -// --------------------------------------------------------------------------- -// The success gate: a variant exists mid-episode, the device gets it after. -// --------------------------------------------------------------------------- - -/// Drives the whole repair story from a script: select the parent, fail it -/// with a node named, propose a fix, select the fix, and — depending on -/// `satisfied_on` — let it win or keep failing until the stall rule ends it. -struct RepairFlow { - judged: Mutex, - satisfied_on: usize, -} - -impl RepairFlow { - fn new(satisfied_on: usize) -> Arc { - Arc::new(Self { - judged: Mutex::new(0), - satisfied_on, - }) - } -} - -#[async_trait] -impl LlmProvider for RepairFlow { - async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { - let user = request["messages"][1]["content"] - .as_str() - .unwrap_or_default() - .to_string(); - Ok(match request["tier"].as_str().unwrap_or_default() { - "select" => { - // Variant ids are content-derived, so the script cannot know - // them ahead — it reads the listing it was shown, the way a - // real selector would. - let ids: Vec<&str> = user - .lines() - .filter_map(|line| line.trim().strip_prefix("- id: ")) - .collect(); - let chosen = ids - .iter() - .find(|id| id.contains("-fix-")) - .or_else(|| ids.first()); - json!({ "workflow_id": chosen, "why": "it matches", "inputs": {} }) - } - "judge" => { - let mut judged = self.judged.lock().expect("lock"); - *judged += 1; - if *judged >= self.satisfied_on { - json!({ "satisfied": true, "gap": "" }) - } else { - json!({ - "satisfied": false, "blocker": "goal_not_met", - "gap": "the summary never landed", - "attributed_to": "start", "advanced": false - }) - } - } - "repair" => json!({ - "ops": [{ "op": "update_node_config", "id": "start", - "config": { "note": "fixed" } }], - "why": "repointed the binding" - }), - "consolidate" => json!({ "lessons": [], "corroborate": [] }), - other => panic!("no `{other}` call belongs in this flow"), - }) - } -} - -fn permissive() -> Arc { - #[derive(Debug, Default)] - struct Permissive; - impl tinyflows::store::HostPolicy for Permissive {} - Arc::new(Permissive) -} - -#[tokio::test] -async fn the_device_receives_a_variant_only_after_the_goal_run_succeeds() { - use tinyflows_adaptive::workflows::compat::Layered; - use tinyflows_adaptive::workflows::conformance::record; - use tinyflows_adaptive::workflows::memory::MemoryVault; - use tinyflows_adaptive::workflows::{Snapshot, Vault}; - - // The device owns the original; our writable layer starts empty. - let device = Arc::new(MemoryVault::new()); - device.put(&record("pr-review")).await.expect("put"); - let ours = Arc::new(MemoryVault::new()); - let stacked = Layered::new( - vec![("device".into(), device.clone() as Arc)], - ours.clone(), - ); - - let snapshot = Snapshot::load(&stacked, permissive()).await.expect("load"); - let store: Arc = Arc::new(snapshot.clone()); - let caps = Capabilities { - llm: RepairFlow::new(2), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - - let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run("ep-gate", &Goal::new("summarise the open pull requests")) - .await - .expect("run"); - - assert_eq!(finished.status, EpisodeStatus::Satisfied); - assert_eq!( - finished.attempts, 2, - "the parent failed once and its variant closed the goal" - ); - - // Mid-episode the variant lived only in the snapshot. The gate is the - // host's one `if`, and it is open: - assert_eq!(snapshot.pending(), 1); - snapshot.flush(&stacked).await.expect("flush"); - - let landed = ours.load().await.expect("load"); - assert_eq!(landed.len(), 1); - assert!( - landed[0].id.starts_with("pr-review-fix-"), - "{}", - landed[0].id - ); - assert_eq!( - device.load().await.expect("load").len(), - 1, - "the parent's home holds exactly what it held before" - ); -} - -#[tokio::test] -async fn a_failed_goal_run_leaves_no_residue_anywhere_durable() { - use tinyflows_adaptive::workflows::compat::Layered; - use tinyflows_adaptive::workflows::conformance::record; - use tinyflows_adaptive::workflows::memory::MemoryVault; - use tinyflows_adaptive::workflows::{Snapshot, Vault}; - - let device = Arc::new(MemoryVault::new()); - device.put(&record("pr-review")).await.expect("put"); - let ours = Arc::new(MemoryVault::new()); - let stacked = Layered::new( - vec![("device".into(), device.clone() as Arc)], - ours.clone(), - ); - - let snapshot = Snapshot::load(&stacked, permissive()).await.expect("load"); - let store: Arc = Arc::new(snapshot.clone()); - let caps = Capabilities { - llm: RepairFlow::new(usize::MAX), // never satisfied; the stall ends it - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - - let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) - .run( - "ep-no-residue", - &Goal::new("summarise the open pull requests"), - ) - .await - .expect("run"); - - assert!(matches!(finished.status, EpisodeStatus::StoodDown(_))); - assert!( - snapshot.pending() >= 1, - "repairs were proposed and buffered along the way" - ); - - // The gate stays closed: no flush. The knowledge is not lost with the - // graphs — the ledger kept the trail, durably, on the server side. - assert!(ours.load().await.expect("load").is_empty()); - assert_eq!(device.load().await.expect("load").len(), 1); - assert!( - !ledger.rows("ep-no-residue").await.expect("rows").is_empty(), - "the attempts are on the record even though no graph was kept" - ); -} - -#[tokio::test] -async fn an_errand_answers_the_goal_without_leaving_a_procedure_behind() { - // The whole claim of the errand path, end to end: a goal with no procedure - // in it is answered in one turn, and the shelf is exactly as it was. - struct Triage { - tiers: Mutex>, - } - #[async_trait] - impl LlmProvider for Triage { - async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { - let tier = request["tier"].as_str().unwrap_or_default().to_string(); - self.tiers.lock().expect("lock").push(tier.clone()); - Ok(match tier.as_str() { - "select" => json!({ - "workflow_id": null, "errand": true, - "why": "one turn of work, no procedure in it" - }), - "judge" => json!({ - "satisfied": true, "blocker": "", "gap": "", "advanced": true - }), - // Reached only if the loop wrongly authored or consolidated — - // both are asserted absent below. - _ => json!({ "why": "should not be asked", "inputs": {}, "steps": [] }), - }) - } - } - - let provider = Arc::new(Triage { - tiers: Mutex::new(Vec::new()), - }); - let caps = Capabilities { - llm: provider.clone(), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let store = store("errand"); - // A workflow on the shelf, so `select` is genuinely asked rather than - // short-circuited — this test is about the answer, not about the cold-store - // path, which `select`'s own tests cover. - let seeded = store.list().expect("list").len(); - - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - let engine = Loop { - ledger: &ledger, - store: &store, - caps: &caps, - facts: &HostFacts::unknown(), - runner: &runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - }; - - let finished = engine - .run( - "ep-errand", - &Goal::new("how much disk is this directory using"), - ) - .await - .expect("the episode runs"); - - assert_eq!(finished.status, EpisodeStatus::Satisfied); - assert_eq!(finished.attempts, 1, "one turn, not a retry loop"); - - let rows = ledger.rows("ep-errand").await.expect("rows"); - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].approach_sig, "errand"); - assert!( - rows[0].workflow_id.is_none(), - "an errand scores no procedure, because there is none" - ); - - // The point of the whole path: nothing filed. A one-off on the shelf is - // worse than useless — it dilutes every later selection with a row that - // matches once and can never match again. - assert_eq!( - store.list().expect("list").len(), - seeded, - "an errand must not be kept" - ); - // Asserted on the *calls*, not on the result. An empty lesson list is also - // what a consolidator that ran and found nothing returns, so the weaker - // assertion would pass with the gate removed entirely. - let tiers = provider.tiers.lock().expect("lock").clone(); - assert!( - !tiers.iter().any(|t| t == "consolidate"), - "a plain errand must not pay a consolidation call: {tiers:?}" - ); - assert!( - !tiers.iter().any(|t| t == "author"), - "nor an authoring one: {tiers:?}" - ); - assert!(finished.lessons.is_empty()); -} - -#[tokio::test] -async fn a_failed_errand_escalates_to_authoring_instead_of_repeating_itself() { - // The guard that stops the cheap path becoming a trap. If one turn did not - // do it, the goal was never an errand — and a model that keeps saying it is - // must not be able to spend the whole budget on identical single turns. - struct Insistent { - seen: Mutex>, - } - #[async_trait] - impl LlmProvider for Insistent { - async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { - let tier = request["tier"].as_str().unwrap_or_default().to_string(); - self.seen.lock().expect("lock").push(tier.clone()); - Ok(match tier.as_str() { - // Always insists, every attempt. - "select" => json!({ "workflow_id": null, "errand": true, "why": "trivial" }), - "judge" => json!({ - "satisfied": false, "blocker": "goal_not_met", - "gap": "it did not finish", "advanced": false - }), - "consolidate" => json!({ "lessons": [], "corroborate": [] }), - _ => json!({ - "why": "a real plan", - "inputs": {}, - "steps": [{ "id": "attempt", "run": "echo attempt-done" }], - }), - }) - } - } - let provider = Arc::new(Insistent { - seen: Mutex::new(Vec::new()), - }); - let caps = Capabilities { - llm: provider.clone(), - ..mock_capabilities() - }; - let ledger = MemoryLedger::new(); - let store = store("errand-escalate"); - let runner = Local { - caps: &caps, - workspace: &Unobserved, - }; - let engine = Loop { - ledger: &ledger, - store: &store, - caps: &caps, - facts: &HostFacts::unknown(), - runner: &runner, - clock: &Frozen, - budget: Default::default(), - conn: None, - }; - - let goal = Goal::new("do something that only looks trivial"); - engine.attempt("ep-esc", &goal).await.expect("attempt one"); - engine.attempt("ep-esc", &goal).await.expect("attempt two"); - - let rows = ledger.rows("ep-esc").await.expect("rows"); - assert_eq!(rows.len(), 2); - assert_eq!(rows[0].approach_sig, "errand"); - assert!( - rows[1].approach_sig.starts_with("authored:"), - "the second attempt must be a real plan, got {}", - rows[1].approach_sig - ); -} diff --git a/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs b/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs new file mode 100644 index 00000000..a62b2009 --- /dev/null +++ b/crates/tinyflows-adaptive/tests/driver/driver_part_01_tests.rs @@ -0,0 +1,287 @@ +#[tokio::test] +async fn the_model_can_still_refuse_a_graph_the_gate_let_through() { + // Parameterised and reusable-looking, but only meaningful for the one goal + // it was written for. The gate cannot see that; a reader can. + let llm = succeeding(parameterised(), false); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("refused"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-refused", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + + assert!(store.list().expect("list").is_empty()); +} + +#[tokio::test] +async fn the_next_episode_selects_what_the_last_one_learned() { + // The whole point, end to end. Episode one finds a cold store and authors; + // episode two finds the procedure episode one filed. + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("acquire"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-first", &Goal::new("review the PRs on acme/thing")) + .await + .expect("first"); + + let learned = store.list().expect("list")[0].id.clone(); + llm.seen.lock().expect("lock").clear(); + + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-second", &Goal::new("review the PRs on other/repo")) + .await + .expect("second"); + + // The selector was offered it, with the evidence from episode one. + let offered = llm.seen.lock().expect("lock")[0]["messages"][1]["content"] + .as_str() + .unwrap_or_default() + .to_string(); + assert!(offered.contains(&learned), "{offered}"); + assert!( + offered.contains("run 1×, satisfied 1×"), + "carrying what it earned: {offered}" + ); +} + +// --------------------------------------------------------------------------- +// The two stores are independent: any backend beside any other. +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn a_ledger_and_a_vault_of_different_kinds_drive_the_same_loop() { + // `Ledger` and `Vault` are separate traits with separate handles, so the + // host mixes them freely — sqlite ledger beside a Mongo vault, or either + // beside memory. Nothing in the loop knows which it got. + use tinyflows_adaptive::ledger::sqlite::SqliteLedger; + use tinyflows_adaptive::workflows::{Snapshot, Vault, memory::MemoryVault}; + + let dir = std::env::temp_dir().join(format!("adaptive-mixed-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + + // Durable ledger, ephemeral vault. A deliberately silly pairing, chosen + // because if this compiles and runs then every sensible one does. + let ledger = SqliteLedger::open(dir.join("ledger.db")).expect("ledger"); + let vault = MemoryVault::new(); + + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let policy: Arc = { + #[derive(Debug, Default)] + struct Permissive; + impl tinyflows::store::HostPolicy for Permissive {} + Arc::new(Permissive) + }; + let snapshot = Snapshot::load(&vault, policy).await.expect("snapshot"); + let store: Arc = Arc::new(snapshot.clone()); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-mixed", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + assert_eq!(finished.status, EpisodeStatus::Satisfied); + + // The episode landed in sqlite; the learned procedure is waiting in the + // snapshot for a flush into the vault. + assert!(ledger.episode("ep-mixed").await.expect("read").is_some()); + assert_eq!(snapshot.pending(), 1, "the procedure it learned"); + snapshot.flush(&vault).await.expect("flush"); + assert_eq!(vault.load().await.expect("load").len(), 1); + + let _ = std::fs::remove_dir_all(&dir); +} + +#[tokio::test] +async fn a_lesson_put_in_front_of_a_planner_is_scored_against_what_happened() { + // `applied` is the denominator of a lesson's help rate, and nothing was + // moving it: `score_lesson` had one caller, the corroboration loop, which + // moves both counters together. So every lesson read 0/0 or n/n, the rate + // carried no information, and every ordering built on it was inert. + use tinyflows_adaptive::ledger::{Lesson, LessonKind}; + + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let id = ledger + .promote( + &Lesson { + id: String::new(), + kind: LessonKind::Strategy, + trigger: "a report that must cite figures".into(), + mechanism: String::new(), + claim: "read them from the source".into(), + applied: 0, + helped: 0, + scope_key: None, + }, + &[], + ) + .await + .expect("promote"); + + let store = store("scored"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-scored", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + + let back = ledger + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == id) + .expect("still there"); + assert_eq!(back.applied, 1, "it was shown to the planner"); + assert_eq!(back.helped, 1, "and the episode was satisfied"); +} + +#[tokio::test] +async fn a_lesson_shown_before_a_failure_moves_only_its_denominator() { + use tinyflows_adaptive::ledger::{Lesson, LessonKind}; + + let llm = authoring(); // its judge always says not-satisfied + let caps = caps_with(llm); + let ledger = MemoryLedger::new(); + let id = ledger + .promote( + &Lesson { + id: String::new(), + kind: LessonKind::Strategy, + trigger: "a class of task".into(), + mechanism: String::new(), + claim: "does not actually help".into(), + applied: 0, + helped: 0, + scope_key: None, + }, + &[], + ) + .await + .expect("promote"); + + let store = store("unhelpful"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-unhelpful", &Goal::new("write the weekly report")) + .await + .expect("run"); + + let back = ledger + .lessons(None) + .await + .expect("lessons") + .into_iter() + .find(|l| l.id == id) + .expect("still there"); + assert!( + back.applied >= 2, + "shown on every attempt: {}", + back.applied + ); + assert_eq!(back.helped, 0, "and it never helped"); +} + +// --------------------------------------------------------------------------- +// The success gate: a variant exists mid-episode, the device gets it after. +// --------------------------------------------------------------------------- + +/// Drives the whole repair story from a script: select the parent, fail it +/// with a node named, propose a fix, select the fix, and — depending on +/// `satisfied_on` — let it win or keep failing until the stall rule ends it. +struct RepairFlow { + judged: Mutex, + satisfied_on: usize, +} + +impl RepairFlow { + fn new(satisfied_on: usize) -> Arc { + Arc::new(Self { + judged: Mutex::new(0), + satisfied_on, + }) + } +} + +#[async_trait] +impl LlmProvider for RepairFlow { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let user = request["messages"][1]["content"] + .as_str() + .unwrap_or_default() + .to_string(); + Ok(match request["tier"].as_str().unwrap_or_default() { + "select" => { + // Variant ids are content-derived, so the script cannot know + // them ahead — it reads the listing it was shown, the way a + // real selector would. + let ids: Vec<&str> = user + .lines() + .filter_map(|line| line.trim().strip_prefix("- id: ")) + .collect(); + let chosen = ids + .iter() + .find(|id| id.contains("-fix-")) + .or_else(|| ids.first()); + json!({ "workflow_id": chosen, "why": "it matches", "inputs": {} }) + } + "judge" => { + let mut judged = self.judged.lock().expect("lock"); + *judged += 1; + if *judged >= self.satisfied_on { + json!({ "satisfied": true, "gap": "" }) + } else { + json!({ + "satisfied": false, "blocker": "goal_not_met", + "gap": "the summary never landed", + "attributed_to": "start", "advanced": false + }) + } + } + "repair" => json!({ + "ops": [{ "op": "update_node_config", "id": "start", + "config": { "note": "fixed" } }], + "why": "repointed the binding" + }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + other => panic!("no `{other}` call belongs in this flow"), + }) + } +} + diff --git a/crates/tinyflows-adaptive/tests/driver/driver_part_02_tests.rs b/crates/tinyflows-adaptive/tests/driver/driver_part_02_tests.rs new file mode 100644 index 00000000..d83d9360 --- /dev/null +++ b/crates/tinyflows-adaptive/tests/driver/driver_part_02_tests.rs @@ -0,0 +1,280 @@ +fn permissive() -> Arc { + #[derive(Debug, Default)] + struct Permissive; + impl tinyflows::store::HostPolicy for Permissive {} + Arc::new(Permissive) +} + +#[tokio::test] +async fn the_device_receives_a_variant_only_after_the_goal_run_succeeds() { + use tinyflows_adaptive::workflows::compat::Layered; + use tinyflows_adaptive::workflows::conformance::record; + use tinyflows_adaptive::workflows::memory::MemoryVault; + use tinyflows_adaptive::workflows::{Snapshot, Vault}; + + // The device owns the original; our writable layer starts empty. + let device = Arc::new(MemoryVault::new()); + device.put(&record("pr-review")).await.expect("put"); + let ours = Arc::new(MemoryVault::new()); + let stacked = Layered::new( + vec![("device".into(), device.clone() as Arc)], + ours.clone(), + ); + + let snapshot = Snapshot::load(&stacked, permissive()).await.expect("load"); + let store: Arc = Arc::new(snapshot.clone()); + let caps = Capabilities { + llm: RepairFlow::new(2), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-gate", &Goal::new("summarise the open pull requests")) + .await + .expect("run"); + + assert_eq!(finished.status, EpisodeStatus::Satisfied); + assert_eq!( + finished.attempts, 2, + "the parent failed once and its variant closed the goal" + ); + + // Mid-episode the variant lived only in the snapshot. The gate is the + // host's one `if`, and it is open: + assert_eq!(snapshot.pending(), 1); + snapshot.flush(&stacked).await.expect("flush"); + + let landed = ours.load().await.expect("load"); + assert_eq!(landed.len(), 1); + assert!( + landed[0].id.starts_with("pr-review-fix-"), + "{}", + landed[0].id + ); + assert_eq!( + device.load().await.expect("load").len(), + 1, + "the parent's home holds exactly what it held before" + ); +} + +#[tokio::test] +async fn a_failed_goal_run_leaves_no_residue_anywhere_durable() { + use tinyflows_adaptive::workflows::compat::Layered; + use tinyflows_adaptive::workflows::conformance::record; + use tinyflows_adaptive::workflows::memory::MemoryVault; + use tinyflows_adaptive::workflows::{Snapshot, Vault}; + + let device = Arc::new(MemoryVault::new()); + device.put(&record("pr-review")).await.expect("put"); + let ours = Arc::new(MemoryVault::new()); + let stacked = Layered::new( + vec![("device".into(), device.clone() as Arc)], + ours.clone(), + ); + + let snapshot = Snapshot::load(&stacked, permissive()).await.expect("load"); + let store: Arc = Arc::new(snapshot.clone()); + let caps = Capabilities { + llm: RepairFlow::new(usize::MAX), // never satisfied; the stall ends it + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run( + "ep-no-residue", + &Goal::new("summarise the open pull requests"), + ) + .await + .expect("run"); + + assert!(matches!(finished.status, EpisodeStatus::StoodDown(_))); + assert!( + snapshot.pending() >= 1, + "repairs were proposed and buffered along the way" + ); + + // The gate stays closed: no flush. The knowledge is not lost with the + // graphs — the ledger kept the trail, durably, on the server side. + assert!(ours.load().await.expect("load").is_empty()); + assert_eq!(device.load().await.expect("load").len(), 1); + assert!( + !ledger.rows("ep-no-residue").await.expect("rows").is_empty(), + "the attempts are on the record even though no graph was kept" + ); +} + +#[tokio::test] +async fn an_errand_answers_the_goal_without_leaving_a_procedure_behind() { + // The whole claim of the errand path, end to end: a goal with no procedure + // in it is answered in one turn, and the shelf is exactly as it was. + struct Triage { + tiers: Mutex>, + } + #[async_trait] + impl LlmProvider for Triage { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let tier = request["tier"].as_str().unwrap_or_default().to_string(); + self.tiers.lock().expect("lock").push(tier.clone()); + Ok(match tier.as_str() { + "select" => json!({ + "workflow_id": null, "errand": true, + "why": "one turn of work, no procedure in it" + }), + "judge" => json!({ + "satisfied": true, "blocker": "", "gap": "", "advanced": true + }), + // Reached only if the loop wrongly authored or consolidated — + // both are asserted absent below. + _ => json!({ "why": "should not be asked", "inputs": {}, "steps": [] }), + }) + } + } + + let provider = Arc::new(Triage { + tiers: Mutex::new(Vec::new()), + }); + let caps = Capabilities { + llm: provider.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("errand"); + // A workflow on the shelf, so `select` is genuinely asked rather than + // short-circuited — this test is about the answer, not about the cold-store + // path, which `select`'s own tests cover. + let seeded = store.list().expect("list").len(); + + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + let finished = engine + .run( + "ep-errand", + &Goal::new("how much disk is this directory using"), + ) + .await + .expect("the episode runs"); + + assert_eq!(finished.status, EpisodeStatus::Satisfied); + assert_eq!(finished.attempts, 1, "one turn, not a retry loop"); + + let rows = ledger.rows("ep-errand").await.expect("rows"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].approach_sig, "errand"); + assert!( + rows[0].workflow_id.is_none(), + "an errand scores no procedure, because there is none" + ); + + // The point of the whole path: nothing filed. A one-off on the shelf is + // worse than useless — it dilutes every later selection with a row that + // matches once and can never match again. + assert_eq!( + store.list().expect("list").len(), + seeded, + "an errand must not be kept" + ); + // Asserted on the *calls*, not on the result. An empty lesson list is also + // what a consolidator that ran and found nothing returns, so the weaker + // assertion would pass with the gate removed entirely. + let tiers = provider.tiers.lock().expect("lock").clone(); + assert!( + !tiers.iter().any(|t| t == "consolidate"), + "a plain errand must not pay a consolidation call: {tiers:?}" + ); + assert!( + !tiers.iter().any(|t| t == "author"), + "nor an authoring one: {tiers:?}" + ); + assert!(finished.lessons.is_empty()); +} + +#[tokio::test] +async fn a_failed_errand_escalates_to_authoring_instead_of_repeating_itself() { + // The guard that stops the cheap path becoming a trap. If one turn did not + // do it, the goal was never an errand — and a model that keeps saying it is + // must not be able to spend the whole budget on identical single turns. + struct Insistent { + seen: Mutex>, + } + #[async_trait] + impl LlmProvider for Insistent { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + let tier = request["tier"].as_str().unwrap_or_default().to_string(); + self.seen.lock().expect("lock").push(tier.clone()); + Ok(match tier.as_str() { + // Always insists, every attempt. + "select" => json!({ "workflow_id": null, "errand": true, "why": "trivial" }), + "judge" => json!({ + "satisfied": false, "blocker": "goal_not_met", + "gap": "it did not finish", "advanced": false + }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + _ => json!({ + "why": "a real plan", + "inputs": {}, + "steps": [{ "id": "attempt", "run": "echo attempt-done" }], + }), + }) + } + } + let provider = Arc::new(Insistent { + seen: Mutex::new(Vec::new()), + }); + let caps = Capabilities { + llm: provider.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("errand-escalate"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + let goal = Goal::new("do something that only looks trivial"); + engine.attempt("ep-esc", &goal).await.expect("attempt one"); + engine.attempt("ep-esc", &goal).await.expect("attempt two"); + + let rows = ledger.rows("ep-esc").await.expect("rows"); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].approach_sig, "errand"); + assert!( + rows[1].approach_sig.starts_with("authored:"), + "the second attempt must be a real plan, got {}", + rows[1].approach_sig + ); +} diff --git a/crates/tinyflows-adaptive/tests/driver_tests.rs b/crates/tinyflows-adaptive/tests/driver_tests.rs new file mode 100644 index 00000000..a5b96d47 --- /dev/null +++ b/crates/tinyflows-adaptive/tests/driver_tests.rs @@ -0,0 +1,456 @@ +//! One instance, many goal runs, and an episode that outlives the process. +//! +//! These test the claim the `driver` module is built on, because it is the one +//! that is expensive to be wrong about: a `Loop` is per **tenant** and a goal +//! run is an **episode id**, so the same instance drives many episodes at once +//! and any instance can pick up an episode any other one started. + +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use serde_json::{Value, json}; +use tinyflows::caps::mock::mock_capabilities; +use tinyflows::caps::{Capabilities, LlmProvider}; +use tinyflows::error::Result as EngineResult; +use tinyflows::store::{FileWorkflowStore, WorkflowStore}; +use tinyflows_adaptive::contracts::Goal; +use tinyflows_adaptive::driver::{Clock, Loop}; +use tinyflows_adaptive::execute::{Local, Unobserved}; +use tinyflows_adaptive::host::HostFacts; +use tinyflows_adaptive::ledger::{EpisodeStatus, Ledger, Page, memory::MemoryLedger}; + +struct Frozen; +impl Clock for Frozen { + fn now(&self) -> String { + "2026-01-01T00:00:00Z".to_string() + } +} + +/// Answers every authoring call the same way, and keeps every request so the +/// tier can be read back off the wire. +struct Always { + reply: Value, + seen: Mutex>, +} + +impl Always { + fn new(reply: Value) -> Arc { + Arc::new(Self { + reply, + seen: Mutex::new(Vec::new()), + }) + } + fn tiers(&self) -> Vec { + self.seen + .lock() + .expect("lock") + .iter() + .map(|r| r["tier"].as_str().unwrap_or("(absent)").to_string()) + .collect() + } +} + +#[async_trait] +impl LlmProvider for Always { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + self.seen.lock().expect("lock").push(request.clone()); + // The tier says which job is asking, so one double can answer them all. + Ok(match request["tier"].as_str().unwrap_or_default() { + "judge" => json!({ + "satisfied": false, "blocker": "goal_not_met", + "gap": "the report has no numbers in it", "advanced": false + }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + "select" => json!({ "workflow_id": null, "why": "nothing fits" }), + _ => self.reply.clone(), + }) + } +} + +fn caps_with(llm: Arc) -> Capabilities { + Capabilities { + llm, + ..mock_capabilities() + } +} + +fn store(tag: &str) -> Arc { + let root = std::env::temp_dir().join(format!("adaptive-driver-{}-{tag}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(root.join("workflows")).expect("temp dir"); + Arc::new(FileWorkflowStore::new( + vec![root.join("workflows")], + root.join("runs"), + )) +} + +fn authoring() -> Arc { + Always::new(json!({ + "why": "nothing stored fits", + "inputs": {}, + "steps": [{ "id": "attempt", "run": "echo attempt-done" }], + })) +} + +#[tokio::test] +async fn one_instance_drives_two_goal_runs_with_independent_counters() { + // The claim the split rests on: the instance holds no per-episode state, so + // two episodes interleaved through it cannot contaminate each other. + let llm = authoring(); + let caps = caps_with(llm); + let ledger = MemoryLedger::new(); + let store = store("two"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + let goal = Goal::new("write the weekly report"); + engine.attempt("ep-a", &goal).await.expect("a1"); + engine.attempt("ep-b", &goal).await.expect("b1"); + engine.attempt("ep-a", &goal).await.expect("a2"); + + let a = ledger.episode("ep-a").await.expect("read").expect("exists"); + let b = ledger.episode("ep-b").await.expect("read").expect("exists"); + assert_eq!(a.attempt, 2); + assert_eq!(b.attempt, 1, "b is untouched by a's two passes"); + assert_eq!(a.stalled, 2, "neither of a's attempts advanced"); + assert_eq!(b.stalled, 1); + + assert_eq!(ledger.rows("ep-a").await.expect("rows").len(), 2); + assert_eq!(ledger.rows("ep-b").await.expect("rows").len(), 1); +} + +#[tokio::test] +async fn a_second_instance_picks_up_an_episode_the_first_one_started() { + // Kill the process mid-episode. Everything the loop needs is in the ledger, + // so a fresh instance continues the numbering rather than starting over + // with a trail that says it has already tried twice. + let ledger = MemoryLedger::new(); + let store = store("resume"); + let goal = Goal::new("write the weekly report"); + + { + let caps = caps_with(authoring()); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let first = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + first.attempt("ep-resume", &goal).await.expect("1"); + first.attempt("ep-resume", &goal).await.expect("2"); + } // the instance goes away, as a deploy would take it + + let unfinished = ledger.episodes(true, Page::ALL).await.expect("episodes"); + assert_eq!(unfinished.len(), 1, "the recovery list a boot reads"); + let recovered = &unfinished[0]; + assert_eq!(recovered.id, "ep-resume"); + assert_eq!(recovered.goal.text, "write the weekly report"); + assert_eq!(recovered.stalled, 2); + + let caps = caps_with(authoring()); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let second = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + let closed = second + .attempt(&recovered.id, &recovered.goal) + .await + .expect("3"); + + assert_eq!( + ledger + .episode("ep-resume") + .await + .expect("read") + .expect("exists") + .attempt, + 3, + "it continued rather than restarting at one" + ); + assert_eq!( + closed.stalled, 3, + "the stall count survived the process that was counting it" + ); +} + +#[tokio::test] +async fn every_inference_request_says_which_job_is_asking() { + // Without this a host cannot route judging and selecting to different + // models, which is the whole point of the tier. + let llm = authoring(); + let caps = caps_with(llm.clone()); + let ledger = MemoryLedger::new(); + let store = store("tiers"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + engine + .attempt("ep-tiers", &Goal::new("write the weekly report")) + .await + .expect("attempt"); + + let tiers = llm.tiers(); + assert!(!tiers.iter().any(|t| t == "(absent)"), "{tiers:?}"); + assert!(tiers.contains(&"author".to_string()), "{tiers:?}"); + assert!(tiers.contains(&"judge".to_string()), "{tiers:?}"); +} + +#[tokio::test] +async fn a_run_drives_to_a_stand_down_and_consolidates_once() { + // The judge never says satisfied and nothing advances, so the stall rule + // ends it. `run` must stop on its own rather than needing a bound of its + // own alongside the one `close` already applies. + let llm = authoring(); + let caps = caps_with(llm.clone()); + let ledger = MemoryLedger::new(); + let store = store("drive"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + let engine = Loop { + ledger: &ledger, + store: &store, + caps: &caps, + facts: &HostFacts::unknown(), + runner: &runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + }; + + let finished = engine + .run("ep-drive", &Goal::new("write the weekly report")) + .await + .expect("run"); + + match &finished.status { + EpisodeStatus::StoodDown(reason) => assert!(reason.contains("no progress"), "{reason}"), + other => panic!("expected a stand-down, got {other:?}"), + } + assert!(finished.attempts >= 2, "{finished:?}"); + assert!(finished.lessons.is_empty(), "nothing generalised"); + + // Consolidation is per episode, not per attempt. + assert_eq!( + llm.tiers().iter().filter(|t| *t == "consolidate").count(), + 1 + ); + + let record = ledger + .episode("ep-drive") + .await + .expect("read") + .expect("exists"); + assert!(matches!(record.status, EpisodeStatus::StoodDown(_))); + assert_ne!( + record.status, + EpisodeStatus::Running, + "a finished episode must leave the recovery list" + ); +} + +// --------------------------------------------------------------------------- +// The loop acquires a skill: authored, worked, kept, then selected. +// --------------------------------------------------------------------------- + +/// A graph parameterised by a declared input, which is what the authoring +/// prompt asks for and what makes a procedure worth keeping. +fn parameterised() -> Value { + json!({ + "why": "review", + "declared": [{ "name": "repo", "description": "the repository", "required": true }], + "inputs": { "repo": "acme/thing" }, + "steps": [{ + "id": "review", + "ask": "Review the open pull requests and summarise them directly." + }], + }) +} + +/// Authors `graph`, judges every run satisfied, and answers the naming call. +struct Succeeds { + authored: Value, + reusable: bool, + seen: Mutex>, +} + +#[async_trait] +impl LlmProvider for Succeeds { + async fn complete(&self, request: Value, _conn: Option<&str>) -> EngineResult { + self.seen.lock().expect("lock").push(request.clone()); + Ok(match request["tier"].as_str().unwrap_or_default() { + "judge" => json!({ "satisfied": true, "gap": "" }), + "consolidate" => json!({ "lessons": [], "corroborate": [] }), + "select" => json!({ "workflow_id": null, "why": "nothing fits yet" }), + "generalise" => json!({ + "name": "Review a repository's pull requests", + "description": "Reviews the open pull requests on a repository. Takes the repository as an input.", + "reusable": self.reusable, + }), + _ => self.authored.clone(), + }) + } +} + +fn succeeding(authored: Value, reusable: bool) -> Arc { + Arc::new(Succeeds { + authored, + reusable, + seen: Mutex::new(Vec::new()), + }) +} + +fn engine_over<'a>( + ledger: &'a dyn Ledger, + store: &'a Arc, + caps: &'a Capabilities, + runner: &'a Local<'a>, + facts: &'a HostFacts, +) -> Loop<'a> { + Loop { + ledger, + store, + caps, + facts, + runner, + clock: &Frozen, + budget: Default::default(), + conn: None, + } +} + +#[tokio::test] +async fn a_graph_that_was_authored_and_worked_becomes_a_stored_procedure() { + // The headline claim: "selects a stored workflow or authors one" is only + // half true if authoring never becomes stored, because then the catalogue + // holds exactly what a person put there and the loop never acquires a skill. + let llm = succeeding(parameterised(), true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("keep"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + assert!(store.list().expect("list").is_empty(), "a cold store"); + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-learn", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + assert_eq!(finished.status, EpisodeStatus::Satisfied); + + let listed = store.list().expect("list"); + assert_eq!(listed.len(), 1, "the procedure was filed: {listed:?}"); + assert!(listed[0].id.starts_with("learned-"), "{}", listed[0].id); + assert!( + listed[0].description.contains("a repository"), + "described as a class, not as the goal: {}", + listed[0].description + ); + + // Scored from the run that earned it — entering at 0/0 would be + // indistinguishable from a procedure nobody has ever run. + let score = ledger.workflow_score(&listed[0].id).await.expect("score"); + assert_eq!((score.applied, score.helped), (1, 1)); +} + +#[tokio::test] +async fn a_graph_that_pasted_its_inputs_is_not_kept() { + // Same run, same success — but the goal's specifics are welded into a + // step, so it matches one task and never another. No model is asked. + // + // The paste sits in a `run` script, not an ask: the intake gate refuses + // ask-pastes outright now, and this test is about the layer BEHIND it — + // keep's own refusal, which still guards every path intake cannot see. + let mut baked = parameterised(); + baked["steps"] = json!([ + { "id": "review", "run": "gh pr list -R acme/thing" }, + { "id": "report", "ask": "Summarise the review output.", "reads": ["review"] } + ]); + + let llm = succeeding(baked, true); + let caps = Capabilities { + llm: llm.clone(), + ..mock_capabilities() + }; + let ledger = MemoryLedger::new(); + let store = store("baked"); + let runner = Local { + caps: &caps, + workspace: &Unobserved, + }; + + let finished = engine_over(&ledger, &store, &caps, &runner, &HostFacts::unknown()) + .run("ep-baked", &Goal::new("review the PRs on acme/thing")) + .await + .expect("run"); + + assert_eq!(finished.status, EpisodeStatus::Satisfied, "it still worked"); + assert!( + store.list().expect("list").is_empty(), + "but it is a one-off" + ); + + let tiers: Vec = llm + .seen + .lock() + .expect("lock") + .iter() + .map(|r| r["tier"].as_str().unwrap_or_default().to_string()) + .collect(); + assert!( + !tiers.iter().any(|t| t == "generalise"), + "the mechanical gate settled it without paying for an opinion: {tiers:?}" + ); +} + +include!("driver/driver_part_01_tests.rs"); +include!("driver/driver_part_02_tests.rs"); diff --git a/crates/tinyflows-adaptive/tests/intake/intake_part_01_tests.rs b/crates/tinyflows-adaptive/tests/intake/intake_part_01_tests.rs new file mode 100644 index 00000000..6870c13a --- /dev/null +++ b/crates/tinyflows-adaptive/tests/intake/intake_part_01_tests.rs @@ -0,0 +1,213 @@ +#[tokio::test] +async fn an_authored_graph_that_does_not_validate_is_an_error_not_a_return_value() { + // Handing it back would turn an authoring mistake into a run-time failure + // that reads like the work failing. The author retries with the refusal + // fed back, so the script holds a model that stays wrong for every round. + let broken = json!({ + "why": "forgot the steps", + "inputs": {}, + }); + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + broken.clone(), + broken.clone(), + broken, + ])); + let caps = caps_with(llm); + let (store, _root) = empty_store("7"); + let ledger = MemoryLedger::new(); + + let err = decide( + &Goal::new("anything"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect_err("an invalid graph must not leave intake"); + assert!(err.to_string().contains("invalid"), "{err}"); +} + +#[tokio::test] +async fn a_disabled_workflow_is_never_offered() { + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + authored_reply("written", None), + ])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("8"); + let mut off = stored("switched-off", "would have matched", None); + off.enabled = false; + store.save(&off).expect("save"); + let ledger = MemoryLedger::new(); + + decide( + &Goal::new("do the thing"), + "ep1", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + assert!( + !llm.prompts()[0].contains("switched-off"), + "offering a disabled workflow invites a choice that cannot be honoured: {}", + llm.prompts()[0] + ); +} + +#[tokio::test] +async fn a_graph_naming_a_worker_this_host_lacks_is_refused_before_it_runs() { + // The whole point of collecting host facts. Without this the graph saves + // cleanly, validates cleanly, and fails at run time — usually overnight, + // to nobody watching. + // + // Three copies: the author feeds refusals back, and this model never + // learns that the worker does not exist. + let insistent = json!({ + "why": "needs an agent", + "inputs": {}, + "steps": [{ "id": "work", "ask": "do the thing", "worker": "desktop" }], + }); + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + insistent.clone(), + insistent.clone(), + insistent, + ])); + let caps = caps_with(llm); + let (store, _root) = empty_store("gated"); + let ledger = MemoryLedger::new(); + + let facts = HostFacts { + workers: vec!["laptop".into(), "ci".into()], + default_worker: Some("laptop".into()), + ..HostFacts::unknown() + }; + + let err = decide( + &Goal::new("do the thing"), + "ep1", + &store, + &ledger, + &facts, + &caps, + None, + ) + .await + .expect_err("a worker this host lacks must not reach the engine"); + + assert!( + err.to_string().contains("desktop"), + "the error names it: {err}" + ); + assert!( + err.to_string().contains("laptop"), + "and offers the alternatives: {err}" + ); +} + +#[tokio::test] +async fn the_authoring_prompt_carries_what_the_host_permits() { + // The facts below say agent work must name a worker, so the reply's ask + // step names one — the same gate this test exists to see rendered. + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + json!({ + "why": "fine", + "inputs": {}, + "steps": [{ "id": "work", "ask": "Do it directly.", "worker": "laptop" }], + }), + ])); + let caps = caps_with(llm.clone()); + let (store, _root) = empty_store("facts-rendered"); + let ledger = MemoryLedger::new(); + + let facts = HostFacts { + workers: vec!["laptop".into()], + default_worker: None, + allow_code: Some(false), + notes: vec!["Only manual triggers fire here.".into()], + ..HostFacts::unknown() + }; + + decide( + &Goal::new("anything"), + "ep1", + &store, + &ledger, + &facts, + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &authoring_prompt(&llm); + assert!(prompt.contains("What this host permits"), "{prompt}"); + assert!(prompt.contains("every agent node must name config.agent_ref")); + assert!(prompt.contains("Only manual triggers fire here.")); +} + +// --------------------------------------------------------------------------- +// Promotion: a repaired family is one row, and score decides which. +// --------------------------------------------------------------------------- + +/// A parent and one variant, both stored and linked, with scores applied. +async fn repaired_family( + tag: &str, + parent: (u32, u32), + variant: (u32, u32), +) -> (FileWorkflowStore, MemoryLedger, std::path::PathBuf) { + let (store, root) = empty_store(tag); + store + .save(&stored("weekly", "writes the weekly report", None)) + .expect("save"); + store + .save(&stored( + "weekly-fix-1", + "writes the weekly report, with the binding corrected", + None, + )) + .expect("save"); + + let ledger = MemoryLedger::new(); + ledger + .link_variant("weekly", "weekly-fix-1") + .await + .expect("link"); + for (id, (applied, helped)) in [("weekly", parent), ("weekly-fix-1", variant)] { + for n in 0..applied { + ledger.score_workflow(id, n < helped).await.expect("score"); + } + } + (store, ledger, root) +} + +/// What the selector was actually shown. +async fn offered(store: &FileWorkflowStore, ledger: &MemoryLedger) -> String { + let llm = std::sync::Arc::new(Scripted::new(vec![ + json!({"workflow_id": "none"}), + authored_reply("fallback", None), + ])); + let caps = caps_with(llm.clone()); + let _ = decide( + &Goal::new("write the weekly report"), + "ep-promo", + store, + ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await; + llm.prompts().first().cloned().unwrap_or_default() +} + diff --git a/crates/tinyflows-adaptive/tests/intake/intake_part_02_tests.rs b/crates/tinyflows-adaptive/tests/intake/intake_part_02_tests.rs new file mode 100644 index 00000000..158c4f5d --- /dev/null +++ b/crates/tinyflows-adaptive/tests/intake/intake_part_02_tests.rs @@ -0,0 +1,281 @@ +#[tokio::test] +async fn a_repaired_family_is_offered_as_one_row_not_two() { + // Two near-identical graphs whose descriptions differ by a clause is not a + // choice, it is noise. + let (store, ledger, _root) = repaired_family("promo-1", (40, 40), (0, 0)).await; + let shown = offered(&store, &ledger).await; + let rows = shown.matches("weekly").count(); + assert!(rows > 0, "the family must be offered at all: {shown}"); + assert!( + !shown.contains("weekly-fix-1"), + "an unproven variant must not appear beside its proven parent: {shown}" + ); +} + +#[tokio::test] +async fn a_fresh_variant_does_not_displace_a_proven_parent() { + let (store, ledger, _root) = repaired_family("promo-2", (40, 40), (0, 0)).await; + let shown = offered(&store, &ledger).await; + assert!(shown.contains("weekly"), "{shown}"); + assert!(!shown.contains("weekly-fix-1"), "{shown}"); +} + +#[tokio::test] +async fn a_variant_that_has_proven_better_is_the_one_offered() { + // Promotion on score, not on having been written. + let (store, ledger, _root) = repaired_family("promo-3", (10, 5), (4, 4)).await; + let shown = offered(&store, &ledger).await; + assert!( + shown.contains("weekly-fix-1"), + "the better member must take the position: {shown}" + ); +} + +#[tokio::test] +async fn a_family_whose_champion_was_already_tried_still_offers_its_variant() { + // The case that matters most and is easiest to get wrong: this episode just + // failed with the parent, so the parent is excluded — and the variant + // exists *because* the parent fell short. Dropping the whole family would + // hide the one graph written for this exact situation. + let (store, ledger, _root) = repaired_family("promo-4", (40, 40), (0, 0)).await; + ledger + .append(&tinyflows_adaptive::ledger::LedgerRow { + id: String::new(), + episode: "ep-promo".into(), + attempt: 1, + approach_sig: "selected:weekly".into(), + approach_desc: "the champion".into(), + workflow_id: Some("weekly".into()), + outcome: "fell short".into(), + cause: String::new(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + }) + .await + .expect("append"); + + let shown = offered(&store, &ledger).await; + assert!( + shown.contains("weekly-fix-1"), + "the variant must survive its champion being excluded: {shown}" + ); +} + +// --------------------------------------------------------------------------- +// The retry edge: attempt four must not be attempt two in different words. +// --------------------------------------------------------------------------- + +async fn with_history(tag: &str) -> (FileWorkflowStore, MemoryLedger, std::path::PathBuf) { + let (store, root) = empty_store(tag); + let ledger = MemoryLedger::new(); + for (attempt, sig, desc, cause) in [ + ( + 1u32, + "authored:aaa", + "fetched the log and summarised it", + "no numbers in it", + ), + ( + 2, + "authored:bbb", + "asked an agent to write it from memory", + "it invented the figures", + ), + ] { + ledger + .append(&tinyflows_adaptive::ledger::LedgerRow { + id: String::new(), + episode: "ep-retry".into(), + attempt, + approach_sig: sig.into(), + approach_desc: desc.into(), + workflow_id: None, + outcome: "fell short".into(), + cause: cause.into(), + cost_usd: 0.0, + at: "2026-01-01T00:00:00Z".into(), + satisfied: false, + advanced: false, + }) + .await + .expect("append"); + } + (store, ledger, root) +} + +#[tokio::test] +async fn the_author_is_shown_what_this_episode_already_tried() { + // Without this the author writes attempt two's graph again, confidently, + // because nothing told it otherwise. The exclusion list only guards + // *selection*; authoring has no structural guard at all. + let (store, ledger, _root) = with_history("retry-1").await; + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + authored_reply("third-idea", None), + ])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-retry", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &authoring_prompt(&llm); + assert!(prompt.contains("Already tried this episode"), "{prompt}"); + assert!( + prompt.contains("asked an agent to write it from memory"), + "{prompt}" + ); + assert!(prompt.contains("it invented the figures"), "{prompt}"); + assert!(prompt.contains("DIFFERENT plan"), "{prompt}"); +} + +#[tokio::test] +async fn the_selector_is_shown_the_same_history_in_the_same_words() { + let (store, ledger, _root) = with_history("retry-2").await; + store + .save(&stored("weekly", "writes the weekly report", None)) + .expect("save"); + + let llm = std::sync::Arc::new(Scripted::new(vec![json!({ + "workflow_id": "weekly", + "why": "it does this", + "inputs": {}, + })])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-retry", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &llm.prompts()[0]; + assert!(prompt.contains("Already tried this episode"), "{prompt}"); + assert!(prompt.contains("no numbers in it"), "{prompt}"); +} + +#[tokio::test] +async fn lessons_from_other_episodes_reach_the_planner() { + // consolidate() was writing these and nothing was reading them — a + // knowledge store that costs money and returns nothing. + let (store, root) = empty_store("retry-3"); + let _ = root; + let ledger = MemoryLedger::new(); + ledger + .promote( + &tinyflows_adaptive::ledger::Lesson { + id: String::new(), + kind: tinyflows_adaptive::ledger::LessonKind::Constraint, + trigger: "a report that must cite figures".into(), + mechanism: "the model has no access to the numbers".into(), + claim: "read them from the source rather than asking an agent".into(), + applied: 0, + helped: 0, + scope_key: None, + }, + &[], + ) + .await + .expect("promote"); + + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + authored_reply("informed", None), + ])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-fresh", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + let prompt = &authoring_prompt(&llm); + assert!(prompt.contains("Learned from earlier episodes"), "{prompt}"); + assert!(prompt.contains("read them from the source"), "{prompt}"); +} + +#[tokio::test] +async fn a_first_attempt_is_told_nothing_it_would_have_to_ignore() { + // An empty history section is noise a model has to read past, and an + // empty "already tried" heading reads as a claim that something was. + let (store, _root) = empty_store("retry-4"); + let ledger = MemoryLedger::new(); + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + authored_reply("first", None), + ])); + let caps = caps_with(llm.clone()); + + decide( + &Goal::new("write the weekly report"), + "ep-first", + &store, + &ledger, + &HostFacts::unknown(), + &caps, + None, + ) + .await + .expect("decide"); + + // Every prompt, not just one: an empty heading is noise whichever planner + // reads it, and the triage call sees the same rendered past the author does. + for prompt in llm.prompts() { + assert!(!prompt.contains("Already tried"), "{prompt}"); + assert!(!prompt.contains("Learned from earlier"), "{prompt}"); + } +} + +#[tokio::test] +async fn two_authored_attempts_leave_two_distinct_signatures() { + // The fingerprint end to end: a differently-shaped graph must not fold into + // the same exclusion-list entry as the one before it. + let (store, _root) = empty_store("retry-5"); + let ledger = MemoryLedger::new(); + + let mut signatures = Vec::new(); + for (n, name) in [(0, "shape-one"), (1, "shape-two")] { + let llm = std::sync::Arc::new(Scripted::new(vec![ + select_declines(), + authored_reply(name, if n == 1 { Some("repo") } else { None }), + ])); + let attempt = decide( + &Goal::new("write the weekly report"), + "ep-sigs", + &store, + &ledger, + &HostFacts::unknown(), + &caps_with(llm), + None, + ) + .await + .expect("decide"); + signatures.push(attempt.approach.signature()); + } + + assert_ne!(signatures[0], signatures[1], "{signatures:?}"); + assert!(signatures[0].starts_with("authored:"), "{signatures:?}"); +} diff --git a/crates/tinyflows-adaptive/tests/intake.rs b/crates/tinyflows-adaptive/tests/intake_tests.rs similarity index 51% rename from crates/tinyflows-adaptive/tests/intake.rs rename to crates/tinyflows-adaptive/tests/intake_tests.rs index e09d324e..bfb42fb1 100644 --- a/crates/tinyflows-adaptive/tests/intake.rs +++ b/crates/tinyflows-adaptive/tests/intake_tests.rs @@ -484,497 +484,5 @@ async fn a_hallucinated_workflow_id_reads_as_a_decline() { ); } -#[tokio::test] -async fn an_authored_graph_that_does_not_validate_is_an_error_not_a_return_value() { - // Handing it back would turn an authoring mistake into a run-time failure - // that reads like the work failing. The author retries with the refusal - // fed back, so the script holds a model that stays wrong for every round. - let broken = json!({ - "why": "forgot the steps", - "inputs": {}, - }); - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - broken.clone(), - broken.clone(), - broken, - ])); - let caps = caps_with(llm); - let (store, _root) = empty_store("7"); - let ledger = MemoryLedger::new(); - - let err = decide( - &Goal::new("anything"), - "ep1", - &store, - &ledger, - &HostFacts::unknown(), - &caps, - None, - ) - .await - .expect_err("an invalid graph must not leave intake"); - assert!(err.to_string().contains("invalid"), "{err}"); -} - -#[tokio::test] -async fn a_disabled_workflow_is_never_offered() { - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - authored_reply("written", None), - ])); - let caps = caps_with(llm.clone()); - let (store, _root) = empty_store("8"); - let mut off = stored("switched-off", "would have matched", None); - off.enabled = false; - store.save(&off).expect("save"); - let ledger = MemoryLedger::new(); - - decide( - &Goal::new("do the thing"), - "ep1", - &store, - &ledger, - &HostFacts::unknown(), - &caps, - None, - ) - .await - .expect("decide"); - - assert!( - !llm.prompts()[0].contains("switched-off"), - "offering a disabled workflow invites a choice that cannot be honoured: {}", - llm.prompts()[0] - ); -} - -#[tokio::test] -async fn a_graph_naming_a_worker_this_host_lacks_is_refused_before_it_runs() { - // The whole point of collecting host facts. Without this the graph saves - // cleanly, validates cleanly, and fails at run time — usually overnight, - // to nobody watching. - // - // Three copies: the author feeds refusals back, and this model never - // learns that the worker does not exist. - let insistent = json!({ - "why": "needs an agent", - "inputs": {}, - "steps": [{ "id": "work", "ask": "do the thing", "worker": "desktop" }], - }); - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - insistent.clone(), - insistent.clone(), - insistent, - ])); - let caps = caps_with(llm); - let (store, _root) = empty_store("gated"); - let ledger = MemoryLedger::new(); - - let facts = HostFacts { - workers: vec!["laptop".into(), "ci".into()], - default_worker: Some("laptop".into()), - ..HostFacts::unknown() - }; - - let err = decide( - &Goal::new("do the thing"), - "ep1", - &store, - &ledger, - &facts, - &caps, - None, - ) - .await - .expect_err("a worker this host lacks must not reach the engine"); - - assert!( - err.to_string().contains("desktop"), - "the error names it: {err}" - ); - assert!( - err.to_string().contains("laptop"), - "and offers the alternatives: {err}" - ); -} - -#[tokio::test] -async fn the_authoring_prompt_carries_what_the_host_permits() { - // The facts below say agent work must name a worker, so the reply's ask - // step names one — the same gate this test exists to see rendered. - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - json!({ - "why": "fine", - "inputs": {}, - "steps": [{ "id": "work", "ask": "Do it directly.", "worker": "laptop" }], - }), - ])); - let caps = caps_with(llm.clone()); - let (store, _root) = empty_store("facts-rendered"); - let ledger = MemoryLedger::new(); - - let facts = HostFacts { - workers: vec!["laptop".into()], - default_worker: None, - allow_code: Some(false), - notes: vec!["Only manual triggers fire here.".into()], - ..HostFacts::unknown() - }; - - decide( - &Goal::new("anything"), - "ep1", - &store, - &ledger, - &facts, - &caps, - None, - ) - .await - .expect("decide"); - - let prompt = &authoring_prompt(&llm); - assert!(prompt.contains("What this host permits"), "{prompt}"); - assert!(prompt.contains("every agent node must name config.agent_ref")); - assert!(prompt.contains("Only manual triggers fire here.")); -} - -// --------------------------------------------------------------------------- -// Promotion: a repaired family is one row, and score decides which. -// --------------------------------------------------------------------------- - -/// A parent and one variant, both stored and linked, with scores applied. -async fn repaired_family( - tag: &str, - parent: (u32, u32), - variant: (u32, u32), -) -> (FileWorkflowStore, MemoryLedger, std::path::PathBuf) { - let (store, root) = empty_store(tag); - store - .save(&stored("weekly", "writes the weekly report", None)) - .expect("save"); - store - .save(&stored( - "weekly-fix-1", - "writes the weekly report, with the binding corrected", - None, - )) - .expect("save"); - - let ledger = MemoryLedger::new(); - ledger - .link_variant("weekly", "weekly-fix-1") - .await - .expect("link"); - for (id, (applied, helped)) in [("weekly", parent), ("weekly-fix-1", variant)] { - for n in 0..applied { - ledger.score_workflow(id, n < helped).await.expect("score"); - } - } - (store, ledger, root) -} - -/// What the selector was actually shown. -async fn offered(store: &FileWorkflowStore, ledger: &MemoryLedger) -> String { - let llm = std::sync::Arc::new(Scripted::new(vec![ - json!({"workflow_id": "none"}), - authored_reply("fallback", None), - ])); - let caps = caps_with(llm.clone()); - let _ = decide( - &Goal::new("write the weekly report"), - "ep-promo", - store, - ledger, - &HostFacts::unknown(), - &caps, - None, - ) - .await; - llm.prompts().first().cloned().unwrap_or_default() -} - -#[tokio::test] -async fn a_repaired_family_is_offered_as_one_row_not_two() { - // Two near-identical graphs whose descriptions differ by a clause is not a - // choice, it is noise. - let (store, ledger, _root) = repaired_family("promo-1", (40, 40), (0, 0)).await; - let shown = offered(&store, &ledger).await; - let rows = shown.matches("weekly").count(); - assert!(rows > 0, "the family must be offered at all: {shown}"); - assert!( - !shown.contains("weekly-fix-1"), - "an unproven variant must not appear beside its proven parent: {shown}" - ); -} - -#[tokio::test] -async fn a_fresh_variant_does_not_displace_a_proven_parent() { - let (store, ledger, _root) = repaired_family("promo-2", (40, 40), (0, 0)).await; - let shown = offered(&store, &ledger).await; - assert!(shown.contains("weekly"), "{shown}"); - assert!(!shown.contains("weekly-fix-1"), "{shown}"); -} - -#[tokio::test] -async fn a_variant_that_has_proven_better_is_the_one_offered() { - // Promotion on score, not on having been written. - let (store, ledger, _root) = repaired_family("promo-3", (10, 5), (4, 4)).await; - let shown = offered(&store, &ledger).await; - assert!( - shown.contains("weekly-fix-1"), - "the better member must take the position: {shown}" - ); -} - -#[tokio::test] -async fn a_family_whose_champion_was_already_tried_still_offers_its_variant() { - // The case that matters most and is easiest to get wrong: this episode just - // failed with the parent, so the parent is excluded — and the variant - // exists *because* the parent fell short. Dropping the whole family would - // hide the one graph written for this exact situation. - let (store, ledger, _root) = repaired_family("promo-4", (40, 40), (0, 0)).await; - ledger - .append(&tinyflows_adaptive::ledger::LedgerRow { - id: String::new(), - episode: "ep-promo".into(), - attempt: 1, - approach_sig: "selected:weekly".into(), - approach_desc: "the champion".into(), - workflow_id: Some("weekly".into()), - outcome: "fell short".into(), - cause: String::new(), - cost_usd: 0.0, - at: "2026-01-01T00:00:00Z".into(), - satisfied: false, - advanced: false, - }) - .await - .expect("append"); - - let shown = offered(&store, &ledger).await; - assert!( - shown.contains("weekly-fix-1"), - "the variant must survive its champion being excluded: {shown}" - ); -} - -// --------------------------------------------------------------------------- -// The retry edge: attempt four must not be attempt two in different words. -// --------------------------------------------------------------------------- - -async fn with_history(tag: &str) -> (FileWorkflowStore, MemoryLedger, std::path::PathBuf) { - let (store, root) = empty_store(tag); - let ledger = MemoryLedger::new(); - for (attempt, sig, desc, cause) in [ - ( - 1u32, - "authored:aaa", - "fetched the log and summarised it", - "no numbers in it", - ), - ( - 2, - "authored:bbb", - "asked an agent to write it from memory", - "it invented the figures", - ), - ] { - ledger - .append(&tinyflows_adaptive::ledger::LedgerRow { - id: String::new(), - episode: "ep-retry".into(), - attempt, - approach_sig: sig.into(), - approach_desc: desc.into(), - workflow_id: None, - outcome: "fell short".into(), - cause: cause.into(), - cost_usd: 0.0, - at: "2026-01-01T00:00:00Z".into(), - satisfied: false, - advanced: false, - }) - .await - .expect("append"); - } - (store, ledger, root) -} - -#[tokio::test] -async fn the_author_is_shown_what_this_episode_already_tried() { - // Without this the author writes attempt two's graph again, confidently, - // because nothing told it otherwise. The exclusion list only guards - // *selection*; authoring has no structural guard at all. - let (store, ledger, _root) = with_history("retry-1").await; - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - authored_reply("third-idea", None), - ])); - let caps = caps_with(llm.clone()); - - decide( - &Goal::new("write the weekly report"), - "ep-retry", - &store, - &ledger, - &HostFacts::unknown(), - &caps, - None, - ) - .await - .expect("decide"); - - let prompt = &authoring_prompt(&llm); - assert!(prompt.contains("Already tried this episode"), "{prompt}"); - assert!( - prompt.contains("asked an agent to write it from memory"), - "{prompt}" - ); - assert!(prompt.contains("it invented the figures"), "{prompt}"); - assert!(prompt.contains("DIFFERENT plan"), "{prompt}"); -} - -#[tokio::test] -async fn the_selector_is_shown_the_same_history_in_the_same_words() { - let (store, ledger, _root) = with_history("retry-2").await; - store - .save(&stored("weekly", "writes the weekly report", None)) - .expect("save"); - - let llm = std::sync::Arc::new(Scripted::new(vec![json!({ - "workflow_id": "weekly", - "why": "it does this", - "inputs": {}, - })])); - let caps = caps_with(llm.clone()); - - decide( - &Goal::new("write the weekly report"), - "ep-retry", - &store, - &ledger, - &HostFacts::unknown(), - &caps, - None, - ) - .await - .expect("decide"); - - let prompt = &llm.prompts()[0]; - assert!(prompt.contains("Already tried this episode"), "{prompt}"); - assert!(prompt.contains("no numbers in it"), "{prompt}"); -} - -#[tokio::test] -async fn lessons_from_other_episodes_reach_the_planner() { - // consolidate() was writing these and nothing was reading them — a - // knowledge store that costs money and returns nothing. - let (store, root) = empty_store("retry-3"); - let _ = root; - let ledger = MemoryLedger::new(); - ledger - .promote( - &tinyflows_adaptive::ledger::Lesson { - id: String::new(), - kind: tinyflows_adaptive::ledger::LessonKind::Constraint, - trigger: "a report that must cite figures".into(), - mechanism: "the model has no access to the numbers".into(), - claim: "read them from the source rather than asking an agent".into(), - applied: 0, - helped: 0, - scope_key: None, - }, - &[], - ) - .await - .expect("promote"); - - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - authored_reply("informed", None), - ])); - let caps = caps_with(llm.clone()); - - decide( - &Goal::new("write the weekly report"), - "ep-fresh", - &store, - &ledger, - &HostFacts::unknown(), - &caps, - None, - ) - .await - .expect("decide"); - - let prompt = &authoring_prompt(&llm); - assert!(prompt.contains("Learned from earlier episodes"), "{prompt}"); - assert!(prompt.contains("read them from the source"), "{prompt}"); -} - -#[tokio::test] -async fn a_first_attempt_is_told_nothing_it_would_have_to_ignore() { - // An empty history section is noise a model has to read past, and an - // empty "already tried" heading reads as a claim that something was. - let (store, _root) = empty_store("retry-4"); - let ledger = MemoryLedger::new(); - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - authored_reply("first", None), - ])); - let caps = caps_with(llm.clone()); - - decide( - &Goal::new("write the weekly report"), - "ep-first", - &store, - &ledger, - &HostFacts::unknown(), - &caps, - None, - ) - .await - .expect("decide"); - - // Every prompt, not just one: an empty heading is noise whichever planner - // reads it, and the triage call sees the same rendered past the author does. - for prompt in llm.prompts() { - assert!(!prompt.contains("Already tried"), "{prompt}"); - assert!(!prompt.contains("Learned from earlier"), "{prompt}"); - } -} - -#[tokio::test] -async fn two_authored_attempts_leave_two_distinct_signatures() { - // The fingerprint end to end: a differently-shaped graph must not fold into - // the same exclusion-list entry as the one before it. - let (store, _root) = empty_store("retry-5"); - let ledger = MemoryLedger::new(); - - let mut signatures = Vec::new(); - for (n, name) in [(0, "shape-one"), (1, "shape-two")] { - let llm = std::sync::Arc::new(Scripted::new(vec![ - select_declines(), - authored_reply(name, if n == 1 { Some("repo") } else { None }), - ])); - let attempt = decide( - &Goal::new("write the weekly report"), - "ep-sigs", - &store, - &ledger, - &HostFacts::unknown(), - &caps_with(llm), - None, - ) - .await - .expect("decide"); - signatures.push(attempt.approach.signature()); - } - - assert_ne!(signatures[0], signatures[1], "{signatures:?}"); - assert!(signatures[0].starts_with("authored:"), "{signatures:?}"); -} +include!("intake/intake_part_01_tests.rs"); +include!("intake/intake_part_02_tests.rs"); diff --git a/crates/tinyflows-catalog/src/import/n8n/mod.rs b/crates/tinyflows-catalog/src/import/n8n/mod.rs index a5727198..aa36bf6d 100644 --- a/crates/tinyflows-catalog/src/import/n8n/mod.rs +++ b/crates/tinyflows-catalog/src/import/n8n/mod.rs @@ -217,7 +217,8 @@ use graph::output_port_name; use node_mapping::trigger_config; #[cfg(test)] use node_mapping::{ - map_code, map_code_node, map_condition, map_http_request, map_split_out, map_switch, + map_code, map_code_node, map_condition, map_http_request, map_http_request_node, map_split_out, + map_switch, }; #[cfg(test)] use serde_json::json; diff --git a/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs b/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs index 88a6b05d..caaf48cb 100644 --- a/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs +++ b/crates/tinyflows-catalog/src/import/n8n/node_mapping.rs @@ -5,6 +5,9 @@ use tinyflows::model::NodeKind; use super::expr::translate_config; +mod http; +use http::{contains_expression_string, named_parameters}; + /// Maps a single n8n node `type` + `parameters` to a tinyflows kind and config. /// Unrecognized types return a `transform` placeholder carrying the original /// type/params under `_n8n_import` and record a warning. @@ -51,10 +54,7 @@ pub(super) fn map_node( map_split_out(params, warnings, n8n_name), ) } - "httpRequest" => ( - NodeKind::HttpRequest, - map_http_request(params, warnings, n8n_name), - ), + "httpRequest" => map_http_request_node(params, warnings, n8n_name), "code" | "function" | "functionItem" => map_code_node(params, warnings, n8n_name), "scheduleTrigger" | "cron" | "interval" => ( NodeKind::Trigger, @@ -185,6 +185,9 @@ fn derive_schedule(params: &Value) -> Option { /// Converts an n8n unit name + count to milliseconds, for the fixed-interval /// (not calendar-based) units n8n exposes. fn interval_to_every_ms(unit: &str, value: f64) -> Option { + if !value.is_finite() || value <= 0.0 { + return None; + } let ms_per_unit = match unit { "seconds" => 1_000.0, "minutes" => 60_000.0, @@ -192,7 +195,8 @@ fn interval_to_every_ms(unit: &str, value: f64) -> Option { "days" => 86_400_000.0, _ => return None, }; - Some(value * ms_per_unit) + let milliseconds = value * ms_per_unit; + (milliseconds.is_finite() && milliseconds >= 1.0).then_some(milliseconds) } /// Maps n8n `if` parameters onto tinyflows' `condition` config. @@ -240,10 +244,13 @@ pub(super) fn map_switch(params: &Value, warnings: &mut Vec, n8n_name: & Value::Object(map) => map, _ => Map::new(), }; - let has_rules = params - .get("rules") - .or_else(|| params.get("rules").and_then(|r| r.get("values"))) - .is_some(); + let has_rules = params.get("rules").is_some_and(|rules| { + rules.as_array().is_some_and(|entries| !entries.is_empty()) + || rules + .get("values") + .and_then(Value::as_array) + .is_some_and(|entries| !entries.is_empty()) + }); if has_rules && !cfg.contains_key("field") && !cfg.contains_key("expression") { warnings.push(format!( "Node '{n8n_name}' is an n8n switch node with a `rules` structure this importer \ @@ -286,16 +293,52 @@ pub(super) fn map_http_request( } if !cfg.contains_key("body") { if let Some(body) = cfg.remove("jsonBody") { - cfg.insert("body".to_string(), body); + match body { + Value::String(text) if text.starts_with("=.item") => { + cfg.insert("body".to_string(), Value::String(text)); + } + Value::String(text) => match serde_json::from_str(&text) { + Ok(body) => { + let body = translate_config(&body, warnings, n8n_name); + if contains_expression_string(&body) { + mark_untranslated_http_config( + &mut cfg, + warnings, + n8n_name, + "JSON body expression", + "jsonBody", + body, + ); + } else { + cfg.insert("body".to_string(), body); + } + } + Err(_) => mark_untranslated_http_config( + &mut cfg, + warnings, + n8n_name, + "JSON body text", + "jsonBody", + Value::String(text), + ), + }, + body => { + cfg.insert("body".to_string(), body); + } + } } else if let Some(body) = cfg.remove("bodyParameters") { match named_parameters(&body) { Some(body) => { cfg.insert("body".to_string(), body); } - None => warnings.push(format!( - "Node '{n8n_name}' has HTTP body parameters in an n8n shape this importer \ - cannot translate; rebuild `config.body` before enabling the flow." - )), + None => mark_untranslated_http_config( + &mut cfg, + warnings, + n8n_name, + "body parameters", + "bodyParameters", + body, + ), } } } @@ -306,10 +349,14 @@ pub(super) fn map_http_request( Some(headers) => { cfg.insert("headers".to_string(), headers); } - None => warnings.push(format!( - "Node '{n8n_name}' has HTTP headers in an n8n shape this importer cannot \ - translate; rebuild `config.headers` before enabling the flow." - )), + None => mark_untranslated_http_config( + &mut cfg, + warnings, + n8n_name, + "headers", + "headerParameters", + headers, + ), } } cfg.entry("method".to_string()) @@ -317,17 +364,45 @@ pub(super) fn map_http_request( Value::Object(cfg) } -/// Converts n8n's `{parameters:[{name,value}]}` collection to the object shape -/// tinyflows uses for HTTP bodies and headers. -fn named_parameters(value: &Value) -> Option { - let entries = value.get("parameters").and_then(Value::as_array)?; - let mut mapped = Map::new(); - for entry in entries { - let name = entry.get("name").and_then(Value::as_str)?; - let value = entry.get("value").cloned().unwrap_or(Value::Null); - mapped.insert(name.to_string(), value); - } - Some(Value::Object(mapped)) +fn mark_untranslated_http_config( + cfg: &mut Map, + warnings: &mut Vec, + n8n_name: &str, + part: &str, + source_key: &str, + source: Value, +) { + warnings.push(format!( + "Node '{n8n_name}' has HTTP {part} in an n8n shape this importer cannot translate; \ + imported as an editable placeholder. Rebuild the request before enabling the flow." + )); + let import = cfg + .entry("_n8n_import".to_string()) + .or_insert_with(|| json!({ + "original_type": "httpRequest", + "untranslated_http_config": true, + "note": "HTTP configuration could not be translated safely; rebuild before changing this placeholder to http_request.", + "untranslated": {}, + })); + import["untranslated"][source_key] = source; +} + +pub(super) fn map_http_request_node( + params: &Value, + warnings: &mut Vec, + n8n_name: &str, +) -> (NodeKind, Value) { + let config = map_http_request(params, warnings, n8n_name); + let untranslated = config + .pointer("/_n8n_import/untranslated_http_config") + .and_then(Value::as_bool) + .unwrap_or(false); + let kind = if untranslated { + NodeKind::Transform + } else { + NodeKind::HttpRequest + }; + (kind, config) } /// Maps n8n `splitOut`/`itemLists` parameters onto tinyflows' `split_out` @@ -379,9 +454,7 @@ pub(super) fn map_code(params: &Value, warnings: &mut Vec, n8n_name: &st .or_insert_with(|| Value::String(lang.to_string())); } } - if let Some(source) = cfg.get("source").and_then(Value::as_str) - && uses_n8n_code_globals(source) - { + if config_uses_n8n_runtime(&Value::Object(cfg.clone())) { warnings.push(format!( "Node '{n8n_name}' is an n8n code node imported as an editable placeholder — it uses \ n8n-only globals (`$json`/`$input`/`items`) and/or a top-level `return`, neither of \ @@ -398,10 +471,7 @@ pub(super) fn map_code_node( n8n_name: &str, ) -> (NodeKind, Value) { let mut config = map_code(params, warnings, n8n_name); - let incompatible = config - .get("source") - .and_then(Value::as_str) - .is_some_and(uses_n8n_code_globals); + let incompatible = config_uses_n8n_runtime(&config); if !incompatible { return (NodeKind::Code, config); } @@ -417,17 +487,12 @@ pub(super) fn map_code_node( (NodeKind::Transform, config) } -/// Whether `source` looks like it relies on n8n's code-node runtime globals -/// or return convention rather than tinyflows' stdin/stdout contract — a -/// cheap textual tell, not a parser, so it only needs to catch the common -/// cases without false-negatives on the (much rarer) code that happens not -/// to need them. -fn uses_n8n_code_globals(source: &str) -> bool { - ["$json", "$input", "$node", "items"] - .iter() - .any(|needle| source.contains(needle)) - || source - .split_whitespace() - .next() - .is_some_and(|first_word| first_word == "return") +fn config_uses_n8n_runtime(config: &Value) -> bool { + let Some(source) = config.get("source").and_then(Value::as_str) else { + return false; + }; + let check_top_level_return = config.get("language").and_then(Value::as_str) != Some("python"); + uses_n8n_code_globals(source, check_top_level_return) } + +include!("node_mapping/javascript.rs"); diff --git a/crates/tinyflows-catalog/src/import/n8n/node_mapping/http.rs b/crates/tinyflows-catalog/src/import/n8n/node_mapping/http.rs new file mode 100644 index 00000000..e5c7f6e5 --- /dev/null +++ b/crates/tinyflows-catalog/src/import/n8n/node_mapping/http.rs @@ -0,0 +1,23 @@ +use serde_json::{Map, Value}; + +pub(super) fn contains_expression_string(value: &Value) -> bool { + match value { + Value::String(text) => text.starts_with('=') && !text.starts_with("=.item"), + Value::Array(items) => items.iter().any(contains_expression_string), + Value::Object(map) => map.values().any(contains_expression_string), + _ => false, + } +} + +/// Converts n8n's `{parameters:[{name,value}]}` collection to the object shape +/// tinyflows uses for HTTP bodies and headers. +pub(super) fn named_parameters(value: &Value) -> Option { + let entries = value.get("parameters").and_then(Value::as_array)?; + let mut mapped = Map::new(); + for entry in entries { + let name = entry.get("name").and_then(Value::as_str)?; + let value = entry.get("value").cloned().unwrap_or(Value::Null); + mapped.insert(name.to_string(), value); + } + Some(Value::Object(mapped)) +} diff --git a/crates/tinyflows-catalog/src/import/n8n/node_mapping/http_regression_tests.rs b/crates/tinyflows-catalog/src/import/n8n/node_mapping/http_regression_tests.rs new file mode 100644 index 00000000..f62c3714 --- /dev/null +++ b/crates/tinyflows-catalog/src/import/n8n/node_mapping/http_regression_tests.rs @@ -0,0 +1,57 @@ +#[test] +fn untranslated_json_body_expression_makes_the_http_node_a_placeholder() { + let mut warnings = Vec::new(); + let (kind, cfg) = map_http_request_node( + &json!({ "jsonBody": "={{ $json.payload + 1 }}" }), + &mut warnings, + "Expression HTTP", + ); + assert_eq!(kind, NodeKind::Transform); + assert_eq!( + cfg["_n8n_import"]["untranslated"]["jsonBody"], + json!("={{ $json.payload + 1 }}") + ); +} + +#[test] +fn unsupported_http_parts_are_all_preserved_for_repair() { + let mut warnings = Vec::new(); + let (_, cfg) = map_http_request_node( + &json!({ + "bodyParameters": { "unsupported": "body" }, + "headerParameters": { "unsupported": "headers" } + }), + &mut warnings, + "Broken HTTP", + ); + assert_eq!( + cfg["_n8n_import"]["untranslated"]["bodyParameters"], + json!({ "unsupported": "body" }) + ); + assert_eq!( + cfg["_n8n_import"]["untranslated"]["headerParameters"], + json!({ "unsupported": "headers" }) + ); +} + +#[test] +fn expressions_inside_serialized_json_bodies_are_translated_or_preserved_for_repair() { + let (kind, cfg) = map_http_request_node( + &json!({ "jsonBody": r#"{"payload":"={{ $json.payload }}"}"# }), + &mut Vec::new(), + "Nested expression HTTP", + ); + assert_eq!(kind, NodeKind::HttpRequest); + assert_eq!(cfg["body"]["payload"], json!("=.item.payload")); + + let (kind, cfg) = map_http_request_node( + &json!({ "jsonBody": r#"{"payload":"={{ $json.payload + 1 }}"}"# }), + &mut Vec::new(), + "Untranslated nested expression HTTP", + ); + assert_eq!(kind, NodeKind::Transform); + assert_eq!( + cfg["_n8n_import"]["untranslated"]["jsonBody"]["payload"], + json!("={{ $json.payload + 1 }}") + ); +} diff --git a/crates/tinyflows-catalog/src/import/n8n/node_mapping/javascript.rs b/crates/tinyflows-catalog/src/import/n8n/node_mapping/javascript.rs new file mode 100644 index 00000000..1cf624b0 --- /dev/null +++ b/crates/tinyflows-catalog/src/import/n8n/node_mapping/javascript.rs @@ -0,0 +1,343 @@ +/// Whether `source` looks like it relies on n8n's code-node runtime globals +/// or return convention rather than tinyflows' stdin/stdout contract — a +/// lightweight lexer. String/comment contents are skipped, and `return` is +/// incompatible only outside a function body. +fn uses_n8n_code_globals(source: &str, check_top_level_return: bool) -> bool { + #[derive(Clone, Copy)] + enum PendingFunctionBody { + Declaration(usize), + Arrow, + } + + #[derive(Clone, Copy)] + struct ItemsBinding { + depth: usize, + expires_at_semicolon: bool, + } + + let bytes = source.as_bytes(); + let mut index = 0; + let mut brace_depth = 0usize; + let mut paren_depth = 0usize; + let mut function_depths = Vec::new(); + let mut pending_function_body = None; + let mut pending_variable_declaration = false; + let mut items_bindings = Vec::new(); + while index < bytes.len() { + let starts_comment = bytes[index] == b'/' + && matches!(bytes.get(index + 1), Some(b'/') | Some(b'*')); + if matches!(pending_function_body, Some(PendingFunctionBody::Arrow)) + && !bytes[index].is_ascii_whitespace() + && bytes[index] != b'{' + && !starts_comment + { + pending_function_body = None; + } + match bytes[index] { + b'/' if bytes.get(index + 1) == Some(&b'/') => { + index += 2; + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + index += 2; + while index + 1 < bytes.len() + && !(bytes[index] == b'*' && bytes[index + 1] == b'/') + { + index += 1; + } + index = (index + 2).min(bytes.len()); + } + b'/' if is_regex_start(bytes, index) => index = skip_regex(bytes, index), + quote @ (b'\'' | b'"') => index = skip_quoted(bytes, index, quote), + b'`' => { + index += 1; + while index < bytes.len() && bytes[index] != b'`' { + if bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + } else if bytes[index] == b'$' && bytes.get(index + 1) == Some(&b'{') { + let start = index + 2; + let end = template_expression_end(bytes, start); + if uses_n8n_code_globals(&source[start..end], check_top_level_return) { + return true; + } + index = (end + 1).min(bytes.len()); + } else { + index += 1; + } + } + index = (index + 1).min(bytes.len()); + } + b'=' if bytes.get(index + 1) == Some(&b'>') => { + pending_function_body = Some(PendingFunctionBody::Arrow); + index += 2; + } + b'(' => { + paren_depth += 1; + index += 1; + } + b')' => { + paren_depth = paren_depth.saturating_sub(1); + index += 1; + } + b'{' => { + brace_depth += 1; + let is_function_body = matches!( + pending_function_body, + Some(PendingFunctionBody::Arrow) + ) || matches!( + pending_function_body, + Some(PendingFunctionBody::Declaration(depth)) if depth == paren_depth + ); + if is_function_body { + function_depths.push(brace_depth); + pending_function_body = None; + } + index += 1; + } + b'}' => { + if function_depths.last() == Some(&brace_depth) { + function_depths.pop(); + } + items_bindings.retain(|binding: &ItemsBinding| binding.depth < brace_depth); + brace_depth = brace_depth.saturating_sub(1); + index += 1; + } + b';' => { + items_bindings.retain(|binding| !binding.expires_at_semicolon); + index += 1; + } + first if first.is_ascii_alphabetic() || matches!(first, b'_' | b'$') => { + let start = index; + index += 1; + while bytes.get(index).is_some_and(|character| { + character.is_ascii_alphanumeric() || matches!(character, b'_' | b'$') + }) { + index += 1; + } + let token = &source[start..index]; + if previous_significant(bytes, start) != Some(b'.') + && (token == "function" + || (!is_control_keyword(token) && method_body_follows(bytes, index))) + { + pending_function_body = Some(PendingFunctionBody::Declaration(paren_depth)); + pending_variable_declaration = false; + } else if matches!(token, "const" | "let" | "var") { + pending_variable_declaration = true; + } else if token == "items" { + let function_parameter = matches!( + pending_function_body, + Some(PendingFunctionBody::Declaration(depth)) if paren_depth > depth + ); + let arrow_parameter = arrow_parameter_scope(bytes, index); + if pending_variable_declaration || function_parameter || arrow_parameter.is_some() + { + let block_scoped_parameter = function_parameter || arrow_parameter == Some(true); + items_bindings.push(ItemsBinding { + depth: brace_depth + usize::from(block_scoped_parameter), + expires_at_semicolon: arrow_parameter == Some(false), + }); + } else if !items_bindings + .iter() + .any(|binding| binding.depth <= brace_depth) + { + return true; + } + pending_variable_declaration = false; + } else if ["$json", "$input", "$node"].contains(&token) + || (check_top_level_return + && token == "return" + && function_depths.is_empty()) + { + return true; + } else { + pending_variable_declaration = false; + } + } + _ => index += 1, + } + } + false +} + +fn arrow_parameter_scope(bytes: &[u8], mut index: usize) -> Option { + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) { + index += 1; + } + if bytes.get(index..index + 2) != Some(b"=>") { + let close = bytes[index..].iter().position(|byte| *byte == b')')? + index; + index = close + 1; + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) { + index += 1; + } + if bytes.get(index..index + 2) != Some(b"=>") { + return None; + } + } + index += 2; + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) { + index += 1; + } + Some(bytes.get(index) == Some(&b'{')) +} + +fn is_control_keyword(token: &str) -> bool { + matches!(token, "if" | "for" | "while" | "switch" | "catch" | "with") +} + +fn method_body_follows(bytes: &[u8], mut index: usize) -> bool { + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) { + index += 1; + } + if bytes.get(index) != Some(&b'(') { + return false; + } + let mut depth = 0usize; + while index < bytes.len() { + match bytes[index] { + quote @ (b'\'' | b'"' | b'`') => index = skip_quoted(bytes, index, quote), + b'(' => { + depth += 1; + index += 1; + } + b')' => { + depth = depth.saturating_sub(1); + index += 1; + if depth == 0 { + while bytes.get(index).is_some_and(u8::is_ascii_whitespace) { + index += 1; + } + return bytes.get(index) == Some(&b'{'); + } + } + _ => index += 1, + } + } + false +} + +fn is_regex_start(bytes: &[u8], index: usize) -> bool { + if bytes[..index] + .iter() + .rev() + .copied() + .find(|byte| !byte.is_ascii_whitespace()) + .is_none_or(|byte| { + matches!( + byte, + b'=' | b'(' | b'[' | b'{' | b',' | b':' | b';' | b'!' | b'?' + | b'&' | b'|' | b'+' | b'-' | b'*' | b'%' | b'^' | b'~' + | b'<' | b'>' + ) + }) + { + return true; + } + + previous_identifier(bytes, index).is_some_and(|token| { + matches!( + token, + b"return" | b"throw" | b"case" | b"delete" | b"typeof" | b"void" | b"instanceof" + ) + }) +} + +fn previous_significant(bytes: &[u8], index: usize) -> Option { + bytes[..index] + .iter() + .rev() + .copied() + .find(|byte| !byte.is_ascii_whitespace()) +} + +fn previous_identifier(bytes: &[u8], index: usize) -> Option<&[u8]> { + let end = bytes[..index] + .iter() + .rposition(|byte| !byte.is_ascii_whitespace())? + + 1; + let start = bytes[..end] + .iter() + .rposition(|byte| !(byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$'))) + .map_or(0, |position| position + 1); + (start < end).then_some(&bytes[start..end]) +} + +fn skip_regex(bytes: &[u8], mut index: usize) -> usize { + index += 1; + let mut in_class = false; + while index < bytes.len() { + match bytes[index] { + b'\\' => index = (index + 2).min(bytes.len()), + b'[' => { + in_class = true; + index += 1; + } + b']' => { + in_class = false; + index += 1; + } + b'/' if !in_class => { + index += 1; + while bytes.get(index).is_some_and(u8::is_ascii_alphabetic) { + index += 1; + } + return index; + } + _ => index += 1, + } + } + index +} + +fn skip_quoted(bytes: &[u8], mut index: usize, quote: u8) -> usize { + index += 1; + while index < bytes.len() { + if bytes[index] == b'\\' { + index = (index + 2).min(bytes.len()); + } else if bytes[index] == quote { + return index + 1; + } else { + index += 1; + } + } + index +} + +fn template_expression_end(bytes: &[u8], mut index: usize) -> usize { + let mut depth = 1usize; + while index < bytes.len() { + match bytes[index] { + b'/' if bytes.get(index + 1) == Some(&b'/') => { + index += 2; + while index < bytes.len() && bytes[index] != b'\n' { + index += 1; + } + } + b'/' if bytes.get(index + 1) == Some(&b'*') => { + index += 2; + while index + 1 < bytes.len() + && !(bytes[index] == b'*' && bytes[index + 1] == b'/') + { + index += 1; + } + index = (index + 2).min(bytes.len()); + } + b'/' if is_regex_start(bytes, index) => index = skip_regex(bytes, index), + quote @ (b'\'' | b'"' | b'`') => index = skip_quoted(bytes, index, quote), + b'{' => { + depth += 1; + index += 1; + } + b'}' => { + depth -= 1; + if depth == 0 { + return index; + } + index += 1; + } + _ => index += 1, + } + } + bytes.len() +} diff --git a/crates/tinyflows-catalog/src/import/n8n/node_mapping/schedule_tests.rs b/crates/tinyflows-catalog/src/import/n8n/node_mapping/schedule_tests.rs new file mode 100644 index 00000000..11f504c3 --- /dev/null +++ b/crates/tinyflows-catalog/src/import/n8n/node_mapping/schedule_tests.rs @@ -0,0 +1,70 @@ +#[test] +fn schedule_trigger_maps_a_fixed_unit_interval_rule() { + let mut warnings = Vec::new(); + let cfg = trigger_config( + "schedule", + &json!({ + "rule": { "interval": [{ "field": "hours", "hoursInterval": 2 }] } + }), + &mut warnings, + "ScheduleTrigger", + ); + assert_eq!( + cfg["schedule"], + json!({ "kind": "every", "every_ms": 7200000.0 }) + ); +} + +#[test] +fn unrecognized_schedule_shape_warns_instead_of_guessing() { + let mut warnings = Vec::new(); + let cfg = trigger_config( + "schedule", + &json!({ "rule": { "interval": [{ "field": "weekday", "weekday": 1 }] } }), + &mut warnings, + "Weekly", + ); + assert!(cfg.get("schedule").is_none()); + assert!( + warnings + .iter() + .any(|w| w.contains("Weekly") && w.contains("could not be translated")) + ); +} + +#[test] +fn multiple_schedule_intervals_warn_instead_of_dropping_cadences() { + let mut warnings = Vec::new(); + let cfg = trigger_config( + "schedule", + &json!({ "rule": { "interval": [ + { "field": "hours", "hoursInterval": 2 }, + { "field": "hours", "hoursInterval": 6 } + ] } }), + &mut warnings, + "Several cadences", + ); + assert!(cfg.get("schedule").is_none()); + assert!(warnings.iter().any(|warning| { + warning.contains("Several cadences") && warning.contains("could not be translated") + })); +} + +#[test] +fn non_positive_or_sub_millisecond_intervals_are_not_scheduled() { + for value in [-1.0, 0.0, 0.000_1, f64::MAX] { + let mut warnings = Vec::new(); + let cfg = trigger_config( + "schedule", + &json!({ "unit": "seconds", "value": value }), + &mut warnings, + "Invalid interval", + ); + assert!(cfg.get("schedule").is_none(), "value={value}: {cfg}"); + assert!( + warnings + .iter() + .any(|warning| warning.contains("could not be translated")) + ); + } +} diff --git a/crates/tinyflows-catalog/src/import/n8n/node_mapping_tests.rs b/crates/tinyflows-catalog/src/import/n8n/node_mapping_tests.rs index b8850be6..e640129f 100644 --- a/crates/tinyflows-catalog/src/import/n8n/node_mapping_tests.rs +++ b/crates/tinyflows-catalog/src/import/n8n/node_mapping_tests.rs @@ -127,6 +127,42 @@ fn http_request_normalizes_json_body_named_body_fields_and_headers() { "JSON HTTP", ); assert_eq!(cfg["body"], json!({ "ready": true })); + + let cfg = map_http_request( + &json!({ "jsonBody": "{\"ready\":true}" }), + &mut warnings, + "Text JSON HTTP", + ); + assert_eq!(cfg["body"], json!({ "ready": true })); + + let cfg = map_http_request( + &json!({ "jsonBody": "={{ $json.payload }}" }), + &mut warnings, + "Expression JSON HTTP", + ); + assert_eq!(cfg["body"], json!("=.item.payload")); + assert!(warnings.is_empty(), "{warnings:?}"); +} + +#[test] +fn invalid_textual_json_body_makes_the_http_node_a_placeholder() { + let mut warnings = Vec::new(); + let (kind, cfg) = map_http_request_node( + &json!({ "jsonBody": "{not json}" }), + &mut warnings, + "Broken HTTP", + ); + assert_eq!(kind, NodeKind::Transform); + assert_eq!(cfg["_n8n_import"]["untranslated_http_config"], json!(true)); + assert_eq!( + cfg["_n8n_import"]["untranslated"]["jsonBody"], + json!("{not json}") + ); + assert!( + warnings + .iter() + .any(|warning| warning.contains("placeholder")) + ); } #[test] @@ -262,12 +298,86 @@ fn incompatible_n8n_code_is_a_placeholder_not_an_executable_code_node() { assert_eq!(kind, NodeKind::Transform); assert_eq!(cfg["_n8n_import"]["original_type"], json!("code")); + let (kind, _) = map_code_node( + &json!({ "jsCode": "const out = transform(input); return out;" }), + &mut warnings, + "Late return", + ); + assert_eq!(kind, NodeKind::Transform); + let (kind, _) = map_code_node( &json!({ "jsCode": "process.stdin.pipe(process.stdout);" }), &mut Vec::new(), "Portable", ); assert_eq!(kind, NodeKind::Code); + + for source in [ + "function id(value) { return value; } module.exports = id(input);", + "const word = \"return\"; module.exports = word;", + "// return is discussed here\nmodule.exports = input;", + "const id = (value) => { return value; }; module.exports = id(input);", + "function pick({value}) { return value; } module.exports = pick(input);", + "const helper = { pick(value) { return value; } }; module.exports = helper.pick(input);", + "module.exports = input.map(function (value) { return value; });", + "module.exports = input.values.map(value => { return value; });", + "module.exports = /return/.test(input);", + "function test(value) { return /$json/.test(value); } module.exports = test(input);", + "const items = input.values; module.exports = items;", + "function map(items) { return items.length; } module.exports = map(input);", + ] { + let (kind, _) = map_code_node(&json!({ "jsCode": source }), &mut Vec::new(), "Portable"); + assert_eq!(kind, NodeKind::Code, "source was downgraded: {source}"); + } + + let (kind, _) = map_code_node( + &json!({ "jsCode": "console.log(`${$json.id}`);" }), + &mut Vec::new(), + "n8n template", + ); + assert_eq!(kind, NodeKind::Transform); + + let (kind, _) = map_code_node( + &json!({ "jsCode": "const f = x => x; if (ok) { return value; }" }), + &mut Vec::new(), + "Top-level return after arrow", + ); + assert_eq!(kind, NodeKind::Transform); + + let (kind, _) = map_code_node( + &json!({ "jsCode": "obj.function(); if (ok) { return value; }" }), + &mut Vec::new(), + "Function property before top-level return", + ); + assert_eq!(kind, NodeKind::Transform); + + let (kind, _) = map_code_node( + &json!({ "jsCode": "console.log(`${/* } */ $json.id}`);" }), + &mut Vec::new(), + "n8n template with comment", + ); + assert_eq!(kind, NodeKind::Transform); + + let (kind, _) = map_code_node( + &json!({ "jsCode": "console.log(items.length); process.stdin.pipe(process.stdout);" }), + &mut Vec::new(), + "Unbound items global", + ); + assert_eq!(kind, NodeKind::Transform); + + let (kind, _) = map_code_node( + &json!({ "pythonCode": "def identity(value): return value\nprint(identity(input()))" }), + &mut Vec::new(), + "Portable Python helper", + ); + assert_eq!(kind, NodeKind::Code); + + let (kind, _) = map_code_node( + &json!({ "jsCode": "function count(items) { return items.length; } console.log(items.length);" }), + &mut Vec::new(), + "Function-local then global items", + ); + assert_eq!(kind, NodeKind::Transform); } #[test] @@ -332,54 +442,5 @@ fn schedule_trigger_maps_a_cron_expression_rule() { ); } -#[test] -fn schedule_trigger_maps_a_fixed_unit_interval_rule() { - let mut warnings = Vec::new(); - let cfg = trigger_config( - "schedule", - &json!({ - "rule": { "interval": [{ "field": "hours", "hoursInterval": 2 }] } - }), - &mut warnings, - "ScheduleTrigger", - ); - assert_eq!( - cfg["schedule"], - json!({ "kind": "every", "every_ms": 7200000.0 }) - ); -} - -#[test] -fn unrecognized_schedule_shape_warns_instead_of_guessing() { - let mut warnings = Vec::new(); - let cfg = trigger_config( - "schedule", - &json!({ "rule": { "interval": [{ "field": "weekday", "weekday": 1 }] } }), - &mut warnings, - "Weekly", - ); - assert!(cfg.get("schedule").is_none()); - assert!( - warnings - .iter() - .any(|w| w.contains("Weekly") && w.contains("could not be translated")) - ); -} - -#[test] -fn multiple_schedule_intervals_warn_instead_of_dropping_cadences() { - let mut warnings = Vec::new(); - let cfg = trigger_config( - "schedule", - &json!({ "rule": { "interval": [ - { "field": "hours", "hoursInterval": 2 }, - { "field": "hours", "hoursInterval": 6 } - ] } }), - &mut warnings, - "Several cadences", - ); - assert!(cfg.get("schedule").is_none()); - assert!(warnings.iter().any(|warning| { - warning.contains("Several cadences") && warning.contains("could not be translated") - })); -} +include!("node_mapping/http_regression_tests.rs"); +include!("node_mapping/schedule_tests.rs"); diff --git a/crates/tinyflows-sqlite/src/drafts_tests.rs b/crates/tinyflows-sqlite/src/drafts_tests.rs index 1a4a5d9e..c98c349b 100644 --- a/crates/tinyflows-sqlite/src/drafts_tests.rs +++ b/crates/tinyflows-sqlite/src/drafts_tests.rs @@ -154,12 +154,21 @@ fn concurrent_updates_to_different_fields_do_not_lose_either_patch() { #[test] fn inactive_draft_locks_are_pruned_instead_of_accumulating() { + let root = PathBuf::from(format!("/draft-lock-prune-{}", Uuid::new_v4())); for index in 0..32 { - let lock = lock_for(Path::new(&format!("/drafts/{index}.json"))); + let lock = lock_for(&root.join(format!("{index}.json"))); drop(lock); } - let final_lock = lock_for(Path::new("/drafts/final.json")); + let final_lock = lock_for(&root.join("final.json")); let registry = DRAFT_LOCKS.get().expect("lock registry"); - assert_eq!(registry.lock().unwrap().len(), 1); + assert_eq!( + registry + .lock() + .unwrap() + .keys() + .filter(|path| path.starts_with(&root)) + .count(), + 1 + ); drop(final_lock); } diff --git a/crates/tinyflows/src/bindings.rs b/crates/tinyflows/src/bindings.rs index 6b87a282..41c981be 100644 --- a/crates/tinyflows/src/bindings.rs +++ b/crates/tinyflows/src/bindings.rs @@ -112,12 +112,43 @@ pub fn parse_node_binding(expr: &str) -> Option { // `.json` is optional, and only counts when it is a whole path segment: a // node field actually named `jsonish` must not be mistaken for the envelope. let (through_envelope, rest) = match rest.strip_prefix(".json") { - Some(after) if after.starts_with('.') => (true, after), + Some(after) if after.starts_with(['.', '[']) => (true, after), _ => (false, rest), }; - let rest = rest.strip_prefix('.')?; - let (field_path, _) = take_field_path(rest)?; + let mut field_path = String::new(); + let mut remainder = rest; + loop { + if let Some(after_open) = remainder.strip_prefix('[') + && let Some(close) = after_open.find(']') + && !after_open[..close].is_empty() + && after_open[..close].chars().all(|ch| ch.is_ascii_digit()) + { + field_path.push('['); + field_path.push_str(&after_open[..close]); + field_path.push(']'); + remainder = &after_open[close + 1..]; + } else if let Some(after_dot) = remainder.strip_prefix('.') + && let Some((tail, after_tail)) = take_field_path(after_dot) + { + if !field_path.is_empty() { + field_path.push('.'); + } + field_path.push_str(&tail); + remainder = after_tail; + } else { + break; + } + } + if field_path.is_empty() { + return None; + } + // The static gates only reject a complete simple binding. A continued jq + // program may recover from a missing path (for example with `//`), so its + // result is not guaranteed to be null and must be left to evaluation. + if !remainder.trim().is_empty() { + return None; + } let through_envelope = through_envelope || matches!(field_path.as_str(), "text" | "raw"); Some(NodeBinding { node_id: node_id.to_string(), @@ -126,14 +157,18 @@ pub fn parse_node_binding(expr: &str) -> Option { }) } -/// The leading `[A-Za-z_][A-Za-z0-9_]*`, and what follows it. +/// The leading `[A-Za-z_][A-Za-z0-9_-]*`, and what follows it. +/// +/// Hyphens are accepted after the first character to match the runtime +/// expression evaluator, where node ids such as `nodes.my-node.item.value` +/// address a literal node key rather than a subtraction expression. fn take_identifier(input: &str) -> Option<(&str, &str)> { let mut end = 0; for (index, ch) in input.char_indices() { let ok = if index == 0 { ch.is_ascii_alphabetic() || ch == '_' } else { - ch.is_ascii_alphanumeric() || ch == '_' + ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' }; if !ok { break; diff --git a/crates/tinyflows/src/engine/build/activation.rs b/crates/tinyflows/src/engine/build/activation.rs index ce3a063b..57c9b100 100644 --- a/crates/tinyflows/src/engine/build/activation.rs +++ b/crates/tinyflows/src/engine/build/activation.rs @@ -87,188 +87,9 @@ impl HandlerData { // port-command node drives only the successors of the port it // emitted on (`port`, defaulting to `main`); everything else emits // a plain update and follows its static/conditional edge. - let emit = |mut update: Value, port: Option<&str>, routed_items: &[Item]| { - // Only a non-lane activation stamps the node's slot: the - // stamp is how a loop head tells its own re-entry from a - // stale arm, and a lane slot is not that. - if lane.is_none() { - stamp_activation_step(&mut update, &node.id, ctx.step); - } - - // Inside a lane, routing carries the lane onward. Every - // successor is re-scheduled as a `Send` holding this - // activation's output and the same lane identity, so the - // whole downstream path runs once per lane rather than - // once in total. - // - // Except a gather: that is where lanes end. A gather is - // scheduled as a plain activation, and plain activations - // dedupe by node, so N lanes converge on one gather rather - // than activating it N times. - if let Some(lane) = lane.as_ref() { - let emitted = port.unwrap_or("main"); - let targets: Vec = match &routing { - HandlerRouting::Plain => plain_targets_by_port - .iter() - .find(|(port, _)| port == emitted) - .map(|(_, targets)| targets.clone()) - .unwrap_or_default(), - HandlerRouting::FanOut(targets) => targets.clone(), - HandlerRouting::PortCommand(groups) => groups - .iter() - .find(|(p, _)| p == emitted) - .map(|(_, targets)| targets.clone()) - .unwrap_or_default(), - }; - let routed: Vec = targets - .into_iter() - .map(|target| { - if gather_nodes.contains(&target) { - RouteTarget::Node(target.into()) - } else { - let envelope = - lane_envelope(&lane.origin, lane.index, lane.count, routed_items) - .unwrap_or(Value::Null); - RouteTarget::Send(crate::graph::Send::new(target, envelope)) - } - }) - .collect(); - return NodeResult::Command(Command::route(routed).with_update(update)); - } - - match &routing { - HandlerRouting::Plain => NodeResult::Update(update), - HandlerRouting::FanOut(targets) => { - NodeResult::Command(Command::goto(targets.clone()).with_update(update)) - } - HandlerRouting::PortCommand(groups) => { - let emitted = port.unwrap_or("main"); - let targets: Vec = groups - .iter() - .find(|(p, _)| p == emitted) - .map(|(_, targets)| targets.clone()) - .unwrap_or_default(); - NodeResult::Command(Command::goto(targets).with_update(update)) - } - } - }; - - // Cooperative cancellation, checked at the node boundary before - // any real work. When the run's token is cancelled this node - // becomes a no-op: it emits an empty update on the default port - // and — crucially — does **not** fan out (a plain `Update`, not - // `emit`), so a fan-out node's parallel successors are not - // scheduled. Downstream nodes reached by static edges will hit - // this same check and no-op in turn, so the run winds down without - // starting further node work. The engine reports it as cancelled. - if token.is_cancelled() { - tracing::info!(node = %node.id, "run cancelled; skipping node work"); - let mut update = items_update(&node.id, &[], None)?; - stamp_activation_step(&mut update, &node.id, ctx.step); - return Ok(NodeResult::Update(update)); - } + let emit = include!("activation/routing.rs"); - if is_trigger { - // The trigger payload is pre-seeded into the state; no-op update - // (still fanning out if the trigger has parallel successors). - return Ok(emit(json!({}), None, &[])); - } - - // Human-in-the-loop approval gate. A node whose config sets - // `requires_approval: true` must not execute until its id is - // listed in the run input's `approvals` array (readable at - // `state["run"]["trigger"]["approvals"]`). Until then it pauses - // the run via a graph interrupt, so its downstream never - // runs and the run reports the pending node. - let requires_approval = node - .config - .get("requires_approval") - .and_then(Value::as_bool) - .unwrap_or(false); - if requires_approval { - // Human-in-the-loop **denial**. A resume delivered with a - // structured value `{ "rejected": [, …] }` (see - // `resume_with_checkpointer_journaled_observed`) denies the - // named gate rather than approving it: the gate emits an - // error item on its `error` port when one is wired (so a - // recovery branch can handle the rejection), or fails the run - // when it has no `error` port. Checked before the approval - // branch so a denial always wins over the bare-resume approval. - let denied = resume_value - .as_ref() - .and_then(|v| v.get("rejected")) - .and_then(Value::as_array) - .is_some_and(|rejected| { - rejected - .iter() - .filter_map(Value::as_str) - .any(|id| id == node.id) - }); - if denied { - tracing::info!(node = %node.id, has_error_edge, "approval gate denied"); - let item = Item::new(json!({ - "error": { - "message": "approval denied", - "node": node.id, - "denied": true, - } - })); - if has_error_edge { - // Route the denial to the `error` port so a recovery - // sub-graph runs. Use `emit`: when the gate's error-port - // recovery edges fan out (≥2 same-port successors) the - // node is command-routed and has no conditional router to - // key on the recorded port, so the branches must be driven - // directly via a `Command::goto`; a single/mixed-port error - // edge falls back to a plain update the conditional-edge - // router consumes. - return Ok(emit( - items_update(&node.id, std::slice::from_ref(&item), Some("error"))?, - Some("error"), - std::slice::from_ref(&item), - )); - } - // No error branch to route to — fail the run so the denial - // is not silently swallowed. - return Err(GraphError::Graph(format!( - "approval gate '{}' was denied and has no `error` port to route to", - node.id - ))); - } - let approved = state.get("run").is_some_and(|run| { - // Two places, because approvals reach a run two - // ways: inside an object trigger payload (the - // original spelling, kept working) and through - // `RunInput::with_approvals`, which is the only one - // available when the trigger is not an object. - let listed = |approvals: Option<&Value>| { - approvals.and_then(Value::as_array).is_some_and(|ids| { - ids.iter().filter_map(Value::as_str).any(|id| id == node.id) - }) - }; - listed(run.get("approvals")) - || listed( - run.get("trigger") - .and_then(|trigger| trigger.get("approvals")), - ) - }); - // `approved_by_resume` is set when a checkpointed resume - // delivered an approval (bare `true`, or this gate listed in - // the structured `approved` array) to this interrupted gate. - if !approved && !approved_by_resume { - tracing::info!(node = %node.id, "node paused awaiting approval"); - let payload = if node.config.is_null() { - json!({}) - } else { - node.config.clone() - }; - return Ok(NodeResult::Interrupt(Interrupt { - id: node.id.clone(), - node: node.id.clone().into(), - payload, - })); - } - } + include!("activation/gates.rs"); // What this activation reads, derived from a given run state. // @@ -277,41 +98,7 @@ impl HandlerData { // expression scope, the resolved config — has to be re-derived from the // patched state or the node would run against one state while the // debugger showed another. - let derive = |state: &Value| -> (Vec, Value, Value) { - // Which set of incoming edges this activation draws from. A node - // with no back-edges always uses its forward edges. A loop head - // uses its forward edges on the first activation (the seed) and - // its back-edges on every re-entry, detected by whether it has - // already recorded an output slot. Without this the seed's items - // are re-delivered alongside the body's on every iteration. - let re_entry = !back_incoming.is_empty() - && state - .get("nodes") - .and_then(|nodes| nodes.get(&node.id)) - .is_some_and(|slot| !slot.is_null()); - // A lane activation carries its own work. It must not read - // predecessor slots: every branch of a super-step sees the same - // committed snapshot, so `collect_input` would hand all N lanes - // the identical items. - let input = if let Some(lane_arg) = lane_send_arg.as_ref() { - lane_input(Some(lane_arg)) - } else if re_entry { - let latest_step = back_incoming - .iter() - .filter_map(|(pred, _)| state["nodes"][pred]["_activation_step"].as_u64()) - .max(); - collect_input_since(state, &back_incoming, latest_step) - } else { - collect_input(state, &incoming) - }; - let run_meta = state.get("run").cloned().unwrap_or(Value::Null); - // Every completed node's output slot, keyed by id. Handed to the - // executor so `=`-expressions can address any upstream node - // (`nodes..item.`), not just the direct predecessors - // flattened into `input` — see `crate::nodes::expr_scope`. - let nodes_state = state.get("nodes").cloned().unwrap_or(Value::Null); - (input, run_meta, nodes_state) - }; + let derive = include!("activation/input.rs"); let (mut input, mut run_meta, mut nodes_state) = derive(&state); // Per-node error policy, read from free-form `node.config` (no model diff --git a/crates/tinyflows/src/engine/build/activation/gates.rs b/crates/tinyflows/src/engine/build/activation/gates.rs new file mode 100644 index 00000000..de455e2d --- /dev/null +++ b/crates/tinyflows/src/engine/build/activation/gates.rs @@ -0,0 +1,120 @@ +{ + // Cooperative cancellation, checked at the node boundary before + // any real work. When the run's token is cancelled this node + // becomes a no-op: it emits an empty update on the default port + // and — crucially — does **not** fan out (a plain `Update`, not + // `emit`), so a fan-out node's parallel successors are not + // scheduled. Downstream nodes reached by static edges will hit + // this same check and no-op in turn, so the run winds down without + // starting further node work. The engine reports it as cancelled. + if token.is_cancelled() { + tracing::info!(node = %node.id, "run cancelled; skipping node work"); + let mut update = items_update(&node.id, &[], None)?; + if lane.is_none() { + stamp_activation_step(&mut update, &node.id, ctx.step); + } + return Ok(NodeResult::Update(update)); + } + + if is_trigger { + // The trigger payload is pre-seeded into the state; no-op update + // (still fanning out if the trigger has parallel successors). + return Ok(emit(json!({}), None, &[])); + } + + // Human-in-the-loop approval gate. A node whose config sets + // `requires_approval: true` must not execute until its id is + // listed in the run input's `approvals` array (readable at + // `state["run"]["trigger"]["approvals"]`). Until then it pauses + // the run via a graph interrupt, so its downstream never + // runs and the run reports the pending node. + let requires_approval = node + .config + .get("requires_approval") + .and_then(Value::as_bool) + .unwrap_or(false); + if requires_approval { + // Human-in-the-loop **denial**. A resume delivered with a + // structured value `{ "rejected": [, …] }` (see + // `resume_with_checkpointer_journaled_observed`) denies the + // named gate rather than approving it: the gate emits an + // error item on its `error` port when one is wired (so a + // recovery branch can handle the rejection), or fails the run + // when it has no `error` port. Checked before the approval + // branch so a denial always wins over the bare-resume approval. + let denied = resume_value + .as_ref() + .and_then(|v| v.get("rejected")) + .and_then(Value::as_array) + .is_some_and(|rejected| { + rejected + .iter() + .filter_map(Value::as_str) + .any(|id| id == node.id) + }); + if denied { + tracing::info!(node = %node.id, has_error_edge, "approval gate denied"); + let item = Item::new(json!({ + "error": { + "message": "approval denied", + "node": node.id, + "denied": true, + } + })); + if has_error_edge { + // Route the denial to the `error` port so a recovery + // sub-graph runs. Use `emit`: when the gate's error-port + // recovery edges fan out (≥2 same-port successors) the + // node is command-routed and has no conditional router to + // key on the recorded port, so the branches must be driven + // directly via a `Command::goto`; a single/mixed-port error + // edge falls back to a plain update the conditional-edge + // router consumes. + return Ok(emit( + items_update(&node.id, std::slice::from_ref(&item), Some("error"))?, + Some("error"), + std::slice::from_ref(&item), + )); + } + // No error branch to route to — fail the run so the denial + // is not silently swallowed. + return Err(GraphError::Graph(format!( + "approval gate '{}' was denied and has no `error` port to route to", + node.id + ))); + } + let approved = state.get("run").is_some_and(|run| { + // Two places, because approvals reach a run two + // ways: inside an object trigger payload (the + // original spelling, kept working) and through + // `RunInput::with_approvals`, which is the only one + // available when the trigger is not an object. + let listed = |approvals: Option<&Value>| { + approvals.and_then(Value::as_array).is_some_and(|ids| { + ids.iter().filter_map(Value::as_str).any(|id| id == node.id) + }) + }; + listed(run.get("approvals")) + || listed( + run.get("trigger") + .and_then(|trigger| trigger.get("approvals")), + ) + }); + // `approved_by_resume` is set when a checkpointed resume + // delivered an approval (bare `true`, or this gate listed in + // the structured `approved` array) to this interrupted gate. + if !approved && !approved_by_resume { + tracing::info!(node = %node.id, "node paused awaiting approval"); + let payload = if node.config.is_null() { + json!({}) + } else { + node.config.clone() + }; + return Ok(NodeResult::Interrupt(Interrupt { + id: node.id.clone(), + node: node.id.clone().into(), + payload, + })); + } + } +} diff --git a/crates/tinyflows/src/engine/build/activation/input.rs b/crates/tinyflows/src/engine/build/activation/input.rs new file mode 100644 index 00000000..85dd2b98 --- /dev/null +++ b/crates/tinyflows/src/engine/build/activation/input.rs @@ -0,0 +1,35 @@ +|state: &Value| -> (Vec, Value, Value) { + // Which set of incoming edges this activation draws from. A node + // with no back-edges always uses its forward edges. A loop head + // uses its forward edges on the first activation (the seed) and + // its back-edges on every re-entry, detected by whether it has + // already recorded an output slot. Without this the seed's items + // are re-delivered alongside the body's on every iteration. + let re_entry = !back_incoming.is_empty() + && state + .get("nodes") + .and_then(|nodes| nodes.get(&node.id)) + .is_some_and(|slot| !slot.is_null()); + // A lane activation carries its own work. It must not read + // predecessor slots: every branch of a super-step sees the same + // committed snapshot, so `collect_input` would hand all N lanes + // the identical items. + let input = if let Some(lane_arg) = lane_send_arg.as_ref() { + lane_input(Some(lane_arg)) + } else if re_entry { + let latest_step = back_incoming + .iter() + .filter_map(|(pred, _)| state["nodes"][pred]["_activation_step"].as_u64()) + .max(); + collect_input_since(state, &back_incoming, latest_step) + } else { + collect_input(state, &incoming) + }; + let run_meta = state.get("run").cloned().unwrap_or(Value::Null); + // Every completed node's output slot, keyed by id. Handed to the + // executor so `=`-expressions can address any upstream node + // (`nodes..item.`), not just the direct predecessors + // flattened into `input` — see `crate::nodes::expr_scope`. + let nodes_state = state.get("nodes").cloned().unwrap_or(Value::Null); + (input, run_meta, nodes_state) +} diff --git a/crates/tinyflows/src/engine/build/activation/routing.rs b/crates/tinyflows/src/engine/build/activation/routing.rs new file mode 100644 index 00000000..5891a9c0 --- /dev/null +++ b/crates/tinyflows/src/engine/build/activation/routing.rs @@ -0,0 +1,65 @@ +|mut update: Value, port: Option<&str>, routed_items: &[Item]| { + // Only a non-lane activation stamps the node's slot: the + // stamp is how a loop head tells its own re-entry from a + // stale arm, and a lane slot is not that. + if lane.is_none() { + stamp_activation_step(&mut update, &node.id, ctx.step); + } + + // Inside a lane, routing carries the lane onward. Every + // successor is re-scheduled as a `Send` holding this + // activation's output and the same lane identity, so the + // whole downstream path runs once per lane rather than + // once in total. + // + // Except a gather: that is where lanes end. A gather is + // scheduled as a plain activation, and plain activations + // dedupe by node, so N lanes converge on one gather rather + // than activating it N times. + if let Some(lane) = lane.as_ref() { + let emitted = port.unwrap_or("main"); + let targets: Vec = match &routing { + HandlerRouting::Plain => plain_targets_by_port + .iter() + .find(|(port, _)| port == emitted) + .map(|(_, targets)| targets.clone()) + .unwrap_or_default(), + HandlerRouting::FanOut(targets) => targets.clone(), + HandlerRouting::PortCommand(groups) => groups + .iter() + .find(|(p, _)| p == emitted) + .map(|(_, targets)| targets.clone()) + .unwrap_or_default(), + }; + let routed: Vec = targets + .into_iter() + .map(|target| { + if gather_nodes.contains(&target) { + RouteTarget::Node(target.into()) + } else { + let envelope = + lane_envelope(&lane.origin, lane.index, lane.count, routed_items) + .unwrap_or(Value::Null); + RouteTarget::Send(crate::graph::Send::new(target, envelope)) + } + }) + .collect(); + return NodeResult::Command(Command::route(routed).with_update(update)); + } + + match &routing { + HandlerRouting::Plain => NodeResult::Update(update), + HandlerRouting::FanOut(targets) => { + NodeResult::Command(Command::goto(targets.clone()).with_update(update)) + } + HandlerRouting::PortCommand(groups) => { + let emitted = port.unwrap_or("main"); + let targets: Vec = groups + .iter() + .find(|(p, _)| p == emitted) + .map(|(_, targets)| targets.clone()) + .unwrap_or_default(); + NodeResult::Command(Command::goto(targets).with_update(update)) + } + } +} diff --git a/crates/tinyflows/src/engine/resumable.rs b/crates/tinyflows/src/engine/resumable.rs index 8417cf7f..ef6667c9 100644 --- a/crates/tinyflows/src/engine/resumable.rs +++ b/crates/tinyflows/src/engine/resumable.rs @@ -447,309 +447,4 @@ enum Continuation { Retry, } -impl Continuation { - /// The command that carries this continuation into the runtime. - fn command(self) -> Command { - match self { - Self::Approvals { approved, rejected } => { - // Approvals recorded for downstream visibility. On resume the - // interrupted gate is approved because the resume value reaches - // it via `NodeContext::resume`; the `with_update` mirrors - // `ResumableRun::resume` (the runtime ignores it on resume, so - // the resume value is the real approval channel). - let update = json!({ - "run": { "trigger": { "approvals": approved.clone() } } - }); - if !rejected.is_empty() { - tracing::info!(?rejected, "resuming with denied approval gate(s)"); - } - // Always a structured resume value carrying the explicit - // `approved` and `rejected` gate id lists. Each interrupted gate - // decides for itself: gates in `approved` proceed, gates in - // `rejected` route to their `error` port (or fail), and gates in - // neither stay pending. This is essential when several parallel - // gates are interrupted and the host resolves only some of them - // — a bare `true` would blanket-approve every interrupt - // regardless of the host's decision. - let value = json!({ "approved": approved, "rejected": rejected }); - Command::resume(value).with_update(update) - } - // Deliberately empty. A failed node is re-entered from its start - // with the state the boundary committed; a resume *value* would be - // delivered to `NodeContext::resume` and read as an approval - // decision by any gate that happened to be in the pending set. - Self::Retry => Command::new(), - } - } -} - -/// Shared implementation of the checkpointed continue path: rebuilds the graph -/// (optionally journaled), re-attaches the same `checkpointer`, and resumes -/// `thread_id`. Returns the outcome plus the resumed execution's -/// runtime-minted run ids. -async fn resume_with_checkpointer_inner( - workflow: &CompiledWorkflow, - capabilities: &Capabilities, - checkpointer: Arc>, - thread_id: &str, - continuation: Continuation, - journal: Option>, - observer: &Arc, -) -> Result<(RunOutcome, GraphRunIds)> { - let steps: Arc>> = Arc::new(Mutex::new(Vec::new())); - - // Rebuild the identical graph and re-attach the SAME checkpointer, so - // `resume` loads the state persisted under `thread_id`. Node handlers fire - // `observer.on_step_finish` for every node that runs after the interrupt - // boundary, so a host observer sees the resumed steps live. - let terminal_error: Arc>> = Arc::new(Mutex::new(None)); - let mut config = RunConfig::new(workflow)?.with_checkpointer(checkpointer, thread_id); - if let Some(journal) = journal { - config = config.with_journal(journal); - } - let (compiled, _trigger_id) = build_graph( - workflow, - capabilities, - observer, - &steps, - &terminal_error, - &config, - )?; - - let execution = compiled.resume(thread_id, continuation.command()).await; - let execution = match execution { - Ok(execution) => execution, - Err(error) => { - let structured = terminal_error - .lock() - .expect("terminal error mutex poisoned") - .take(); - return Err(structured.unwrap_or_else(|| EngineError::Capability(error.to_string()))); - } - }; - - let pending_approvals: Vec = execution - .interrupts - .iter() - .map(|interrupt| interrupt.id.clone()) - .collect(); - - let graph_run_ids = GraphRunIds { - run_id: execution.run_id.as_str().to_string(), - root_run_id: execution.root_run_id.as_str().to_string(), - }; - - Ok(( - RunOutcome { - output: execution.state, - pending_approvals, - // Checkpointed resume does not (yet) thread a caller token; a - // cancellable resume goes through `resume_cancellable`. - cancelled: false, - }, - graph_run_ids, - )) -} - -/// What a failed run left behind, and what it would take to continue it. -/// -/// A run that fails does not necessarily lose its work. On a checkpointed -/// thread the runtime folds the branches that already completed into committed -/// state and writes a **failure boundary** — a checkpoint whose pending nodes -/// are the node that failed and the not-yet-run tail of its step. Everything -/// before it is durable and does not have to happen twice. -/// -/// The engine reports the failure as an `Err`, which is the right shape for a -/// caller that just wants to know the run did not finish. This is the question -/// that error cannot answer: *is there something to continue, and where did it -/// stop?* Read it after a failed run to decide between fixing and retrying, -/// and re-running from the trigger. -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct FailureBoundary { - /// The node whose handler failed. - pub failed_node: String, - /// The error as the runtime rendered it, for diagnosis. - pub error: String, - /// The checkpoint holding the committed prefix — the id - /// [`ResumeTarget::Checkpoint`](crate::graph::ResumeTarget) addresses. - pub checkpoint_id: String, - /// Which superstep the run reached. - pub step: usize, - /// The nodes a continue would run: the failed one, and whatever else in - /// its step had not run when it aborted. - pub pending: Vec, -} - -/// Read the failure boundary a thread's latest checkpoint records, if it is one. -/// -/// `Ok(None)` for a thread that has no checkpoint, or whose latest is an -/// ordinary boundary — a completed run, or one paused at an approval gate. -/// Those are not failures and have nothing to continue *from a failure*. -/// -/// Deliberately a separate read rather than a field on the error. A failed run -/// already returns [`EngineError`], every caller handles that, and widening it -/// would make every one of them carry a concept most do not use. Asking -/// afterwards also reads the way the decision is actually made: the run -/// failed — is it worth continuing? -/// -/// # Errors -/// When the checkpointer cannot be read. -pub async fn failure_boundary( - checkpointer: &Arc>, - thread_id: &str, -) -> Result> { - let checkpoint = checkpointer - .get(thread_id, None) - .await - .map_err(|error| EngineError::Capability(error.to_string()))?; - let Some(checkpoint) = checkpoint else { - return Ok(None); - }; - // `failed_node` is what makes a boundary a *failure* boundary — an - // interrupt boundary and a terminal one both lack it. Reading the key - // rather than a status field keeps this to one checkpoint load. - let Some(failed_node) = checkpoint - .metadata - .get("failed_node") - .and_then(Value::as_str) - else { - return Ok(None); - }; - Ok(Some(FailureBoundary { - failed_node: failed_node.to_string(), - error: checkpoint - .metadata - .get("error") - .and_then(Value::as_str) - .unwrap_or_default() - .to_string(), - checkpoint_id: checkpoint.checkpoint_id.clone(), - step: checkpoint - .metadata - .get("step") - .and_then(Value::as_u64) - .and_then(|step| usize::try_from(step).ok()) - .unwrap_or(0), - pending: checkpoint - .next_nodes - .iter() - .map(ToString::to_string) - .collect(), - })) -} - -/// Continue a **failed** run from where it stopped: re-run the node that -/// failed and the not-yet-run tail of its step, on the state the failure -/// boundary committed. -/// -/// The counterpart of [`resume_with_checkpointer`] for the failure path. That -/// one answers a pause with a decision; this one answers a break with another -/// go, and carries no resume value — there is nothing to decide, only work to -/// redo. -/// -/// Two reasons to reach for this over re-running the workflow: -/// -/// * **Side effects.** A prefix that posted a comment, opened a pull request -/// or charged something does not do it twice. Re-running from the trigger is -/// not a neutral choice for a graph with effects in it; it is a second set -/// of them. -/// * **Cost.** A prefix step can be a whole coding session. Paying for it -/// again to reach the same failed node buys nothing. -/// -/// **The graph must be the one that failed.** Node handlers are rebuilt from -/// `workflow`, and the committed state is keyed by node id, so a `workflow` -/// whose prefix differs from the one that ran will re-enter the tail on state -/// it would never have produced — a run that goes green and is quietly wrong. -/// Editing a *later* node is the supported case and the useful one: fix the -/// step that failed, continue, keep the prefix. A caller that changed anything -/// at or upstream of `failed_node` must re-run from the trigger instead, and -/// [`failure_boundary`] names that node so the check is possible. -/// -/// # Errors -/// [`EngineError::Capability`] when the thread has no checkpoint, or the -/// checkpoint schedules nothing to run — a completed run has no tail, and -/// asking it to continue is a caller mistake worth naming rather than a -/// silently empty outcome. Otherwise as [`run`]. -pub async fn retry_with_checkpointer( - workflow: &CompiledWorkflow, - capabilities: &Capabilities, - checkpointer: Arc>, - thread_id: &str, -) -> Result { - let observer = Arc::new(crate::observability::NoopObserver) as Arc; - let (outcome, _run_ids) = resume_with_checkpointer_inner( - workflow, - capabilities, - checkpointer, - thread_id, - Continuation::Retry, - None, - &observer, - ) - .await?; - Ok(outcome) -} - -/// Like [`retry_with_checkpointer`], but journaled and observed — the shape a -/// host that records runs actually needs. -/// -/// The journaled counterpart of -/// [`resume_with_checkpointer_journaled_observed`], and for the same reason: a -/// host whose run records are built from observed steps must see the continued -/// leg the same way it saw the first one, or the record it writes claims the -/// tail never ran. -/// -/// # Errors -/// Same as [`retry_with_checkpointer`]. -pub async fn retry_with_checkpointer_journaled_observed( - workflow: &CompiledWorkflow, - capabilities: &Capabilities, - checkpointer: Arc>, - thread_id: &str, - journal: Arc, - observer: &Arc, -) -> Result { - let (outcome, graph_run_ids) = resume_with_checkpointer_inner( - workflow, - capabilities, - checkpointer, - thread_id, - Continuation::Retry, - Some(journal), - observer, - ) - .await?; - Ok(JournaledRunOutcome { - outcome, - graph_run_ids, - }) -} - -/// Like [`retry_with_checkpointer`], but reports live progress to `observer`. -/// -/// The observer sees `on_step_finish` for every node that runs *after* the -/// failure boundary — which is the point: a host watching a continued run -/// should see the work that is actually happening, not a replay of the prefix -/// that is not. -/// -/// # Errors -/// Same as [`retry_with_checkpointer`]. -pub async fn retry_with_checkpointer_observed( - workflow: &CompiledWorkflow, - capabilities: &Capabilities, - checkpointer: Arc>, - thread_id: &str, - observer: &Arc, -) -> Result { - let (outcome, _run_ids) = resume_with_checkpointer_inner( - workflow, - capabilities, - checkpointer, - thread_id, - Continuation::Retry, - None, - observer, - ) - .await?; - Ok(outcome) -} +include!("resumable/continuation.rs"); diff --git a/crates/tinyflows/src/engine/resumable/continuation.rs b/crates/tinyflows/src/engine/resumable/continuation.rs new file mode 100644 index 00000000..1e9331b7 --- /dev/null +++ b/crates/tinyflows/src/engine/resumable/continuation.rs @@ -0,0 +1,306 @@ +impl Continuation { + /// The command that carries this continuation into the runtime. + fn command(self) -> Command { + match self { + Self::Approvals { approved, rejected } => { + // Approvals recorded for downstream visibility. On resume the + // interrupted gate is approved because the resume value reaches + // it via `NodeContext::resume`; the `with_update` mirrors + // `ResumableRun::resume` (the runtime ignores it on resume, so + // the resume value is the real approval channel). + let update = json!({ + "run": { "trigger": { "approvals": approved.clone() } } + }); + if !rejected.is_empty() { + tracing::info!(?rejected, "resuming with denied approval gate(s)"); + } + // Always a structured resume value carrying the explicit + // `approved` and `rejected` gate id lists. Each interrupted gate + // decides for itself: gates in `approved` proceed, gates in + // `rejected` route to their `error` port (or fail), and gates in + // neither stay pending. This is essential when several parallel + // gates are interrupted and the host resolves only some of them + // — a bare `true` would blanket-approve every interrupt + // regardless of the host's decision. + let value = json!({ "approved": approved, "rejected": rejected }); + Command::resume(value).with_update(update) + } + // Deliberately empty. A failed node is re-entered from its start + // with the state the boundary committed; a resume *value* would be + // delivered to `NodeContext::resume` and read as an approval + // decision by any gate that happened to be in the pending set. + Self::Retry => Command::new(), + } + } +} + +/// Shared implementation of the checkpointed continue path: rebuilds the graph +/// (optionally journaled), re-attaches the same `checkpointer`, and resumes +/// `thread_id`. Returns the outcome plus the resumed execution's +/// runtime-minted run ids. +async fn resume_with_checkpointer_inner( + workflow: &CompiledWorkflow, + capabilities: &Capabilities, + checkpointer: Arc>, + thread_id: &str, + continuation: Continuation, + journal: Option>, + observer: &Arc, +) -> Result<(RunOutcome, GraphRunIds)> { + let steps: Arc>> = Arc::new(Mutex::new(Vec::new())); + + // Rebuild the identical graph and re-attach the SAME checkpointer, so + // `resume` loads the state persisted under `thread_id`. Node handlers fire + // `observer.on_step_finish` for every node that runs after the interrupt + // boundary, so a host observer sees the resumed steps live. + let terminal_error: Arc>> = Arc::new(Mutex::new(None)); + let mut config = RunConfig::new(workflow)?.with_checkpointer(checkpointer, thread_id); + if let Some(journal) = journal { + config = config.with_journal(journal); + } + let (compiled, _trigger_id) = build_graph( + workflow, + capabilities, + observer, + &steps, + &terminal_error, + &config, + )?; + + let execution = compiled.resume(thread_id, continuation.command()).await; + let execution = match execution { + Ok(execution) => execution, + Err(error) => { + let structured = terminal_error + .lock() + .expect("terminal error mutex poisoned") + .take(); + return Err(structured.unwrap_or_else(|| EngineError::Capability(error.to_string()))); + } + }; + + let pending_approvals: Vec = execution + .interrupts + .iter() + .map(|interrupt| interrupt.id.clone()) + .collect(); + + let graph_run_ids = GraphRunIds { + run_id: execution.run_id.as_str().to_string(), + root_run_id: execution.root_run_id.as_str().to_string(), + }; + + Ok(( + RunOutcome { + output: execution.state, + pending_approvals, + // Checkpointed resume does not (yet) thread a caller token; a + // cancellable resume goes through `resume_cancellable`. + cancelled: false, + }, + graph_run_ids, + )) +} + +/// What a failed run left behind, and what it would take to continue it. +/// +/// A run that fails does not necessarily lose its work. On a checkpointed +/// thread the runtime folds the branches that already completed into committed +/// state and writes a **failure boundary** — a checkpoint whose pending nodes +/// are the node that failed and the not-yet-run tail of its step. Everything +/// before it is durable and does not have to happen twice. +/// +/// The engine reports the failure as an `Err`, which is the right shape for a +/// caller that just wants to know the run did not finish. This is the question +/// that error cannot answer: *is there something to continue, and where did it +/// stop?* Read it after a failed run to decide between fixing and retrying, +/// and re-running from the trigger. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FailureBoundary { + /// The node whose handler failed. + pub failed_node: String, + /// The error as the runtime rendered it, for diagnosis. + pub error: String, + /// The checkpoint holding the committed prefix — the id + /// [`ResumeTarget::Checkpoint`](crate::graph::ResumeTarget) addresses. + pub checkpoint_id: String, + /// Which superstep the run reached. + pub step: usize, + /// The nodes a continue would run: the failed one, and whatever else in + /// its step had not run when it aborted. + pub pending: Vec, +} + +/// Read the failure boundary a thread's latest checkpoint records, if it is one. +/// +/// `Ok(None)` for a thread that has no checkpoint, or whose latest is an +/// ordinary boundary — a completed run, or one paused at an approval gate. +/// Those are not failures and have nothing to continue *from a failure*. +/// +/// Deliberately a separate read rather than a field on the error. A failed run +/// already returns [`EngineError`], every caller handles that, and widening it +/// would make every one of them carry a concept most do not use. Asking +/// afterwards also reads the way the decision is actually made: the run +/// failed — is it worth continuing? +/// +/// # Errors +/// When the checkpointer cannot be read. +pub async fn failure_boundary( + checkpointer: &Arc>, + thread_id: &str, +) -> Result> { + let checkpoint = checkpointer + .get(thread_id, None) + .await + .map_err(|error| EngineError::Capability(error.to_string()))?; + let Some(checkpoint) = checkpoint else { + return Ok(None); + }; + // `failed_node` is what makes a boundary a *failure* boundary — an + // interrupt boundary and a terminal one both lack it. Reading the key + // rather than a status field keeps this to one checkpoint load. + let Some(failed_node) = checkpoint + .metadata + .get("failed_node") + .and_then(Value::as_str) + else { + return Ok(None); + }; + Ok(Some(FailureBoundary { + failed_node: failed_node.to_string(), + error: checkpoint + .metadata + .get("error") + .and_then(Value::as_str) + .unwrap_or_default() + .to_string(), + checkpoint_id: checkpoint.checkpoint_id.clone(), + step: checkpoint + .metadata + .get("step") + .and_then(Value::as_u64) + .and_then(|step| usize::try_from(step).ok()) + .unwrap_or(0), + pending: checkpoint + .next_nodes + .iter() + .map(ToString::to_string) + .collect(), + })) +} + +/// Continue a **failed** run from where it stopped: re-run the node that +/// failed and the not-yet-run tail of its step, on the state the failure +/// boundary committed. +/// +/// The counterpart of [`resume_with_checkpointer`] for the failure path. That +/// one answers a pause with a decision; this one answers a break with another +/// go, and carries no resume value — there is nothing to decide, only work to +/// redo. +/// +/// Two reasons to reach for this over re-running the workflow: +/// +/// * **Side effects.** A prefix that posted a comment, opened a pull request +/// or charged something does not do it twice. Re-running from the trigger is +/// not a neutral choice for a graph with effects in it; it is a second set +/// of them. +/// * **Cost.** A prefix step can be a whole coding session. Paying for it +/// again to reach the same failed node buys nothing. +/// +/// **The graph must be the one that failed.** Node handlers are rebuilt from +/// `workflow`, and the committed state is keyed by node id, so a `workflow` +/// whose prefix differs from the one that ran will re-enter the tail on state +/// it would never have produced — a run that goes green and is quietly wrong. +/// Editing a *later* node is the supported case and the useful one: fix the +/// step that failed, continue, keep the prefix. A caller that changed anything +/// at or upstream of `failed_node` must re-run from the trigger instead, and +/// [`failure_boundary`] names that node so the check is possible. +/// +/// # Errors +/// [`EngineError::Capability`] when the thread has no checkpoint, or the +/// checkpoint schedules nothing to run — a completed run has no tail, and +/// asking it to continue is a caller mistake worth naming rather than a +/// silently empty outcome. Otherwise as [`run`]. +pub async fn retry_with_checkpointer( + workflow: &CompiledWorkflow, + capabilities: &Capabilities, + checkpointer: Arc>, + thread_id: &str, +) -> Result { + let observer = Arc::new(crate::observability::NoopObserver) as Arc; + let (outcome, _run_ids) = resume_with_checkpointer_inner( + workflow, + capabilities, + checkpointer, + thread_id, + Continuation::Retry, + None, + &observer, + ) + .await?; + Ok(outcome) +} + +/// Like [`retry_with_checkpointer`], but journaled and observed — the shape a +/// host that records runs actually needs. +/// +/// The journaled counterpart of +/// [`resume_with_checkpointer_journaled_observed`], and for the same reason: a +/// host whose run records are built from observed steps must see the continued +/// leg the same way it saw the first one, or the record it writes claims the +/// tail never ran. +/// +/// # Errors +/// Same as [`retry_with_checkpointer`]. +pub async fn retry_with_checkpointer_journaled_observed( + workflow: &CompiledWorkflow, + capabilities: &Capabilities, + checkpointer: Arc>, + thread_id: &str, + journal: Arc, + observer: &Arc, +) -> Result { + let (outcome, graph_run_ids) = resume_with_checkpointer_inner( + workflow, + capabilities, + checkpointer, + thread_id, + Continuation::Retry, + Some(journal), + observer, + ) + .await?; + Ok(JournaledRunOutcome { + outcome, + graph_run_ids, + }) +} + +/// Like [`retry_with_checkpointer`], but reports live progress to `observer`. +/// +/// The observer sees `on_step_finish` for every node that runs *after* the +/// failure boundary — which is the point: a host watching a continued run +/// should see the work that is actually happening, not a replay of the prefix +/// that is not. +/// +/// # Errors +/// Same as [`retry_with_checkpointer`]. +pub async fn retry_with_checkpointer_observed( + workflow: &CompiledWorkflow, + capabilities: &Capabilities, + checkpointer: Arc>, + thread_id: &str, + observer: &Arc, +) -> Result { + let (outcome, _run_ids) = resume_with_checkpointer_inner( + workflow, + capabilities, + checkpointer, + thread_id, + Continuation::Retry, + None, + observer, + ) + .await?; + Ok(outcome) +} diff --git a/crates/tinyflows/src/gates/gates_tests.rs b/crates/tinyflows/src/gates/gates_tests.rs index d9d6e8d7..82d74550 100644 --- a/crates/tinyflows/src/gates/gates_tests.rs +++ b/crates/tinyflows/src/gates/gates_tests.rs @@ -85,9 +85,9 @@ fn a_real_expression_is_not_mistaken_for_prose() { #[test] fn reading_an_agents_output_without_the_envelope_is_refused() { let graph = graph(json!([ - { "id": "fetch", "kind": "agent", "name": "Fetch", "config": { "prompt": "get it" } }, + { "id": "fetch-agent", "kind": "agent", "name": "Fetch", "config": { "prompt": "get it" } }, { "id": "notify", "kind": "tool_call", "name": "Notify", - "config": { "slug": "demo:echo", "args": { "text": "=nodes.fetch.item.title" } } }, + "config": { "slug": "demo:echo", "args": { "text": "=nodes.fetch-agent.item.title" } } }, ])); let failures = failures(&graph); @@ -96,7 +96,7 @@ fn reading_an_agents_output_without_the_envelope_is_refused() { assert!(failures[0].contains("args.text"), "{failures:?}"); // The message has to carry the correction, not just the complaint. assert!( - failures[0].contains("=nodes.fetch.item.json.title"), + failures[0].contains("=nodes.fetch-agent.item.json.title"), "{failures:?}" ); } @@ -167,16 +167,20 @@ fn a_binding_to_a_node_that_does_not_exist_is_left_to_the_engine() { #[test] fn an_expression_that_is_not_a_node_binding_is_not_second_guessed() { let graph = graph(json!([ + { "id": "fetch", "kind": "agent", "name": "Fetch", "config": {} }, { "id": "notify", "kind": "tool_call", "name": "Notify", "config": { "slug": "demo:echo", "args": { "text": "=.item.text | ascii_downcase", - "count": "=run.trigger.n" } } }, + "count": "=run.trigger.n", + "fallback": "=nodes.fetch.item.missing // \"fallback\"" } } }, ])); // A gate that guessed at arbitrary jq would refuse graphs that work. assert!(failures(&graph).is_empty(), "{:?}", failures(&graph)); } +include!("indexed_binding_tests.rs"); + // ---- the error surface ---- #[test] diff --git a/crates/tinyflows/src/gates/indexed_binding_tests.rs b/crates/tinyflows/src/gates/indexed_binding_tests.rs new file mode 100644 index 00000000..02feeef5 --- /dev/null +++ b/crates/tinyflows/src/gates/indexed_binding_tests.rs @@ -0,0 +1,40 @@ +#[test] +fn an_indexed_missing_envelope_binding_is_still_rejected() { + let graph = graph(json!([ + { "id": "fetch", "kind": "agent", "name": "Fetch", "config": {} }, + { "id": "notify", "kind": "tool_call", "name": "Notify", + "config": { "slug": "demo:echo", + "args": { "text": "=nodes.fetch.item.results[0].title" } } }, + ])); + + let failures = failures(&graph); + assert_eq!(failures.len(), 1, "{failures:?}"); + assert!(failures[0].contains("results[0].title"), "{failures:?}"); +} + +#[test] +fn an_indexed_array_inside_the_envelope_is_accepted() { + let graph = graph(json!([ + { "id": "fetch", "kind": "agent", "name": "Fetch", "config": { + "output_parser": { "schema": { + "type": "array", + "items": { "type": "object", "properties": { + "title": { "type": "string" } + } } + } } + } }, + { "id": "notify", "kind": "tool_call", "name": "Notify", + "config": { "slug": "demo:echo", + "args": { "text": "=nodes.fetch.item.json[0].title" } } }, + ])); + + assert!(failures(&graph).is_empty(), "{:?}", failures(&graph)); + assert_eq!( + parse_node_binding("=nodes.fetch.item.json[0].title"), + Some(crate::bindings::NodeBinding { + node_id: "fetch".to_string(), + through_envelope: true, + field_path: "[0].title".to_string(), + }) + ); +} diff --git a/crates/tinyflows/src/testkit/tools/registry.rs b/crates/tinyflows/src/testkit/tools/registry.rs index 7ca675fe..48a47c6e 100644 --- a/crates/tinyflows/src/testkit/tools/registry.rs +++ b/crates/tinyflows/src/testkit/tools/registry.rs @@ -404,258 +404,7 @@ impl TestkitRegistry { } } -/// A run trace, bounded and optionally narrowed to one node. -fn project_trace(trace: &RunTrace, node_id: Option<&str>) -> Value { - let steps: Vec<_> = match node_id { - Some(id) => trace.steps_for(id).into_iter().cloned().collect(), - None => trace.steps.clone(), - }; - let value = json!({ - "summary": trace.summary(), - "steps": steps, - "calls": trace.calls, - "diagnosis": trace.diagnosis, - }); - // Bounded because a run over a large payload would otherwise put the whole - // thing in a context window. - crate::evidence::bounded_evidence(&value) -} - -/// The null bindings, projected as the pointer they are meant to be. -fn null_binding_report(trace: &RunTrace) -> Vec { - trace - .null_bindings() - .into_iter() - .map(|(node, binding)| { - json!({ - "nodeId": node, - "location": binding.location, - "expression": binding.expression, - "readsFrom": binding.reads_from, - }) - }) - .collect() -} - -fn str_arg(args: &Value, name: &str) -> Result { - args.get(name) - .and_then(Value::as_str) - .map(str::to_string) - .ok_or_else(|| { - ToolError::new( - ToolErrorCode::InvalidArguments, - format!("missing required string argument {name:?}"), - ) - }) -} - -fn graph_arg(args: &Value) -> Result { - let graph = args.get("graph").ok_or_else(|| { - ToolError::new( - ToolErrorCode::InvalidArguments, - "missing required argument \"graph\"".to_string(), - ) - })?; - serde_json::from_value(graph.clone()).map_err(|err| { - ToolError::new( - ToolErrorCode::InvalidGraph, - format!("the graph did not parse: {err}"), - ) - }) -} - -fn run_input(args: &Value) -> RunInput { - let mut input = RunInput::new(args.get("trigger").cloned().unwrap_or(Value::Null)); - if let Some(Value::Object(inputs)) = args.get("inputs") { - input.inputs = inputs.clone(); - } - if let Some(Value::Array(approvals)) = args.get("approvals") { - input.approvals = approvals - .iter() - .filter_map(Value::as_str) - .map(str::to_string) - .collect(); - } - input -} - -/// Build the mock rules a call programmed. -fn mocks_from(args: &Value) -> Result { - let mut mocks = MockCaps::new(); - let Some(Value::Array(rules)) = args.get("mocks") else { - return Ok(mocks); - }; - for rule in rules { - let capability = rule - .get("capability") - .and_then(Value::as_str) - .ok_or_else(|| { - ToolError::new( - ToolErrorCode::InvalidArguments, - "each mock rule needs a capability".to_string(), - ) - })?; - let target = rule.get("target").and_then(Value::as_str).unwrap_or("*"); - let respond = respond_from(rule)?; - mocks = match capability { - "tools" => mocks.on_tool(target, respond), - "http" => mocks.on_http(target, respond), - "llm" => mocks.on_llm(respond), - "agent" => mocks.on_agent(target, respond), - "code" => mocks.on_code(respond), - "shell" => mocks.on_shell(respond), - other => { - return Err(ToolError::new( - ToolErrorCode::InvalidArguments, - format!("unknown capability {other:?}"), - )); - } - }; - if let Some(node_id) = rule.get("node_id").and_then(Value::as_str) { - mocks = mocks.only_from(node_id); - } - } - Ok(mocks) -} - -/// One programmed response. -fn respond_from(rule: &Value) -> Result { - let base = if let Some(Value::Array(entries)) = rule.get("sequence") { - let mut sequence = Vec::new(); - for entry in entries { - sequence.push(respond_from(entry)?); - } - Respond::Sequence(sequence) - } else if let Some(error) = rule.get("error").and_then(Value::as_str) { - Respond::error(error) - } else if let Some(schema) = rule.get("schema") { - Respond::schema(schema.clone()) - } else if let Some(value) = rule.get("value") { - Respond::value(value.clone()) - } else { - Respond::Echo - }; - Ok(match rule.get("delay_ms").and_then(Value::as_u64) { - Some(ms) => Respond::after(Duration::from_millis(ms), base), - None => base, - }) -} - -/// The breakpoint a `flow_debug.breakpoint` call described. -fn breakpoint_spec(args: &Value) -> Result { - let any = args.get("any").and_then(Value::as_bool).unwrap_or(false); - let target = match (any, args.get("node_id").and_then(Value::as_str)) { - (true, _) => NodeTarget::Any, - (false, Some(id)) => NodeTarget::Id(id.to_string()), - (false, None) => { - return Err(ToolError::new( - ToolErrorCode::InvalidArguments, - "a breakpoint needs a node_id, or any:true for every node".to_string(), - )); - } - }; - - let on_error = args - .get("on_error") - .and_then(Value::as_bool) - .unwrap_or(false); - // An on-error breakpoint has to break *after* the node — that is the only - // phase at which a failure exists — so default the phase to match the - // intent rather than making the caller work it out. - let before = args - .get("before") - .and_then(Value::as_bool) - .unwrap_or(!on_error); - let after = args - .get("after") - .and_then(Value::as_bool) - .unwrap_or(on_error); - - let mut conditions = Vec::new(); - if on_error { - conditions.push(Condition::OnError); - } - if let Some(n) = args.get("activation").and_then(Value::as_u64) { - conditions.push(Condition::Activation(n as u32)); - } - if let Some(expr) = args.get("expr").and_then(Value::as_str) { - conditions.push(Condition::Expr(expr.to_string())); - } - let condition = match conditions.len() { - 0 => Condition::Always, - 1 => conditions.remove(0), - _ => Condition::All(conditions), - }; - - Ok(BreakpointSpec { - target, - before, - after, - condition, - mode: PauseMode::Live, - max_hits: args - .get("once") - .and_then(Value::as_bool) - .unwrap_or(false) - .then_some(1), - }) -} - -/// The command a `flow_debug.release` call described. -fn debug_command(args: &Value) -> Result { - let command = args.get("command").and_then(Value::as_str).ok_or_else(|| { - ToolError::new( - ToolErrorCode::InvalidArguments, - "release needs a command".to_string(), - ) - })?; - Ok(match command { - "continue" => DebugCommand::Continue, - "step" => DebugCommand::Step, - "skip" => DebugCommand::Skip, - "detach" => DebugCommand::Detach, - "fail" => DebugCommand::Fail( - args.get("message") - .and_then(Value::as_str) - .unwrap_or("failed from the debugger") - .to_string(), - ), - "patch" => DebugCommand::Patch( - args.get("patch") - .cloned() - .unwrap_or(Value::Object(Map::new())), - ), - "override" => { - let items = match args.get("items") { - Some(Value::Array(values)) => values.iter().map(json_to_item).collect(), - Some(single) => vec![json_to_item(single)], - None => Vec::new(), - }; - DebugCommand::Override { - items, - port: args.get("port").and_then(Value::as_str).map(str::to_string), - } - } - other => { - return Err(ToolError::new( - ToolErrorCode::InvalidArguments, - format!("unknown command {other:?}"), - )); - } - }) -} - -/// Accept either a full item (`{"json": …}`) or a bare payload. -/// -/// An agent writing `items: [{"ok": true}]` means the payload; requiring the -/// envelope would be a papercut with no upside. -fn json_to_item(value: &Value) -> Item { - match value.get("json") { - Some(payload) => Item::new(payload.clone()), - None => Item::new(value.clone()), - } -} - +include!("registry/helpers.rs"); #[cfg(test)] #[path = "registry_tests.rs"] mod tests; diff --git a/crates/tinyflows/src/testkit/tools/registry/helpers.rs b/crates/tinyflows/src/testkit/tools/registry/helpers.rs new file mode 100644 index 00000000..cc68a957 --- /dev/null +++ b/crates/tinyflows/src/testkit/tools/registry/helpers.rs @@ -0,0 +1,252 @@ +/// A run trace, bounded and optionally narrowed to one node. +fn project_trace(trace: &RunTrace, node_id: Option<&str>) -> Value { + let steps: Vec<_> = match node_id { + Some(id) => trace.steps_for(id).into_iter().cloned().collect(), + None => trace.steps.clone(), + }; + let value = json!({ + "summary": trace.summary(), + "steps": steps, + "calls": trace.calls, + "diagnosis": trace.diagnosis, + }); + // Bounded because a run over a large payload would otherwise put the whole + // thing in a context window. + crate::evidence::bounded_evidence(&value) +} + +/// The null bindings, projected as the pointer they are meant to be. +fn null_binding_report(trace: &RunTrace) -> Vec { + trace + .null_bindings() + .into_iter() + .map(|(node, binding)| { + json!({ + "nodeId": node, + "location": binding.location, + "expression": binding.expression, + "readsFrom": binding.reads_from, + }) + }) + .collect() +} + +fn str_arg(args: &Value, name: &str) -> Result { + args.get(name) + .and_then(Value::as_str) + .map(str::to_string) + .ok_or_else(|| { + ToolError::new( + ToolErrorCode::InvalidArguments, + format!("missing required string argument {name:?}"), + ) + }) +} + +fn graph_arg(args: &Value) -> Result { + let graph = args.get("graph").ok_or_else(|| { + ToolError::new( + ToolErrorCode::InvalidArguments, + "missing required argument \"graph\"".to_string(), + ) + })?; + serde_json::from_value(graph.clone()).map_err(|err| { + ToolError::new( + ToolErrorCode::InvalidGraph, + format!("the graph did not parse: {err}"), + ) + }) +} + +fn run_input(args: &Value) -> RunInput { + let mut input = RunInput::new(args.get("trigger").cloned().unwrap_or(Value::Null)); + if let Some(Value::Object(inputs)) = args.get("inputs") { + input.inputs = inputs.clone(); + } + if let Some(Value::Array(approvals)) = args.get("approvals") { + input.approvals = approvals + .iter() + .filter_map(Value::as_str) + .map(str::to_string) + .collect(); + } + input +} + +/// Build the mock rules a call programmed. +fn mocks_from(args: &Value) -> Result { + let mut mocks = MockCaps::new(); + let Some(Value::Array(rules)) = args.get("mocks") else { + return Ok(mocks); + }; + for rule in rules { + let capability = rule + .get("capability") + .and_then(Value::as_str) + .ok_or_else(|| { + ToolError::new( + ToolErrorCode::InvalidArguments, + "each mock rule needs a capability".to_string(), + ) + })?; + let target = rule.get("target").and_then(Value::as_str).unwrap_or("*"); + let respond = respond_from(rule)?; + mocks = match capability { + "tools" => mocks.on_tool(target, respond), + "http" => mocks.on_http(target, respond), + "llm" => mocks.on_llm(respond), + "agent" => mocks.on_agent(target, respond), + "code" => mocks.on_code(respond), + "shell" => mocks.on_shell(respond), + other => { + return Err(ToolError::new( + ToolErrorCode::InvalidArguments, + format!("unknown capability {other:?}"), + )); + } + }; + if let Some(node_id) = rule.get("node_id").and_then(Value::as_str) { + mocks = mocks.only_from(node_id); + } + } + Ok(mocks) +} + +/// One programmed response. +fn respond_from(rule: &Value) -> Result { + let base = if let Some(Value::Array(entries)) = rule.get("sequence") { + let mut sequence = Vec::new(); + for entry in entries { + sequence.push(respond_from(entry)?); + } + Respond::Sequence(sequence) + } else if let Some(error) = rule.get("error").and_then(Value::as_str) { + Respond::error(error) + } else if let Some(schema) = rule.get("schema") { + Respond::schema(schema.clone()) + } else if let Some(value) = rule.get("value") { + Respond::value(value.clone()) + } else { + Respond::Echo + }; + Ok(match rule.get("delay_ms").and_then(Value::as_u64) { + Some(ms) => Respond::after(Duration::from_millis(ms), base), + None => base, + }) +} + +/// The breakpoint a `flow_debug.breakpoint` call described. +fn breakpoint_spec(args: &Value) -> Result { + let any = args.get("any").and_then(Value::as_bool).unwrap_or(false); + let target = match (any, args.get("node_id").and_then(Value::as_str)) { + (true, _) => NodeTarget::Any, + (false, Some(id)) => NodeTarget::Id(id.to_string()), + (false, None) => { + return Err(ToolError::new( + ToolErrorCode::InvalidArguments, + "a breakpoint needs a node_id, or any:true for every node".to_string(), + )); + } + }; + + let on_error = args + .get("on_error") + .and_then(Value::as_bool) + .unwrap_or(false); + // An on-error breakpoint has to break *after* the node — that is the only + // phase at which a failure exists — so default the phase to match the + // intent rather than making the caller work it out. + let before = args + .get("before") + .and_then(Value::as_bool) + .unwrap_or(!on_error); + let after = args + .get("after") + .and_then(Value::as_bool) + .unwrap_or(on_error); + + let mut conditions = Vec::new(); + if on_error { + conditions.push(Condition::OnError); + } + if let Some(n) = args.get("activation").and_then(Value::as_u64) { + conditions.push(Condition::Activation(n as u32)); + } + if let Some(expr) = args.get("expr").and_then(Value::as_str) { + conditions.push(Condition::Expr(expr.to_string())); + } + let condition = match conditions.len() { + 0 => Condition::Always, + 1 => conditions.remove(0), + _ => Condition::All(conditions), + }; + + Ok(BreakpointSpec { + target, + before, + after, + condition, + mode: PauseMode::Live, + max_hits: args + .get("once") + .and_then(Value::as_bool) + .unwrap_or(false) + .then_some(1), + }) +} + +/// The command a `flow_debug.release` call described. +fn debug_command(args: &Value) -> Result { + let command = args.get("command").and_then(Value::as_str).ok_or_else(|| { + ToolError::new( + ToolErrorCode::InvalidArguments, + "release needs a command".to_string(), + ) + })?; + Ok(match command { + "continue" => DebugCommand::Continue, + "step" => DebugCommand::Step, + "skip" => DebugCommand::Skip, + "detach" => DebugCommand::Detach, + "fail" => DebugCommand::Fail( + args.get("message") + .and_then(Value::as_str) + .unwrap_or("failed from the debugger") + .to_string(), + ), + "patch" => DebugCommand::Patch( + args.get("patch") + .cloned() + .unwrap_or(Value::Object(Map::new())), + ), + "override" => { + let items = match args.get("items") { + Some(Value::Array(values)) => values.iter().map(json_to_item).collect(), + Some(single) => vec![json_to_item(single)], + None => Vec::new(), + }; + DebugCommand::Override { + items, + port: args.get("port").and_then(Value::as_str).map(str::to_string), + } + } + other => { + return Err(ToolError::new( + ToolErrorCode::InvalidArguments, + format!("unknown command {other:?}"), + )); + } + }) +} + +/// Accept either a full item (`{"json": …}`) or a bare payload. +/// +/// An agent writing `items: [{"ok": true}]` means the payload; requiring the +/// envelope would be a papercut with no upside. +fn json_to_item(value: &Value) -> Item { + match value.get("json") { + Some(payload) => Item::new(payload.clone()), + None => Item::new(value.clone()), + } +} + diff --git a/crates/tinyflows/tests/interception_e2e/interception_part_01_tests.rs b/crates/tinyflows/tests/interception_e2e/interception_part_01_tests.rs new file mode 100644 index 00000000..c8e5cd1f --- /dev/null +++ b/crates/tinyflows/tests/interception_e2e/interception_part_01_tests.rs @@ -0,0 +1,129 @@ +/// The frame can resolve a node's bindings without executing it — the +/// inspection a breakpoint needs, and the thing that turns "it produced null" +/// into a pointer at the binding that did. +#[tokio::test] +async fn a_frame_resolves_bindings_without_executing() { + struct Capture(Mutex>); + + #[async_trait] + impl StepInterceptor for Capture { + async fn intercept(&self, frame: StepFrame<'_>) -> StepAction { + if frame.phase == StepPhase::Before && frame.node.id == "call" { + let (_resolved, nulls) = frame.resolved_config(); + let mut seen = self.0.lock().expect("capture lock"); + for null in nulls { + seen.push((null.location, null.expression)); + } + } + StepAction::Continue { state_patch: None } + } + } + + let mut graph = graph(); + // A binding onto a field no upstream node produces: legal, resolves to + // null, and does nothing at run time. Exactly the failure a green run hides. + graph.nodes[1].config = json!({ + "slug": "svc.do", + "args": { "to": "=nodes.t.item.missing_field" } + }); + let compiled = compile(&graph).expect("compile"); + let hook = Arc::new(Capture(Mutex::new(Vec::new()))); + + run_intercepted( + &compiled, + json!({}), + &mock_capabilities(), + &(Arc::new(NoopObserver) as Arc), + CancellationToken::new(), + hook.clone(), + ) + .await + .expect("run"); + + let seen = hook.0.lock().expect("capture lock").clone(); + assert_eq!( + seen, + vec![( + "args.to".to_string(), + "=nodes.t.item.missing_field".to_string() + )], + "the frame should report the null binding and where it was written" + ); +} + +/// A node that failed once and then succeeded must not be reported as failed. +/// +/// Regression: the engine's retry loop keeps the last failed attempt's error +/// even after a later attempt succeeds, so an `After` frame that surfaced +/// `last_err` unconditionally showed a recovered node as a failed one — and +/// would fire every on-error breakpoint on it. +#[tokio::test] +async fn a_recovered_retry_reports_no_error_to_the_interceptor() { + use std::sync::atomic::{AtomicUsize, Ordering}; + + /// Fails the first call, succeeds thereafter. + struct Flaky(AtomicUsize); + + #[async_trait] + impl tinyflows::caps::ToolInvoker for Flaky { + async fn invoke( + &self, + _slug: &str, + _args: Value, + _conn: Option<&str>, + ) -> tinyflows::error::Result { + if self.0.fetch_add(1, Ordering::SeqCst) == 0 { + Err(tinyflows::error::EngineError::Capability( + "transient".into(), + )) + } else { + Ok(json!({ "ok": true })) + } + } + } + + /// Records whether the after-frame carried an error, per node. + struct SawError(Mutex>); + + #[async_trait] + impl StepInterceptor for SawError { + async fn intercept(&self, frame: StepFrame<'_>) -> StepAction { + if frame.phase == StepPhase::After { + self.0 + .lock() + .expect("lock") + .push((frame.node.id.clone(), frame.error.is_some())); + } + StepAction::Continue { state_patch: None } + } + } + + let mut graph = graph(); + graph.nodes[1].config = json!({ "slug": "svc.do", "retry": { "max_attempts": 2 } }); + let compiled = compile(&graph).expect("compile"); + + let mut caps = mock_capabilities(); + caps.tools = Arc::new(Flaky(AtomicUsize::new(0))); + let hook = Arc::new(SawError(Mutex::new(Vec::new()))); + + run_intercepted( + &compiled, + json!({}), + &caps, + &(Arc::new(NoopObserver) as Arc), + CancellationToken::new(), + hook.clone(), + ) + .await + .expect("the retry recovers, so the run completes"); + + let seen = hook.0.lock().expect("lock").clone(); + let call = seen + .iter() + .find(|(id, _)| id == "call") + .expect("the retrying node reports an after-frame"); + assert!( + !call.1, + "a node that recovered on retry must not surface an error to the interceptor" + ); +} diff --git a/crates/tinyflows/tests/interception_e2e.rs b/crates/tinyflows/tests/interception_e2e_tests.rs similarity index 73% rename from crates/tinyflows/tests/interception_e2e.rs rename to crates/tinyflows/tests/interception_e2e_tests.rs index 33af8d42..c5f51ff6 100644 --- a/crates/tinyflows/tests/interception_e2e.rs +++ b/crates/tinyflows/tests/interception_e2e_tests.rs @@ -389,132 +389,4 @@ async fn a_before_state_patch_is_visible_downstream() { ); } -/// The frame can resolve a node's bindings without executing it — the -/// inspection a breakpoint needs, and the thing that turns "it produced null" -/// into a pointer at the binding that did. -#[tokio::test] -async fn a_frame_resolves_bindings_without_executing() { - struct Capture(Mutex>); - - #[async_trait] - impl StepInterceptor for Capture { - async fn intercept(&self, frame: StepFrame<'_>) -> StepAction { - if frame.phase == StepPhase::Before && frame.node.id == "call" { - let (_resolved, nulls) = frame.resolved_config(); - let mut seen = self.0.lock().expect("capture lock"); - for null in nulls { - seen.push((null.location, null.expression)); - } - } - StepAction::Continue { state_patch: None } - } - } - - let mut graph = graph(); - // A binding onto a field no upstream node produces: legal, resolves to - // null, and does nothing at run time. Exactly the failure a green run hides. - graph.nodes[1].config = json!({ - "slug": "svc.do", - "args": { "to": "=nodes.t.item.missing_field" } - }); - let compiled = compile(&graph).expect("compile"); - let hook = Arc::new(Capture(Mutex::new(Vec::new()))); - - run_intercepted( - &compiled, - json!({}), - &mock_capabilities(), - &(Arc::new(NoopObserver) as Arc), - CancellationToken::new(), - hook.clone(), - ) - .await - .expect("run"); - - let seen = hook.0.lock().expect("capture lock").clone(); - assert_eq!( - seen, - vec![( - "args.to".to_string(), - "=nodes.t.item.missing_field".to_string() - )], - "the frame should report the null binding and where it was written" - ); -} - -/// A node that failed once and then succeeded must not be reported as failed. -/// -/// Regression: the engine's retry loop keeps the last failed attempt's error -/// even after a later attempt succeeds, so an `After` frame that surfaced -/// `last_err` unconditionally showed a recovered node as a failed one — and -/// would fire every on-error breakpoint on it. -#[tokio::test] -async fn a_recovered_retry_reports_no_error_to_the_interceptor() { - use std::sync::atomic::{AtomicUsize, Ordering}; - - /// Fails the first call, succeeds thereafter. - struct Flaky(AtomicUsize); - - #[async_trait] - impl tinyflows::caps::ToolInvoker for Flaky { - async fn invoke( - &self, - _slug: &str, - _args: Value, - _conn: Option<&str>, - ) -> tinyflows::error::Result { - if self.0.fetch_add(1, Ordering::SeqCst) == 0 { - Err(tinyflows::error::EngineError::Capability( - "transient".into(), - )) - } else { - Ok(json!({ "ok": true })) - } - } - } - - /// Records whether the after-frame carried an error, per node. - struct SawError(Mutex>); - - #[async_trait] - impl StepInterceptor for SawError { - async fn intercept(&self, frame: StepFrame<'_>) -> StepAction { - if frame.phase == StepPhase::After { - self.0 - .lock() - .expect("lock") - .push((frame.node.id.clone(), frame.error.is_some())); - } - StepAction::Continue { state_patch: None } - } - } - - let mut graph = graph(); - graph.nodes[1].config = json!({ "slug": "svc.do", "retry": { "max_attempts": 2 } }); - let compiled = compile(&graph).expect("compile"); - - let mut caps = mock_capabilities(); - caps.tools = Arc::new(Flaky(AtomicUsize::new(0))); - let hook = Arc::new(SawError(Mutex::new(Vec::new()))); - - run_intercepted( - &compiled, - json!({}), - &caps, - &(Arc::new(NoopObserver) as Arc), - CancellationToken::new(), - hook.clone(), - ) - .await - .expect("the retry recovers, so the run completes"); - - let seen = hook.0.lock().expect("lock").clone(); - let call = seen - .iter() - .find(|(id, _)| id == "call") - .expect("the retrying node reports an after-frame"); - assert!( - !call.1, - "a node that recovered on retry must not surface an error to the interceptor" - ); -} +include!("interception_e2e/interception_part_01_tests.rs");