From 3a9a8e3cb955b99254f0d0a2160e6dcfd7a8c7d8 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 18:13:11 -0700 Subject: [PATCH 01/26] Add shared-vfs skeleton with canonical interned paths Introduces a new zero-dependency crate that sits permanently at the bottom of the dependency stack, holding the generic virtual filesystem machinery: one shared error type, the value types backends exchange, and canonical interned virtual paths. Paths are normalized and interned the moment the API receives them, so claim lookups stay pointer-cheap and aliases cannot slip past the claims tables. The only way to form a path is through a crate-private canonicalizer, so canonicalization at receipt is enforced by visibility rather than convention. A self-policing test reads the crate manifest and fails if any dependency table gains an entry. - `crates/shared-vfs/Cargo.toml` declares an empty `[dependencies]` table carrying the zero-dependency rule in comments; `the_manifest_declares_no_dependencies` reads the manifest through `CARGO_MANIFEST_DIR` and fails if any dependency table carries an entry. - `VfsPath` wraps an interned id so equality is an integer compare; `canonicalize` is crate-private, making it the only way to form one. - `Interner` is hand-rolled on std behind a poison-safe `OnceLock` mutex; each string is leaked once, so resolution is a vector index and a panicking writer cannot leave the tables half-updated. - `FileType` names all seven POSIX kinds with no catch-all, so an exotic backend node cannot be hidden. - `Stat` uses optional fields for mode, modified, and created, so a backend that does not track a field says `None` rather than fabricating a value. - `canonicalize` treats backslashes as separators, collapses duplicates, removes dot segments, pops exactly one segment per dotdot, and rejects traversal past the root plus relative and empty paths; case is preserved and significant. Ten tests cover the matrix and interning identity. - `canonicalize` and `Interner::intern` carry `#[allow(dead_code)]`: only tests call them today, with the production caller still to arrive. Design: new facade @ crates/shared-vfs/src/lib.rs boundary: pub Design: new global-state @ crates/shared-vfs/src/path.rs::interner Design: new newtype @ crates/shared-vfs/src/path.rs::VfsPath boundary: pub Design: new value-object @ crates/shared-vfs/src/path.rs::VfsPath boundary: pub Design: new newtype @ crates/shared-vfs/src/path.rs::VfsPathBuf boundary: pub Design: new value-object @ crates/shared-vfs/src/path.rs::VfsPathBuf boundary: pub Design: new value-object @ crates/shared-vfs/src/types.rs::FileType boundary: pub Design: new bag-of-state @ crates/shared-vfs/src/types.rs::GrepQuery boundary: pub Deferred: canonicalize and Interner::intern carry #[allow(dead_code)] with no production caller until Access arrives Plan: vibe/2026-09-11-3-vfs-foundation.md --- Cargo.lock | 4 + crates/shared-vfs/AGENTS.md | 7 + crates/shared-vfs/Cargo.toml | 16 + crates/shared-vfs/src/error.rs | 51 +++ crates/shared-vfs/src/lib.rs | 48 ++ crates/shared-vfs/src/path.rs | 248 ++++++++++ crates/shared-vfs/src/types.rs | 104 +++++ vibe/2026-09-11-3-vfs-foundation.md | 678 ++++++++++++++++++++++++++++ vibe/ACTIVE | 1 + vibe/vibe-ledger.md | 7 + 10 files changed, 1164 insertions(+) create mode 100644 crates/shared-vfs/AGENTS.md create mode 100644 crates/shared-vfs/Cargo.toml create mode 100644 crates/shared-vfs/src/error.rs create mode 100644 crates/shared-vfs/src/lib.rs create mode 100644 crates/shared-vfs/src/path.rs create mode 100644 crates/shared-vfs/src/types.rs create mode 100644 vibe/2026-09-11-3-vfs-foundation.md create mode 100644 vibe/ACTIVE diff --git a/Cargo.lock b/Cargo.lock index a7c88fb88..8a0c98c35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6101,6 +6101,10 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "shared-vfs" +version = "0.3.0" + [[package]] name = "shellexpand" version = "3.1.2" diff --git a/crates/shared-vfs/AGENTS.md b/crates/shared-vfs/AGENTS.md new file mode 100644 index 000000000..c35ff313d --- /dev/null +++ b/crates/shared-vfs/AGENTS.md @@ -0,0 +1,7 @@ +# shared-vfs + +Generic virtual filesystem machinery: the permanent bottom of the dependency stack. + +- std only. No dependencies, workspace or external. The manifest test enforces this; never weaken it. +- No promptforge policy: no /_promptforge paths, no Store, no run concepts. +- The public surface is load-bearing: add defaulted methods, never change existing signatures. Every edit rebuilds the whole stack. diff --git a/crates/shared-vfs/Cargo.toml b/crates/shared-vfs/Cargo.toml new file mode 100644 index 000000000..aa3cbf3fb --- /dev/null +++ b/crates/shared-vfs/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "shared-vfs" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +description = "PromptForge shared virtual filesystem machinery: canonical interned paths, claims, routing, and backends" + +# Zero-dependency rule: std only. No dependencies, workspace or external. +# The manifest test enforces this; never weaken it. +[dependencies] + +[lints] +workspace = true diff --git a/crates/shared-vfs/src/error.rs b/crates/shared-vfs/src/error.rs new file mode 100644 index 000000000..220b241ed --- /dev/null +++ b/crates/shared-vfs/src/error.rs @@ -0,0 +1,51 @@ +//! The error type shared by every VFS layer. + +use std::fmt; + +/// The one error type returned by every virtual filesystem operation. +/// +/// `#[non_exhaustive]` so new kinds can ship without breaking match arms +/// in downstream crates; the public surface of this crate is load-bearing. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum VfsError { + /// The path does not exist in the serving backend. + NotFound(String), + /// The operation is not permitted: a read-only mount or a policy denial. + PermissionDenied(String), + /// The path already exists where creation required absence. + AlreadyExists(String), + /// The path is malformed or escapes the virtual namespace root. + InvalidPath(String), + /// A directory operation named a non-directory. + NotADirectory(String), + /// A file operation named a directory. + IsADirectory(String), + /// A directory removal without `recursive` named a non-empty directory. + DirectoryNotEmpty(String), + /// The serving backend does not implement the operation. + Unsupported(String), + /// The operation conflicts with another live identity's claim. + Conflict(String), + /// The serving backend failed for any other reason. + Backend(String), +} + +impl fmt::Display for VfsError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::NotFound(m) => write!(f, "not found: {m}"), + Self::PermissionDenied(m) => write!(f, "permission denied: {m}"), + Self::AlreadyExists(m) => write!(f, "already exists: {m}"), + Self::InvalidPath(m) => write!(f, "invalid path: {m}"), + Self::NotADirectory(m) => write!(f, "not a directory: {m}"), + Self::IsADirectory(m) => write!(f, "is a directory: {m}"), + Self::DirectoryNotEmpty(m) => write!(f, "directory not empty: {m}"), + Self::Unsupported(m) => write!(f, "unsupported operation: {m}"), + Self::Conflict(m) => write!(f, "conflicting claim: {m}"), + Self::Backend(m) => write!(f, "backend failure: {m}"), + } + } +} + +impl std::error::Error for VfsError {} diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs new file mode 100644 index 000000000..8009a961d --- /dev/null +++ b/crates/shared-vfs/src/lib.rs @@ -0,0 +1,48 @@ +//! Generic virtual filesystem machinery: canonical interned paths, the +//! claims model, the mount router, and backends. +//! +//! This crate is the permanent bottom of the dependency stack: std only, +//! no workspace or external crates, and no promptforge policy (no +//! `/_promptforge` paths, no Store, no run concepts). + +mod error; +mod path; +mod types; + +pub use error::VfsError; +pub use path::{VfsPath, VfsPathBuf}; +pub use types::{Entry, FileType, GrepMatch, GrepQuery, GrepResults, Stat}; + +#[cfg(test)] +mod tests { + /// The zero-dependency rule is load-bearing: this crate compiles alone + /// and never rebuilds for a dependency rev, so the manifest must never + /// declare a dependency. This test reads the crate's own Cargo.toml and + /// fails if any dependency table carries an entry. + #[test] + fn the_manifest_declares_no_dependencies() -> Result<(), std::io::Error> { + let manifest = std::fs::read_to_string( + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("Cargo.toml"), + )?; + let mut section = String::new(); + for raw_line in manifest.lines() { + let line = raw_line.trim(); + if line.starts_with('[') { + section = line.trim_matches(['[', ']']).to_owned(); + continue; + } + if line.is_empty() || line.starts_with('#') { + continue; + } + let is_dependency_table = section == "dependencies" + || section == "dev-dependencies" + || section == "build-dependencies" + || (section.starts_with("target.") && section.ends_with(".dependencies")); + assert!( + !is_dependency_table, + "zero-dependency rule violated: [{section}] declares `{line}`" + ); + } + Ok(()) + } +} diff --git a/crates/shared-vfs/src/path.rs b/crates/shared-vfs/src/path.rs new file mode 100644 index 000000000..924906018 --- /dev/null +++ b/crates/shared-vfs/src/path.rs @@ -0,0 +1,248 @@ +//! Canonical, interned virtual paths. +//! +//! Paths are canonicalized at the moment the API receives them and interned, +//! so claim lookups are pointer-cheap and aliases cannot slip past the +//! claims tables. The only way to form a [`VfsPath`] is through +//! `canonicalize`, which is crate-private: canonicalization at receipt is +//! enforced by visibility, not convention. + +use std::collections::HashMap; +use std::fmt; +use std::sync::{Mutex, MutexGuard, OnceLock, PoisonError}; + +use crate::error::VfsError; + +/// Hand-rolled string interner on std. Strings are leaked once each, so +/// resolution is a vector index and equality is an integer compare. +struct Interner { + ids: HashMap<&'static str, u32>, + strings: Vec<&'static str>, +} + +impl Interner { + fn new() -> Self { + Self { + ids: HashMap::new(), + strings: Vec::new(), + } + } + + // Callers arrive with `Access` in a later step; only tests intern today. + #[allow(dead_code)] + fn intern(&mut self, s: &str) -> u32 { + if let Some(&id) = self.ids.get(s) { + return id; + } + let leaked: &'static str = Box::leak(s.into()); + let Ok(id) = u32::try_from(self.strings.len()) else { + panic!("vfs path interner exhausted") + }; + self.strings.push(leaked); + self.ids.insert(leaked, id); + id + } + + fn resolve(&self, id: u32) -> &'static str { + match self.strings.get(id as usize) { + Some(s) => s, + None => panic!("vfs path id {id} was never interned"), + } + } +} + +/// Poison-safe lock: each guard scope is one complete mutation, so a +/// panicking writer cannot leave the tables half-updated and recovery is +/// safe. +fn interner() -> MutexGuard<'static, Interner> { + static INTERNER: OnceLock> = OnceLock::new(); + INTERNER + .get_or_init(|| Mutex::new(Interner::new())) + .lock() + .unwrap_or_else(PoisonError::into_inner) +} + +/// Canonical, interned virtual path. Produced by `canonicalize` at the +/// moment the API receives a path; interning makes claim lookups +/// pointer-cheap and guarantees alias detection. +#[derive(Clone, Copy, PartialEq, Eq, Hash)] +pub struct VfsPath { + id: u32, +} + +impl VfsPath { + /// Returns the canonical string for this path. + #[must_use] + pub fn as_str(&self) -> &'static str { + interner().resolve(self.id) + } + + /// Returns an owned copy of this path. + #[must_use] + pub fn to_buf(&self) -> VfsPathBuf { + VfsPathBuf(self.as_str().into()) + } +} + +impl fmt::Debug for VfsPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "VfsPath({:?})", self.as_str()) + } +} + +impl fmt::Display for VfsPath { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +/// Owned canonical virtual path, for places that outlive an interned +/// reference or arrive owned (grep roots, symlink targets). +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct VfsPathBuf(String); + +impl VfsPathBuf { + /// Returns the canonical string. + #[must_use] + pub fn as_str(&self) -> &str { + &self.0 + } +} + +impl From for VfsPathBuf { + fn from(path: VfsPath) -> Self { + path.to_buf() + } +} + +impl fmt::Display for VfsPathBuf { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(&self.0) + } +} + +/// Canonicalizes a virtual path at API receipt. +/// +/// The internal namespace is POSIX-shaped: rooted, forward slashes, strict. +/// The lexical rules are: backslashes from Windows hosts count as +/// separators; duplicate separators collapse; `.` segments vanish; `..` +/// pops exactly one segment and popping past the root is rejected; a +/// trailing slash is dropped; the root canonicalizes to itself. Case is +/// preserved and significant (POSIX semantics): paths differing only in +/// case are distinct. Relative and empty paths are rejected. +// Callers arrive with `Access` in a later step; only tests canonicalize today. +#[allow(dead_code)] +pub(crate) fn canonicalize(path: &str) -> Result { + if path.is_empty() { + return Err(VfsError::InvalidPath("empty path".into())); + } + let normalized = path.replace('\\', "/"); + if !normalized.starts_with('/') { + return Err(VfsError::InvalidPath(format!( + "relative path is not in the virtual namespace: {path:?}" + ))); + } + let mut segments: Vec<&str> = Vec::new(); + for segment in normalized.split('/') { + match segment { + "" | "." => {} + ".." => { + if segments.pop().is_none() { + return Err(VfsError::InvalidPath(format!( + "path escapes the namespace root: {path:?}" + ))); + } + } + _ => segments.push(segment), + } + } + let canonical = if segments.is_empty() { + "/".to_owned() + } else { + let mut s = String::with_capacity(normalized.len() + 1); + for segment in &segments { + s.push('/'); + s.push_str(segment); + } + s + }; + let id = interner().intern(&canonical); + Ok(VfsPath { id }) +} + +#[cfg(test)] +mod tests { + use super::{VfsPath, canonicalize}; + use crate::VfsError; + + fn canonical(path: &str) -> Result { + Ok(canonicalize(path)?.as_str().to_owned()) + } + + #[test] + fn the_namespace_root_canonicalizes_to_itself() -> Result<(), VfsError> { + assert_eq!(canonical("/")?, "/"); + Ok(()) + } + + #[test] + fn duplicate_separators_collapse_to_one() -> Result<(), VfsError> { + assert_eq!(canonical("/a//b///c")?, "/a/b/c"); + Ok(()) + } + + #[test] + fn dot_segments_are_removed() -> Result<(), VfsError> { + assert_eq!(canonical("/a/./b/./c")?, "/a/b/c"); + Ok(()) + } + + #[test] + fn dotdot_pops_exactly_one_segment() -> Result<(), VfsError> { + assert_eq!(canonical("/a/b/../c")?, "/a/c"); + assert_eq!(canonical("/a/..")?, "/"); + Ok(()) + } + + #[test] + fn a_trailing_slash_is_dropped() -> Result<(), VfsError> { + assert_eq!(canonical("/a/b/")?, "/a/b"); + Ok(()) + } + + #[test] + fn backslashes_from_windows_hosts_are_separators() -> Result<(), VfsError> { + assert_eq!(canonical("/a\\b/c")?, "/a/b/c"); + Ok(()) + } + + #[test] + fn traversal_past_the_root_is_rejected() { + assert!(canonicalize("/..").is_err()); + assert!(canonicalize("/a/../../b").is_err()); + } + + #[test] + fn relative_and_empty_paths_are_rejected() { + assert!(canonicalize("").is_err()); + assert!(canonicalize("a/b").is_err()); + assert!(canonicalize("./a").is_err()); + } + + #[test] + fn case_is_preserved_and_significant() -> Result<(), VfsError> { + assert_eq!(canonical("/ReadMe.md")?, "/ReadMe.md"); + let upper: VfsPath = canonicalize("/ReadMe.md")?; + let lower: VfsPath = canonicalize("/readme.md")?; + assert_ne!(upper, lower); + Ok(()) + } + + #[test] + fn identical_paths_intern_to_one_entry() -> Result<(), VfsError> { + let first = canonicalize("/a/b")?; + let second = canonicalize("/a/./b/")?; + assert_eq!(first, second); + assert!(std::ptr::eq(first.as_str(), second.as_str())); + Ok(()) + } +} diff --git a/crates/shared-vfs/src/types.rs b/crates/shared-vfs/src/types.rs new file mode 100644 index 000000000..06a93a578 --- /dev/null +++ b/crates/shared-vfs/src/types.rs @@ -0,0 +1,104 @@ +//! Value types exchanged with backends: entries, metadata, and grep. + +use std::time::SystemTime; + +use crate::path::VfsPathBuf; + +/// The seven POSIX kinds, named rather than lumped: a virtual `/dev/null` +/// (char device) is a plausible backend, and an `Other` kind would hide it. +/// The engine adapter maps the first four directly and the three specials +/// to `File` with a trace. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum FileType { + /// A regular file. + File, + /// A directory. + Directory, + /// A symbolic link. + Symlink, + /// A named pipe. + Fifo, + /// A socket. + Socket, + /// A character device. + CharDevice, + /// A block device. + BlockDevice, +} + +/// Metadata for one path. +/// +/// Options preserve honesty: a backend that does not track a field says +/// `None` rather than fabricating (an invented mtime is nondeterministic; +/// a constant one makes `ls -t` sort garbage). +#[non_exhaustive] +#[derive(Debug, Clone)] +pub struct Stat { + /// What kind of node this is. + pub file_type: FileType, + /// Size in bytes. + pub size: u64, + /// POSIX mode bits, when the backend tracks them. + pub mode: Option, + /// Last modification time, when the backend tracks it. + pub modified: Option, + /// Creation time, when the backend tracks it. + pub created: Option, +} + +/// One directory entry. +/// +/// `description` is the annotation column; it is `None` outside +/// `/_promptforge` and the engine adapter drops it. `Entry` is designed +/// to grow: annotations live here. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub struct Entry { + /// The entry's name within its directory. + pub name: String, + /// The entry's metadata. + pub stat: Stat, + /// Optional annotation shown beside the entry. + pub description: Option, +} + +/// One grep request against the namespace. +#[non_exhaustive] +#[derive(Debug, Clone)] +pub struct GrepQuery { + /// The text or pattern to search for. + pub pattern: String, + /// The directory the search is rooted at. + pub root: VfsPathBuf, + /// Whether `pattern` is a regular expression. + pub is_regex: bool, + /// Whether matching ignores case. + pub case_insensitive: bool, + /// An optional glob restricting which files are searched. + pub glob_filter: Option, + /// An optional cap on returned matches. + pub max_results: Option, +} + +/// One grep hit. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GrepMatch { + /// The path of the file containing the hit. + pub path: String, + /// The 1-based line number of the hit. + pub line_number: usize, + /// The full text of the matching line. + pub line: String, +} + +/// The outcome of one grep request. +#[non_exhaustive] +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct GrepResults { + /// The hits, in backend order. + pub matches: Vec, + /// Whether `max_results` cut the result set short. + pub truncated: bool, +} diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md new file mode 100644 index 000000000..5d31a72ac --- /dev/null +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -0,0 +1,678 @@ +--- +name: Promptforge Vfs Foundation +overview: "Focused foundation layer: a Vfs/VfsAccess trait pair with a claims model (ExecId-attributed access via an RAII Access capability; concurrent write conflicts are fatal), a mount router with universal cross-platform path handling that subsumes Bashkit's filesystem needs, a Policy layer with reason-carrying verdicts (AllowAll v1, modes in promptforge-vfs), a host filesystem backend, the Store rewritten as a public concrete facade (no trait) over Vfs mounted at /_promptforge/store with the Lua store table unchanged, and executor::run() taking VfsRef instead of StoreRef." +todos: + - id: vfs-trait + content: "Design the Vfs/VfsAccess trait pair and VfsRef handle: sync, bytes-based, Store-derived semantics plus stat/list/grep, acquire/release on the backend trait" + status: pending + - id: router + content: "Build the mount router: builder-style mount installation, longest-prefix dispatch, lazy per-mount acquire, nestable, universal path canonicalization (Windows/macOS/Linux)" + status: pending + - id: mem-backend + content: Implement the in-memory Vfs backend carrying former MemStore semantics + status: pending + - id: hostdir-backend + content: "IN SCOPE (without it the Vfs is useless): host backend in shared-vfs (std::fs is std - the zero-dependency rule holds) with HostBackend::identity() and HostBackend::rooted(dir). Stage 1 thin: direct std::fs ops, lexical+canonicalize containment, failure-atomic writes (sibling temp + rename). Stage 2 hardening toward the Bashkit RealFs oracle (resolver trio, symlink policies, Windows long paths and device names) as the threat model demands" + status: pending + - id: policy + content: "Implement the Policy layer: Op and Verdict (Deny/Ask carry reason strings), per-op check in Access before the claims check, AllowAll in shared-vfs, ModePolicy (Ask/Plan/Agent) in promptforge-vfs with a UI-flippable shared mode" + status: pending + - id: store-rewrite + content: Rewrite Store as a public concrete facade (no trait) over Vfs mounted at /_promptforge/store; Lua store table unchanged; WriteScope registry deleted in favor of the claims model; hosts seed/extract declared input/output keys through it + status: pending + - id: executor-api + content: Change executor::run() to take VfsRef instead of StoreRef; RunContext builds the Store facade internally + status: pending + - id: store-yield + content: Make Lua store operations leaf yields in the coroutine protocol (new Request/Answer variants, one dispatch arm, spawn_blocking over the sync Vfs); uniform for all backends - no inline fast path + status: pending + - id: bashkit-adapter + content: "SPIKE (deliverable is evidence, not integration): implement the Bashkit FsBackend adapter over VfsRef as a path dependency against the local clone at bashkit/, compile-check, and smoke-test an ls/cat/grep script against mounted backends. Toolchain already verified compatible: workspace runs stable 1.98, Bashkit's 1.95 pin applies only inside its own repo. Success proves the trait subsumes Bashkit; a mapping failure here is the spike working as intended, cheaply" + status: pending + - id: claims + content: "Implement the claims model: ExecId vending, Access RAII capability (acquire/spawn/borrow/move/drop), readers/writers claims tables keyed by interned canonical paths, canonicalize (stub-to-intern acceptable in v1), fatal determinism RunErrorKind, scheduler installation of the current access per chain step" + status: pending + - id: tests + content: Port the promptforge-store test suite onto the rewritten Store; add router and path-canonicalization matrices + status: pending +isProject: false +--- + +# Promptforge Vfs Foundation + + + +## Product Requirements + +One filesystem abstraction replaces the run-scoped Store trait and serves every future consumer: Lua, the model's tools, and the Bashkit engine. The Store survives as a public concrete facade with an unchanged Lua surface, mounted inside the namespace it used to stand apart from. The executor's public API pivots from StoreRef to VfsRef. Everything outside this foundation layer is explicitly deferred. + +- Problem and users: the harness needs one filesystem abstraction. Today the Store is a run-scoped, text-only trait; Bashkit needs a filesystem backend; run records and terminals need a virtual namespace; Windows path handling is a documented model-failure source. Users are promptforge authors and, downstream, the model inside every run. +- Goals: + - One `Vfs` trait plus a cloneable `VfsRef` handle, shaped like the proven Store/StoreRef pattern. + - The trait subsumes Bashkit's filesystem needs so the adapter is mechanical. + - A router installs multiple overlays into one namespace and behaves identically on Windows, macOS, and Linux. + - The Store becomes a public concrete facade (no trait) over Vfs, mounted at `/_promptforge/store`; the Lua `store` table is behaviorally unchanged, and hosts seed declared inputs and extract declared outputs through it. + - `executor::run()` takes `VfsRef` in place of `StoreRef`. +- Non-goals: the do_shell dispatcher, git builtins, approval policy, the SQLite run-record backend, terminal mirrors, and the Bashkit integration itself (adapter readiness only). +- Success criteria: the existing promptforge-store test suite passes against the rewritten Store; the executor's doc example compiles and runs with VfsRef; a Bashkit script can ls/cat/grep across mounted backends through the adapter. +- Constraints: sync trait (the backends are sync-native; the executor provides asynchrony at the yield boundary); bytes at the trait level with text conveniences above; existing Store semantics preserved exactly (error kinds, anchor-edit rules, numbered reads, write-conflict detection - now via the claims model); no loose files; shared-vfs is std-only with zero dependencies. +- Open questions: + - Where the Store's bytes physically live at rest (memory backend vs SQLite-backed) - memory for this phase. + +## Functional Specification + +Three actors share one handle with three different views: Lua through the unchanged store table, the executor through the facade it constructs, and later the engine through the adapter. The operation set is the Store's proven semantics widened by stat, annotated list, and first-class grep. Validation and canonicalization happen once at the handle boundary. The Lua-visible error vocabulary is frozen. + +- Actors and workflows: + - The Lua VM drives the `store` table exactly as today; every operation routes through the Store facade into Vfs. + - The executor receives one VfsRef per run and constructs the Store facade over it internally. + - The production host (pattern: wg21-paperflow/crates/papergate/src/app.rs) gets a stock VfsRef, seeds the prompt's declared input keys through `vfs.store()`, runs, and extracts the declared output keys - all without any real files; a missing declared output is an explicit contract error naming the prompt's promise. + - Bashkit (later phase) consumes the same handle through the adapter; the model's file tools (later phase) consume it directly. +- Inputs and outputs: + - The prompt's frontmatter declares the host contract: `input: { path, description }` and `output: { path, description }` keys name store paths the host seeds before the run and extracts after (pattern: wg21-paperflow/crates/papergate/papergate.md). + - Vfs core operations: read (bytes), read_range (1-based inclusive line range, verbatim and numbered variants), write, append, str_replace (anchor-unique), remove (strict: absent is NotFound, directories need the recursive flag; the Store facade keeps Lua's idempotent delete by mapping NotFound to Ok), exists, glob, list (entries with size, type, optional description), stat, grep (pattern over a subtree). + - grep has a default implementation (read and scan) so simple backends get it free; indexed backends override later. + - mkdir, rename, copy are first-class; symlink, read_link, chmod exist but default to unsupported. +- States and validation: + - Mount tables are fixed at construction; a per-run router is cheap Arc-clones plus the run's mounts. + - All Vfs access is attributed: every operation flows through an Access capability carrying an ExecId, and the claims tables (readers and writers maps from interned canonical path to live identities) live in the handle. The WriteScope registry is deleted. + - Virtual paths are validated and canonicalized at the VfsRef boundary, never inside backends. +- Errors and recovery: + - A determinism violation (two live claims on one path from different identities, at least one a write) is a fatal RunErrorKind that terminates the run instantly, naming the path, both identities, and both claim kinds. It is not catchable from Lua. + - The Lua-visible StoreError vocabulary is preserved unchanged (NotFound, InvalidPath, InvalidRange, AnchorNotFound, AnchorAmbiguous, InvalidAnchor, WriteRace, InvalidPattern). + - Vfs has its own error kind set; the Store facade maps between them at the boundary. + - Writes to read-only mounts fail with a clear read-only error, never partial application. +- Security and privacy behavior: + - Read-only mounts are enforced by the backend, not by convention. + - Backends never see uncanonicalized paths; traversal escape from a mount prefix is rejected at the router. +- Acceptance criteria: + - Lua prompts using the store table behave identically before and after the rewrite, verified by the ported test suite. + - A single VfsRef serves the Store mount, a memory scratch mount, and a second overlay simultaneously with correct longest-prefix routing. + + + + +## Technical Design + +The design mirrors the proven Store/StoreRef shape one level down: a sync trait behind a poison-safe cloneable handle, with a router that is itself an implementation so overlays nest. The Store inverts from public trait to public concrete facade over a prefix-scoped capability. All host-OS path complexity is confined to the host backend; the internal namespace is POSIX-shaped everywhere. Four crates are touched: shared-vfs and promptforge-vfs are new; promptforge-store and promptforge-core change. + +- Architecture: + - Vfs is a sync trait; VfsRef is the Arc-plus-mutex cloneable handle mirroring StoreRef's proven pattern, including poison-safe locking. + - The router is itself a Vfs implementation, so overlays nest and the executor sees one uniform handle. Mounts install builder-style at construction (mount consumes and returns self; the built table is immutable and Arc-shared); the router's acquire returns a routing access that resolves the longest-prefix mount per operation and acquires each backend's access lazily on first touch. Claims live above the router: the Access wrapper checks the fully-resolved canonical path before routing, so conflicts are caught regardless of which backend serves the path. + - The Store inverts: from public trait with pluggable backends to a public concrete facade (no trait) over a prefix-scoped VfsRef, exposed as vfs.store(). + - All access is capability-based: the backend trait Vfs has only acquire/release/read_only; every operation lives on the backend's VfsAccess trait object, and the public RAII capability Access wraps it with the ExecId, the claims check, and per-call locking. Drop releases the identity and its claims - cancellation, panics, and early returns cannot leak claims. +- Modules and interfaces: + - Two crates, split generic machinery from promptforge policy: + - `shared-vfs` (new crate at promptforge/crates/shared-vfs, matching the shared-* family conventions: plain shared-* package name, version.workspace, publish = false; the workspace globs crates/* so no members edit is needed). It depends on nothing: std only, no workspace crates, no external crates - yours: "shared-vfs must not depend on anything else." The interner, claims tables, and locking are all hand-rolled on std. It holds only generic machinery: the traits, the types, the Router, the claims, the handle, a generic memory backend. Nothing in it knows the string "/_promptforge" exists. It is the permanent bottom of the dependency stack. + - `promptforge-vfs` (new crate at promptforge/crates/promptforge-vfs): the promptforge policy layer - the /_promptforge mount layout, the stock constructors (empty() with the store mount preinstalled), and the Store-facing conventions. It depends on shared-vfs; promptforge-store and promptforge-core sit above it. + - The manifest expresses the zero-dependency rule: an empty `[dependencies]` table with a comment stating the rule, and nothing else. Fast builds are designed in, not hoped for: zero dependencies means the crate compiles alone and never rebuilds for a dependency rev; dyn at every boundary (Box, Box) keeps our code out of dependents' codegen units; profile knobs stay in the workspace root manifest. The named cost: editing shared-vfs rebuilds everything above it, so its surface must stay stable - which this plan's inlined declarations serve. + - The complete public surface of shared-vfs: traits `Vfs` and `VfsAccess` (backend authors) and `Policy`; types `VfsRef`, `VfsRefBuilder`, `Access`, `ExecId` (opaque: no public constructor - it appears in the `Vfs::acquire` signature, so it must be nameable, but only the handle vends them), `VfsPath`, `VfsPathBuf`, `Entry`, `Stat`, `FileType`, `GrepQuery`, `GrepMatch`, `GrepResults`, `Op`, `Verdict`, `AllowAll`, `VfsError` and its kinds; the memory backend; the host backend (HostBackend::identity / HostBackend::rooted). Everything else is crate-private: `Router` (mounts are installed via VfsRefBuilder), `Volume`, `Claims`, the interner, canonicalize (called inside Access methods, which take &str), and the claims-checking wrapper. +### Public declarations + +```rust +/// One backend behind the virtual namespace. +/// +/// Sync by design: the Lua VM and the executor's single driver thread are +/// synchronous. Bytes at the operation level. `Send` is required, `Sync` +/// is not: the handle serializes access. The only way to touch storage is +/// to acquire an access object bound to an identity. +pub trait Vfs: Send { + /// Acquires an access object bound to `id`. Every operation on the + /// returned object is attributed to that identity: backends that + /// care can know who is touching what; the rest ignore it. + fn acquire(&mut self, id: ExecId) -> Result, VfsError>; + + /// Releases `id`. Also called from the access object's Drop, so + /// teardown paths (cancel, panic, early return) cannot skip it. + fn release(&mut self, id: ExecId) -> Result<(), VfsError>; + + /// Whether this backend rejects all mutations. + fn read_only(&self) -> bool { + false + } +} + +/// One identity's session with a backend. Holds the ExecId. +/// All filesystem operations live here - no access object, no ops. +/// Paths arrive validated, canonicalized, and interned; backends never +/// re-validate. +pub trait VfsAccess: Send { + /// Reads the file at `path` exactly as stored. + fn read(&self, path: &VfsPath) -> Result, VfsError>; + + /// Reads `len` bytes starting at byte `offset`. + /// Default: read whole, slice. Backends that can seek (host + /// directory, SQLite) override and never materialize the file. + /// The handle's line-based ranges are built on this. + fn read_range(&self, path: &VfsPath, offset: u64, len: u64) + -> Result, VfsError>; + + /// Creates or overwrites the file at `path`. + /// + /// Noted but not implemented in v1: a defaulted + /// `write_owned(&mut self, path: &VfsPath, contents: Vec)` + /// delegating to `write`, which the memory overlay would override to + /// move the buffer with zero copies. Add when profiling calls for it. + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError>; + + /// Appends to the file at `path`, creating it if absent. + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError>; + + /// Removes the file, link, or directory at `path`. + /// Absent is NotFound; a directory without `recursive` is an error. + /// On a symlink, removes the link, never the target. + /// (The Store facade keeps Lua's idempotent delete by mapping + /// NotFound to Ok - strictness lives in the trait, kindness in + /// the facade.) + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError>; + + /// A confirmed absence is `Ok(false)`; a backend failure is `Err`. + fn exists(&self, path: &VfsPath) -> Result; + + /// Returns stored paths matching `pattern`, sorted. + fn glob(&self, pattern: &str) -> Result, VfsError>; + + /// Lists the directory at `path`. + fn list(&self, path: &VfsPath) -> Result, VfsError>; + + /// Returns metadata for `path`. + fn stat(&self, path: &VfsPath) -> Result; + + /// Creates the directory at `path`. + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError>; + + /// Renames or moves, atomically where the backend allows. + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError>; + + /// Copies the file at `from` to `to`. + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError>; + + /// Replaces the unique occurrence of `old` with `new`. + /// Zero matches and multiple matches are both errors. + /// Default: read, count, replace, write. Override to push down. + fn str_replace(&mut self, path: &VfsPath, old: &str, new: &str) + -> Result<(), VfsError>; + + /// Searches files under the query's root. + /// Default: glob, read, line scan. Override for indexed backends. + fn grep(&self, query: &GrepQuery) -> Result; + + /// POSIX extras; default implementations return Unsupported. + fn symlink(&mut self, target: &VfsPath, link: &VfsPath) -> Result<(), VfsError>; + fn read_link(&self, path: &VfsPath) -> Result; + fn chmod(&mut self, path: &VfsPath, mode: u32) -> Result<(), VfsError>; +} + +/// The seven POSIX kinds, named rather than lumped - a virtual +/// /dev/null (char device) is a plausible backend, and Other would +/// hide it. The engine adapter maps the first four directly and the +/// three specials to File with a trace (unreachable in practice: +/// neither our v1 backends nor the engine's ever produce them). +pub enum FileType { + File, + Directory, + Symlink, + Fifo, + Socket, + CharDevice, + BlockDevice, +} + +pub struct Entry { + pub name: String, + pub stat: Stat, + pub description: Option, // annotation column; None outside /_promptforge; + // the engine adapter drops it +} + +/// Options preserve honesty: a backend that does not track a field +/// says None rather than fabricating (an invented mtime is +/// nondeterministic; a constant one makes `ls -t` sort garbage). +/// The adapter emits the 0o644/0o755 default when mode is None. +pub struct Stat { + pub file_type: FileType, + pub size: u64, + pub mode: Option, + pub modified: Option, + pub created: Option, +} + +pub struct GrepQuery { + pub pattern: String, + pub root: VfsPathBuf, + pub is_regex: bool, + pub case_insensitive: bool, + pub glob_filter: Option, + pub max_results: Option, +} + +pub struct GrepMatch { + pub path: String, + pub line_number: usize, + pub line: String, +} + +pub struct GrepResults { + pub matches: Vec, + pub truncated: bool, +} +``` + +```rust +/// Identity of one serial thread of execution. Process-unique, +/// vended from a process-global monotonic counter. Opaque: no public +/// constructor - it must be nameable (it appears in Vfs::acquire and +/// Access::id), but only the handle vends them. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct ExecId(u64); + +/// Canonical, interned virtual path. Produced by canonicalize() at the +/// moment the API receives a path; interning makes claim lookups +/// pointer-cheap and guarantees alias detection. +pub struct VfsPath { /* interned shared string */ } + +pub struct VfsRef { /* Arc, poison-safe */ } + +impl VfsRef { + pub fn new(backend: impl Vfs + 'static) -> VfsRef; // single backend + pub fn builder() -> VfsRefBuilder; // mount installation + pub fn acquire(&self) -> Access; // the only way in: fresh ExecId + pub fn store(&self, access: &Access) -> Store; // facade bound to the caller's identity + + /// Returns a handle with `backend` mounted at `prefix` over this + /// handle's namespace. The claims table is shared: conflicts are + /// detected across both views of the same storage. + pub fn overlay(&self, prefix: &str, backend: impl Vfs + 'static) -> VfsRef; +} + +/// Mount installation for VfsRef. Mounts are fixed at build(), so the +/// table is immutable and cheap to Arc-share thereafter. +pub struct VfsRefBuilder { /* the Router under construction */ } + +impl VfsRefBuilder { + pub fn mount(self, prefix: &str, backend: impl Vfs + 'static) -> Self; + pub fn build(self) -> VfsRef; +} + +// Construction patterns: +// identity mount: VfsRef::builder().mount("/", HostBackend::identity()).build() +// - virtual C:/Users/x/y IS host C:\Users\x\y +// chroot mount: VfsRef::builder().mount("/", HostBackend::rooted("C:/work")).build() +// - virtual /a/b IS host C:/work/a/b, containment rejects escape +// overlay: base.overlay("/_promptforge/runs", runs_backend) +// - shares the base's claims table, swaps only the backend view +// stock handle: promptforge_vfs::empty() - VfsRef plus the /_promptforge/store +// memory mount; lives in the policy crate because the mount +// layout is promptforge policy, not shared-vfs machinery + +/// The public capability. Holds an ExecId and the backend's access +/// object; every operation canonicalizes the path, checks the claims +/// tables, then locks the backend per call (never across an await). +pub struct Access { /* ExecId + Arc + Box */ } + +impl Access { + /// Returns the capability for a new concurrent thread of execution. + /// Called by the executor when it spawns one: the child gets a + /// fresh ExecId, and my claims are deleted from the tables (they + /// predate the child by construction; a retired claim can never + /// conflict again). The spawn IS the happens-before edge - no + /// fence call, no epochs. + pub fn spawn(&self) -> Access; + pub fn id(&self) -> ExecId; + + pub fn read(&self, path: &str) -> Result, VfsError>; + pub fn read_string(&self, path: &str) -> Result; + pub fn read_range(&self, path: &str, start: usize, end: Option) + -> Result; + pub fn read_range_numbered(&self, path: &str, start: usize, end: Option) + -> Result; + // write, append, str_replace, remove, exists, glob, list, stat, grep: + // same shapes, taking &str +} + +impl Drop for Access { + // releases this identity and deletes its claims +} + +/// What operation is being attempted - the policy matches on this. +pub enum Op { + Read, Write, Append, Delete, Rename, Mkdir, Copy, Grep, // ... +} + +/// The policy's answer. Reasons are load-bearing in both directions: +/// Deny's string flows back to the model as the tool error (its +/// recovery path); Ask's string is what the user sees in the +/// approval dialog (what is being asked, and which rule fired). +pub enum Verdict { + Allow, + Deny(String), + Ask(String), +} + +/// One policy per VfsRef, consulted by Access on every operation, +/// before the claims check. Dynamic through shared state: the host +/// or UI holds the same Arc and changes behavior mid-run. +pub trait Policy: Send { + fn check(&self, op: Op, path: &VfsPath) -> Verdict; +} + +/// v1 ships AllowAll. ModePolicy (Ask / Plan / Agent, markdown-only +/// in Plan) lives in promptforge-vfs - editor policy, not VFS machinery. +pub struct AllowAll; +``` + + - Identity lifecycle maps to VM lifecycle: concurrent threads (fanout arms, async tasks, the walk, host phases) call `acquire()`; blocking children (call chains) borrow the parent's access (no new identity, no false conflicts); transfer of control moves the access object; destruction drops it. The ordered-vs-concurrent distinction is expressed by borrow-vs-spawn, so no flag exists. + - The host backend is in scope and lives in shared-vfs: std::fs is std, so the zero-dependency rule holds. Stage 1 is thin (direct std::fs ops, lexical+canonicalize containment, failure-atomic writes via sibling temp + rename); stage 2 hardens toward the Bashkit RealFs oracle (resolver trio, symlink policies, Windows long paths and device names) as the threat model demands. Two constructors: HostBackend::identity() (virtual path is the host path) and HostBackend::rooted(dir) (chroot-style, containment-enforced). + - The oracle's public interface (Bashkit RealFs, from bashkit/crates/bashkit/src/fs/realfs.rs and lib.rs - the shape our port replicates, minus async): + +```rust +/// Access mode for the real filesystem backend. +pub enum RealFsMode { + ReadOnly, // all write operations return permission denied + ReadWrite, // breaks the sandbox boundary; trusted scripts only +} + +/// Real filesystem backend scoped to a root directory. +/// The root is canonicalized and validated as a directory at construction. +pub struct RealFs { /* root: PathBuf, mode: RealFsMode */ } + +impl RealFs { + pub async fn open(root: impl AsRef, mode: RealFsMode) -> io::Result; + pub fn root(&self) -> &Path; + pub fn mode(&self) -> RealFsMode; + // note: the sync new() is deprecated upstream for blocking; + // ours is sync by design, so HostBackend::rooted is the sync new +} + +// Builder-level mounting (BashBuilder, lib.rs): +// mount_real_readonly(host_path) / mount_real_readonly_at(vfs_path, host_path) +// mount_real_readwrite(host_path) / mount_real_readwrite_at(vfs_path, host_path) +// allowed_mount_paths(...) - the mount allowlist (TM-FS-013) +// is_sensitive_mount_path(host_path) - the sensitive-path denylist check +// Our equivalents: Router::mount(prefix, HostBackend::rooted(dir)) plus the +// Policy layer; the mode maps to our read_only() backend flag. +``` + - `promptforge-store`: the Store facade becomes a public concrete struct (no trait) over a prefix-scoped Access, used by the Lua bindings and by hosts for seeding/extraction; StoreError stays; MemStore/FileStore as public backend types disappear into Vfs backends. + - `promptforge-core`: run() signature change; RunContext::new takes the VfsRef and builds the Store facade for section VMs (see promptforge/crates/promptforge-core/src/execute.rs). + - Bashkit adapter (lives near the future integration crate): implements bashkit::FsBackend over VfsRef; whole-file reads served from read; unsupported operations (symlink, chmod) return the engine's unsupported error. +- File and public API changes: + - `execute::run(prompt, args, resolution, vfs: &VfsRef, config)` replaces the `store: &StoreRef` parameter. + - promptforge-store's public surface stays source-compatible for Lua-facing behavior; the Store trait is removed from the public API. + - Caller migration is one line: `StoreRef::memory()` becomes `promptforge_vfs::empty()`. The stock handle always carries the store mount: empty() means empty of content, not of mounts - a router with a fresh memory backend at `/_promptforge/store`, so callers can seed before run() and extract after. `vfs.store()` returns the public Store facade scoped to the mount; hosts and Lua bindings share it, and callers never hardcode the mount path. run() uses the existing mount; the child-router overlay remains only as a defensive fallback for hand-built routers lacking one. Per-run freshness is caller discipline (one VfsRef per run, or clear between runs), matching papergate's fresh-temp-dir-per-run pattern today. Caller census (this workspace): every external caller uses `StoreRef::memory()` (workshop-server session_agents.rs, promptforge-lua vm.rs and benches, executor tests); `with_files` has no callers outside the store crate's own tests and doc examples, so no convenience constructor is carried over. +- Data, persistence, failure, security, and privacy constraints: + - The trait is bytes-based (`Vec`) so binary content and Bashkit both fit; text helpers (read_string, numbered ranges) live at the VfsRef layer and error on non-UTF-8 where text is required. + - Internal namespace is POSIX-shaped (rooted, forward slashes, strict); all host-OS translation (drive letters, case-insensitive comparison, long-path prefixing, device names) lives only in the future real-FS backend, never in the router or virtual paths. + - Sync trait with an async boundary: store operations become leaf yields in the executor's coroutine protocol (new Request/Answer variants, one dispatch arm, the proven tools.call pattern), answered via spawn_blocking against the sync Vfs. spawn_blocking is task-per-io on a bounded cached pool, never thread-per-io. + - Consistency rule: every store op takes the yield path uniformly, including memory-backed ones. No inline fast path - answering differently by backend makes interleaving behavior backend-dependent, which is exactly the semantic drift the claims model exists to prevent. The channel hop is the price of that invariance, and it is cheap. + - Concurrency model, three separate mechanisms for three separate properties: the mutex gives exclusion (uncontended on the executor's single driver thread; operations never hold it across an await), the ExecId gives attribution (no access object, no operations), and the claims table gives correctness. + - The claims model: the handle holds two maps (readers and writers: interned canonical path to live ExecIds) plus a live-identity set. Every entry is a live, conflict-eligible claim - retired or released claims are deleted, never stored. The conflict rule: a write booms if another live identity appears in the path's readers or writers; a read booms if another live identity appears in its writers; read-read never conflicts. A violation is a fatal RunErrorKind terminating the run on the spot, naming the path, both identities, and both claim kinds. spawn() deletes the parent's claims (the spawn is the happens-before edge); Drop releases the child's. + - Claims are keyed by interned canonical paths produced at API receipt - yours: "it should be calculated / canonicalized at the time the API receives a path. This guarantees we catch aliases." Canonicalization includes the facade's mount-prefix resolution (Lua's `paper.md` and the host's `/_promptforge/store/paper.md` are one key), lexical normalization of the virtual namespace, and the case rule; symlink aliases do not exist in v1. Per your rabbithole caution, v1 may stub canonicalize to interning the passed string; multi-platform host canonicalization lives in the deferred real-FS backend. + - The model is primitive-agnostic: it sees spawn and drop events only, so a fanout implemented in Lua over call_async is enforced identically to any executor-level primitive - which the WriteScope registry (fanout-specific) could never have survived. Claims cover plain writes and appends, which WriteScope never did: papergate's cross-arm appends to evidence.md are caught. + - The prevention pattern the boom teaches: arms write arm-scoped paths and the join merges in arm order - deterministic by construction. + - Failure-atomicity contract (from Bashkit's TM-FS-014): a failed write, copy, or rename leaves source, destination, and accounting unchanged. Append is one lock acquisition, so read-check-write TOCTOU cannot arise (their TM-DOS-034 lesson). + - The policy layer: one Policy object per VfsRef (global, not per-mount; a host wanting per-mount rules multiplexes inside its implementation - yours: "I suppose if the host wants a more rich system they can multiplex it into a single Policy object"). Access consults the policy on every operation, before the claims check (a denied operation never registers a claim). The policy is dynamic through shared state: the UI holds the same Arc and flips modes mid-run (e.g. during user_input), and the next operation sees it - no executor involvement. + - Verdicts carry reasons - yours: "Deny and Ask should have an attached string." Deny's string is the model's recovery path (read-only vs locked vs mode-restricted produce different corrections); Ask's string is what the user sees in the approval dialog (what is being asked, which rule fired). + - Modes are a policy implementation, not VFS machinery: ModePolicy with Ask (deny all mutations), Plan (mutations only to markdown paths), Agent (allow all) lives in promptforge-vfs; modes gate mutations, never reads. The mode policy absorbs the seal: one-way vs reversible is just who still holds the mode handle. The static read_only() backend flag stays - a property of the mount, orthogonal to policy. + - Rust API mechanics: #[non_exhaustive] on Entry, Stat, GrepQuery, FileType, and the error enum (Entry is designed to grow - annotations live there); #[must_use] on Access (an acquire dropped immediately is a bug the compiler can catch); Default where a zero value is meaningful (memory backend); private fields with accessors on every public struct that carries invariants. + - Conscious deviation: VfsAccess has sixteen methods against the one-to-three-methods guideline. The methods are one cohesive capability, not unrelated surface; defaulted methods (read_range, str_replace, grep) keep the required set at eleven; the engine's own FsBackend makes the same choice. Recorded so it is a decision, not a discovery. + +### Crate-private declarations + +```rust +/// Canonicalizes a virtual path at API receipt. v1 may stub to +/// interning the passed string as-is; the real work is lexical +/// normalization of the virtual namespace only (dot segments, duplicate +/// separators, trailing slash, case rule) plus the facade's mount-prefix +/// resolution, so Lua's `paper.md` and the host's +/// `/_promptforge/store/paper.md` are one key. Multi-platform host +/// canonicalization is a rabbithole that lives in the deferred +/// real-FS backend, not here. +/// +/// Crate-private: the only way to form a VfsPath is through Access +/// methods taking &str, so canonicalization at receipt is enforced by +/// visibility, not convention. +fn canonicalize(path: &str) -> Result; + +/// One mounted filesystem instance: its backend and the ledger of +/// who is touching what. The two are separately Arc-shareable so +/// overlay() can share the claims table while swapping the backend. +struct Volume { + backend: Arc>>, + claims: Arc, +} + +/// The bookkeeping of who is touching what. Every entry is a live, +/// conflict-eligible claim; retired or released claims are deleted, +/// never stored. +struct Claims { + readers: HashMap>, + writers: HashMap>, + live: HashSet, +} + +/// The mount table. Backends install at prefixes; longest prefix wins. +/// A Router is itself a Vfs, so routers nest. Crate-private: the public +/// concept is "a VfsRef with these mounts," expressed through +/// VfsRefBuilder; privacy enforces mounts-fixed-at-construction, since +/// nobody outside the crate can hold one. +struct Router { /* BTreeMap> */ } + +impl Router { + // acquire(id) returns a routing access: each op resolves the + // longest-prefix mount and delegates to that backend's access, + // acquired lazily on first touch of that mount. +} +``` + + + + +## Testing Plan + +Parity is the gate: the existing store suite must pass against the rewritten facade unchanged in intent. Around that, new matrices cover the router and the cross-platform path layer, and a smoke test proves the Bashkit adapter end to end. Mount-escape attempts are rejected at the router. No public API outside the four named crates (shared-vfs, promptforge-vfs, promptforge-store, promptforge-core) may change. + +- Unit: + - Path canonicalization matrix per OS convention, including dot segments, mixed separators, casing rules, and traversal rejection. + - Router longest-prefix dispatch, nested mounts, shadowing, and read-only enforcement. + - grep default implementation correctness and match semantics. + - A self-policing manifest test: reads the crate's own Cargo.toml via CARGO_MANIFEST_DIR and asserts the dependency tables are empty, so the zero-dependency rule fails the build instead of eroding. +- Integration and end-to-end: + - Executor run end-to-end with VfsRef, including a fanout whose cross-arm append to one path terminates the run with a determinism violation naming both arms. + - Bashkit adapter smoke: mount memory plus store backends, run an ls/cat/grep script, verify output and exit codes. +- Regression, security, and performance: + - The full existing promptforge-store test suite runs against the rewritten Store unmodified in intent: anchor edits, numbered ranges, idempotent delete, glob grammar, poison handling, write-race detection (now expressed through the claims model). + - Compile-time Send/Sync assertions for VfsRef and Access (both hold dyn and interior mutability; losing either is a major, invisible break). + - Claims lifecycle: borrow-vs-spawn semantics, transfer moves claims, drop releases them, spawn deletes the parent's claims (sequential fanouts stay legal), and alias attempts (facade-relative vs mount-absolute spellings of one file) collide on one interned key. + - Mount-escape attempts (dot segments, absolute re-rooting) rejected at the router. +- Exit criteria: + - All ported tests pass; the executor doc example compiles and runs; the adapter smoke test passes; no public API outside the four named crates changes. + + + + +## Decision Record + +- Decisions: + - Own Vfs trait rather than adopting an engine's - yours: "design our own Vfs and VfsRef (like Store and StoreRef)"; our tools need ranges, annotations, and grep that engine traits lack. + - Sync trait, async boundary - corrected rationale: the trait is sync because the backends are sync-native (memory, std::fs, rusqlite), not because Lua is synchronous. Lua was never the constraint: the executor's yield protocol exists precisely so Lua does not have to be. Store ops become leaf yields answered via spawn_blocking against the sync Vfs, exactly like tools.call - so fanout arms interleave during store I/O instead of stalling the driver (FileStore blocks the driver thread today). Completion-based I/O (io_uring/IOCP via compio) is a backend-and-scheduler concern the boundary absorbs; the trait never changes. + - Bytes at the trait level - Bashkit and future binary content require it; text semantics layer above. + - Borrowed write, with write_owned noted but not implemented - yours: "for the memory vfs overlay, taking ownership of the contents string is a natural fit," tempered by "note the signature but leave it out of the implementation"; the trait documents the defaulted `write_owned(Vec)` hatch for the memory overlay's zero-copy move, to be added only when profiling calls for it. + - Store becomes a concrete facade, not a trait - yours: "it is no longer a trait it is just a regular, private implementation which is exposed to Lua." Refined to public-concrete (not private) because production hosts seed declared inputs and extract declared outputs through it (the papergate pattern). + - Store mounts at `/_promptforge/store` - yours: 'it installs into the overlay: "/_promptforge/store"'. + - grep is a first-class Vfs operation with a default scan implementation - yours: "the Store implementation now calls into the Vfs to read, grep, etc"; the default keeps simple backends free of index work while SQLite/FTS backends can override later. + - Router uses a POSIX-shaped internal namespace on every OS - one canonical form, host translation confined to the real-FS backend. + - Mounts fixed at construction - matches both engines' models and keeps routing immutable and cheap to clone per run. + - Crate placement in the shared-* family - yours: "it should be in a shared-* crate"; shared-vfs sits beside shared-protocol and shared-progress as cross-cutting infrastructure with plain naming and publish = false, at the bottom of the dependency stack. + - Handle-level synchronization over backend-level - yours: "Vfs will be accessed concurrently for sure, because of fanout"; fanout interleaves on one driver thread and sync operations cannot tear, so one mutex at the handle beats Bashkit-style per-backend RwLocks, and the WriteScope registry covers the semantic (not data) race. + - The claims model for determinism - yours: "It's safe, but it's not correct because that's non-deterministic behavior. That means replay is not going to work right." Attribution via ExecId presented at lock acquisition (your token-at-the-mutex), identity tied to the VM lifecycle (creation, transfer, destruction), claims released on destroy, and conflicts fatal: "It should fail loud and terminate instantly." + - RAII access object - yours: "we just put acquire and release on the Vfs, and acquire returns a `Box`... So it is literally impossible to mess up." Drop releases claims, so cancellation and error paths cannot leak; the ordered-vs-concurrent distinction is expressed by borrowing the parent's access versus spawning a fresh one, so no flag exists. + - spawn() over fork() - fork lies twice (children do not inherit claims; the word is Unix-only in a Windows-first product); spawn matches tokio and the Lua call_async vocabulary. + - Volume and Claims as the internal names - Shared named a role, not a concept; the volume is one mounted filesystem instance (backend plus claims ledger), and the claims table is its own named type. Inner was the tolerated-but-weaker idiom. + - Builder-style mount installation with lazy per-mount acquire - mounts are declared in one place and immutable after construction; backends learn about identities only when a path under their prefix is first touched. + - Router crate-private, VfsRefBuilder public - yours: "I dont quite understand why Router is public." The public concept is "a VfsRef with these mounts"; privacy enforces mounts-fixed-at-construction because nobody outside the crate can hold a Router. + - ExecId public but opaque - it must be nameable (it appears in the Vfs::acquire signature and Access::id), but no public constructor; only the handle vends identities. + - Two-crate split, generic machinery vs promptforge policy - shared-vfs holds only generic VFS machinery and never names /_promptforge; promptforge-vfs holds the mount layout and stock constructors. Approved: "Yes. That sounds good." + - overlay() shares the claims table - two handles over the same storage with two claims tables would blind the determinism check exactly where two views coexist; Volume therefore holds backend and claims as separately shareable Arcs. VfsRef itself implements Vfs (forwarding acquire with the given ExecId) so a base handle mounts under a child router. + - WriteScope registry deleted - yours: "I never liked the WriteScope registry anyway." The claims table is the single mechanism and covers plain writes and appends, which WriteScope never did. + - Uniform yield path for all store ops - yours: "I agree with all of that, and make a note about consistency." No inline fast path for memory backends, because backend-dependent answer paths make interleaving behavior backend-dependent; the channel hop is the price of the invariance. + - Wide access trait as a conscious deviation - the filesystem capability is cohesive, defaulted methods keep the required surface at eleven, and the engine's own backend trait makes the same choice; recorded against the small-trait guideline so it is a decision, not a discovery. + - The stock VfsRef always carries the store mount - yours: "The empty Vfs should still support a store"; empty() means empty of content, not of mounts, because the seeding contract requires the store to exist before run() is called. + - vfs.store() as the seeding/extraction handle - the production workflow (stock Vfs, seed declared inputs at known filenames, run, extract declared outputs, no real files) needs a public facade available before run(); hosts and Lua bindings share it. Confirmed: "vfs.store() looks like the correct model." + - vfs.store() ships as an extension trait in promptforge-store, not an inherent method on VfsRef - the Store facade type lives in promptforge-store, which sits above promptforge-vfs and shared-vfs in the dependency stack, so shared-vfs cannot name the type. A prelude-exported extension trait preserves the declared vfs.store(&access) call shape without inverting the stack. + - API symmetry as a deliberate property - yours: "on the Rust side the API is essentially the same which means no new learning surface for integrators." The Store is the scratchpad that survives the context reset when control transfers between sections; Lua's store table and the Rust facade are the same interface in two languages, documented once, and the model's later file tools become a third consumer of the same verbs. + - Host backend in scope in shared-vfs - yours: "without it, this is fucking useless." std::fs is std, so the zero-dependency rule holds; stage 1 thin with atomic writes, stage 2 hardening toward the RealFs oracle. (Supersedes the earlier deferral, which treated papergate's FileStore use as diagnostics-only.) + - Host backend will be ported, not adapted - source exploration of Bashkit's RealFs shows it is tokio::fs throughout (adapting means block_on per call plus their Tokio dependency) and leaves two Windows gaps we require (long-path prefixing, device names); its value is the hardened algorithms and the realfs test suite as a behavioral oracle, not the type. +- Rejected alternatives: + - Co-locating Vfs with run()'s other traits in promptforge-core-support - rejected: that crate holds small host-support primitives (cancel, observe, untrusted guards) while Vfs is a subsystem (router, paths, glob, grep, backends) with consumers well beyond the executor (Lua, hosts, workshop-server, the future engine adapter); the workspace has no run()-traits crate to join, since Tool, ModelResolver, and ToolResolver each live in their domain crates. Revisit: only as an additive umbrella re-export crate for integrator ergonomics, never as a move. + - Async trait - rejected: block_on inside a current-thread driver is a deadlock hazard; revisit only if a backend genuinely needs async I/O. + - Adopting bashkit::FsBackend as the core trait - rejected: whole-file-only reads, five-field metadata, pre-1.0 dependency direction; the adapter isolates it instead. + - Keeping Store as a public trait with pluggable backends - rejected: two filesystem abstractions is the incoherence this plan exists to remove. +- Assumptions, risks, and notes: + - Sync real-FS reads will block the driver thread; acceptable for this phase, offload later if traces show stalls. + - Two glob implementations (Store's matcher and the VFS's) must not drift; port one, delete the other. + - Error mapping between Vfs kinds and StoreError must be total; a missed kind is a Lua-visible behavior change. + - Strictness will bite spawn-and-forget patterns: a parent that keeps writing paths a live task has claimed will boom. Error message quality decides whether authors experience this as guidance or noise. + - The policy seam was pressure-tested against learned allow-rules ("Always allow this directory" via a rule-list policy) and absorbed the feature with zero changes to the trait, Access, claims, or executor - evidence the interface is wide enough before it was needed. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build` (builds only the gateway, the default workspace member; run `npm ci --prefix crates/workshop-server/ui` and `npm ci --prefix crates/gateway-config-ui/ui` once after cloning). Full desktop build: `cargo workshop` (a `.cargo/config.toml` alias for `run -p build-workshop`; add `--release` or `--target ` as needed). +- Focused test command pattern: `cargo nextest run -p ` (nextest config in `.config/nextest.toml`; heavy STT/tool-picker suites are concurrency-limited via test groups). +- Component test command pattern: `cargo nextest run -p `; integration targets also runnable as `cargo test -p --test it ` (CI example: `cargo test -p gateway-stt --test it architecture`). +- Full-suite test command: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --all-features`, plus doctests via `cargo test --workspace --exclude workshop --exclude workshop-server --all-features --doc` (workshop crates are covered separately on Windows: `cargo nextest run --locked -p workshop -p workshop-server` and `cargo test --doc -p workshop -p workshop-server`). +- Linter command: `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings` (workshop crates: `cargo clippy -p workshop -p workshop-server --all-targets -- -D warnings`). Supply chain: `cargo deny check` and `cargo audit`. +- Formatter check command: `cargo fmt --all --check`. +- Docs command: `mdbook build guide` for the user guide; `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server` for API docs (CI runs it with `RUSTDOCFLAGS: -D warnings`). +- Test placement and naming conventions: unit tests live in `src/` modules; integration tests live in `crates//tests/`. Multi-file suites use one target with a `main.rs` entry plus sibling module files (e.g. `crates/promptforge-core/tests/suite/{main,execution,fanout,parsing,shipped,support}.rs`); gateway and workshop-server use a `tests/it/` target. Fixtures sit beside tests (e.g. `tests/prompts/{valid,invalid,execution}`, `workshop-server/tests/fixtures`). Test names are long descriptive snake_case sentences (e.g. `a_process_lifetime_lease_recovers_after_its_owner_is_terminated`). Benches use criterion (`crates/promptforge-core/benches`). Node helper scripts in `tools/` have sibling `.test.mjs` files. +- Directory map: `crates/` holds all 36 workspace crates grouped by product prefix (`promptforge-*` executor/language, `gateway*` inference server, `workshop*` desktop app, `shared-*` cross-product substrate, `build-*` build tooling, `product-integration-tests`); `crates/shared-ui` is a TypeScript+CSS package excluded from the Cargo workspace. `guide/` is the mdbook user guide (four doc sets: Workshop, gateway, prompt language, agent programs). `prompts/` holds example prompt programs. `design/` holds design notes. `tools/` holds Node.js helper scripts (gateway sidecar staging, TTS live checks). `vibe/` holds project governance records: `archdoc.md`, dated decision logs, `ACTIVE`. `.config/nextest.toml` configures nextest; `.cargo/config.toml` sets the static-CRT Windows target and the `workshop` alias; `.github/workflows/` holds CI and release packaging; `images/` holds README assets. +- Component boundaries (per `vibe/archdoc.md`): executor (`promptforge-core` and supporting `promptforge-*` crates) parses and runs prompt pipelines and Lua agent programs; gateway (`gateway*` crates) is an independent server owning model routing, provider credentials, and local inference; CLI (`promptforge` crate) is a thin shell adapter over the executor; Workshop UI (`workshop`, `workshop-server`) is the Tauri desktop shell hosting the executor in-process; store (`promptforge-store`) is the run-scoped virtual filesystem; the Lua VM boundary (`promptforge-lua`) sandboxes prompt code; shared substrate (`shared-*`) carries progress, loopback discovery, protocol, and sidecar facilities. Dependency directions: executor depends on gateway protocol, store, Lua boundary, shared substrate; CLI and Workshop depend on executor, gateway, store, substrate; store and substrate depend on nothing. AGENTS.md enforces four cross-product rules: Gateway crates cannot depend on Workshop or PromptForge product crates; PromptForge crates cannot depend on Gateway or Workshop crates; Workshop crates cannot depend on Gateway crates. +- Conventions summary: Rust edition 2024 on the stable toolchain (`rust-toolchain.toml`), resolver 3, workspace version 0.3.0. Workspace lints forbid `unsafe_code`, deny `clippy::all` plus `unwrap_used`/`expect_used`, and warn on missing docs. Comments explain non-obvious constraints and cite upstream issue URLs for platform workarounds. Behavior changes ship with tests in the same change; structural enforcement (parsers, snapshots, allowlists, topology checks) requires explicit user approval. Cargo features gate real constraints (toolchain or native build), never product shape. Runtime and serve paths never compile native dependencies, exit the process, or install process-global state. Long-running work reports through `shared-progress`. The two web UIs are TypeScript bundled by esbuild through Cargo build scripts, with Node.js 22 required. + + + + +## Execution Instructions + + + +### Step 1: shared-vfs skeleton, value types, and canonical paths [completed] + +- Component: shared-vfs core +- Create `crates/shared-vfs` (package `shared-vfs`, version.workspace, publish = false, empty `[dependencies]` table carrying the zero-dependency comment; the workspace `crates/*` glob needs no members edit) with the `src/lib.rs` module layout. +- Write `crates/shared-vfs/AGENTS.md` following the per-crate convention (every workspace crate has one), with exactly these three rules: "std only. No dependencies, workspace or external. The manifest test enforces this; never weaken it." / "No promptforge policy: no /_promptforge paths, no Store, no run concepts." / "The public surface is load-bearing: add defaulted methods, never change existing signatures. Every edit rebuilds the whole stack." +- Define `VfsError` and its kinds (#[non_exhaustive]), `FileType` (seven POSIX kinds), `Entry` (name, stat, description annotation column), `Stat` (honest Option fields), `GrepQuery`, `GrepMatch`, `GrepResults` (#[non_exhaustive], private fields with accessors where invariants exist). +- Implement the hand-rolled interner, `VfsPath`/`VfsPathBuf`, and crate-private `canonicalize()` (lexical normalization of the POSIX-shaped namespace: dot segments, duplicate separators, trailing slash, case rule; stub-to-intern acceptable in v1). +- Tests: the self-policing manifest test (reads the crate's own Cargo.toml via CARGO_MANIFEST_DIR and asserts the dependency tables are empty) and the path canonicalization matrix (dot segments, mixed separators, casing rules, traversal rejection). + + + + + +### Step 2: Vfs and VfsAccess traits and policy types + +- Component: shared-vfs core +- Define `trait Vfs` (acquire/release/read_only, Send, sync by design), `trait VfsAccess` (sixteen methods; defaults: read_range slices a whole read, str_replace is read-count-replace-write, grep is glob-read-line-scan, symlink/read_link/chmod return Unsupported), `trait Policy`, `enum Op`, `enum Verdict` (Deny and Ask carry reason strings), and `AllowAll`. +- Tests: default read_range, str_replace (zero and multiple matches are both errors), and grep match semantics against an in-test stub backend; unsupported defaults return the right error kind. + + + + + +### Step 3: handle, Access capability, and claims tables + +- Component: shared-vfs core +- Implement `ExecId` (opaque, process-global monotonic counter, no public constructor), `Volume` (backend and claims as separately Arc-shareable), `Claims` (readers/writers maps from interned VfsPath to live ExecIds plus the live set; retired or released claims are deleted, never stored), `VfsRef` (`Arc`, poison-safe locking, `VfsRef::new` and `acquire()`), and `Access` (#[must_use]; canonicalizes at receipt, consults the Policy before the claims check so a denied operation never registers a claim, registers claims, locks the backend per call; `spawn()` vends a fresh ExecId and deletes the parent's claims; Drop releases the identity and its claims). +- Conflict rule: a write fails when another live identity appears in the path's readers or writers; a read fails on another live writer; read-read never conflicts. The violation is a dedicated VfsError kind naming the path, both identities, and both claim kinds (the executor maps it to the fatal RunErrorKind in step 10). +- Tests: claims lifecycle (borrow vs spawn, transfer moves claims, drop releases, spawn deletes the parent's claims so sequential fanouts stay legal), alias collision (facade-relative and mount-absolute spellings of one file land on one interned key), the conflict matrix, and compile-time Send/Sync assertions for VfsRef and Access. + + + + + +### Step 4: router, builder, and overlays + +- Component: shared-vfs router +- Implement crate-private `Router` (BTreeMap mount table, longest-prefix dispatch, lazy per-mount acquire on first touch), `impl Vfs for Router` (routers nest), `impl Vfs for VfsRef` (forwards acquire with the given ExecId, so a base handle mounts under a child router), `VfsRefBuilder` (mount consumes and returns self; build() freezes the table), and `VfsRef::overlay()` (shares the claims table, swaps only the backend view). +- Enforce at the router: writes to read-only mounts fail with a clear read-only error and never partially apply, and traversal escape from a mount prefix is rejected. +- Tests: longest-prefix dispatch, nested mounts, shadowing, read-only enforcement, mount-escape rejection (dot segments, absolute re-rooting), and one VfsRef serving the store mount, a memory scratch mount, and a second overlay simultaneously. + + + + + +### Step 5: memory backend + +- Component: shared-vfs backends +- Implement the generic in-memory backend in shared-vfs carrying former MemStore semantics (Default where a zero value is meaningful; acquire/release accept ExecId attribution as a no-op). +- Tests: the full operation surface (read, read_range, write, append, remove strictness, exists, glob, list, stat, mkdir, rename, copy) against the memory backend. + + + + + +### Step 6: host backend, stage 1 thin + +- Component: shared-vfs backends +- Implement `HostBackend::identity()` (virtual path is the host path) and `HostBackend::rooted(dir)` (chroot-style) in shared-vfs over direct std::fs: lexical plus canonicalize containment for rooted, failure-atomic writes (sibling temp file plus rename; a failed write, copy, or rename leaves source, destination, and accounting unchanged), and the read_only flag rejecting all mutations. +- Tests: containment escape rejection, atomic-write failure cases, and round trips in temp directories. + + + + + +### Step 7: promptforge-vfs policy crate + +- Component: promptforge-vfs +- Create `crates/promptforge-vfs` (depends on shared-vfs only): the `/_promptforge/store` mount layout, the `empty()` stock constructor (a router with a fresh memory backend at the store mount; empty of content, not of mounts), and `ModePolicy` (Ask denies all mutations, Plan allows mutations only to markdown paths, Agent allows all; modes gate mutations, never reads) behind a UI-flippable shared Arc so a mode change mid-run takes effect on the next operation. +- Tests: mode flip visibility through the shared handle, Plan markdown-only enforcement, and empty() carrying the store mount. + + + + + +### Step 8: Store facade rewrite and parity suite + +- Component: promptforge-store +- Rewrite `promptforge-store`: the Store trait, MemStore, and FileStore disappear from the public API; a public concrete `Store` facade wraps a prefix-scoped Access and is exposed as `vfs.store(&access)` via a prelude-exported extension trait (see decision record). Preserve the StoreError vocabulary exactly with a total VfsError-to-StoreError mapping, anchor-edit rules, numbered reads, idempotent delete (NotFound maps to Ok), and the glob grammar (port one glob implementation, delete the other). Delete the WriteScope registry. +- Tests: the full existing promptforge-store suite ported onto the facade unmodified in intent (anchor edits, numbered ranges, idempotent delete, glob grammar, poison handling, write-race detection now expressed through the claims model). This suite is the parity gate. + + + + + +### Step 9: executor API pivot to VfsRef + +- Component: promptforge-core +- Change `execute::run(prompt, args, resolution, vfs: &VfsRef, config)`; `RunContext::new` takes the VfsRef and builds the Store facade internally; run() overlays a fresh memory store only as a defensive fallback for hand-built routers lacking the mount; the scheduler installs the current Access per chain step. +- Migrate callers: `StoreRef::memory()` becomes `promptforge_vfs::empty()` in workshop-server session_agents.rs, promptforge-lua vm.rs and benches, and executor tests; `with_files` is not carried over. The executor doc example compiles and runs with VfsRef. +- Tests: executor run end-to-end with VfsRef, and the papergate-shaped seed-run-extract round trip on a stock VfsRef with no real files (a missing declared output is an explicit contract error naming the prompt's promise). + + + + + +### Step 10: store operations as leaf yields + +- Component: promptforge-core +- Add the new Request/Answer variants and one dispatch arm (the proven tools.call pattern) so every Lua store operation becomes a leaf yield answered via spawn_blocking against the sync Vfs: uniform for all backends, no inline fast path. Map the claims-violation VfsError to the fatal determinism RunErrorKind that terminates the run instantly and is not catchable from Lua. +- Tests: a fanout whose cross-arm append to one path terminates the run with a determinism violation naming both arms, the arm-scoped-writes plus ordered-merge fixture passing, and interleaving invariance across memory and host backends. + + + + + +### Step 11: Bashkit adapter spike + +- Component: bashkit adapter spike +- Implement `bashkit::FsBackend` over VfsRef as a path dependency against the local `bashkit/` clone: whole-file reads served from read, symlink/chmod return the engine's unsupported error, FileType maps the first four kinds directly and the three specials to File with a trace, Stat mode None emits the 0o644/0o755 defaults, and the adapter captures the current ExecId at exec start. +- Compile-check and smoke-test an ls/cat/grep script across mounted memory and store backends, verifying output and exit codes. The deliverable is evidence that the trait subsumes Bashkit; a mapping failure is the spike working as intended. + + + +- Deferred and out of scope: do_shell dispatcher and argv pattern, git builtins and commit path, SQLite run-record backend and /_promptforge renderers, terminal mirrors, annotated listings, Bashkit integration proper (hooks, TraceMode, analyze), kaish fallback, stage-2 host-backend hardening. + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..ab3148d63 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-09-11-3-vfs-foundation.md diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index 49b9ba944..4b3acd792 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -75,3 +75,10 @@ - Step 2: Flip the version gate to 0-only and migrate prompts (DEBT-UPM-04) - FULL verify: build, fmt, clippy, workspace suite, workshop crates, doctests all passed (nextest fallback). - Decision: removed `SUPPORTED_MAJOR` outright rather than repurposing it, since the literal `Some(0)` arm left no uses | Falsifier: a future arm or message that needs a named supported-major constant. - Decision: updated the `UnsupportedVersion` display text to "supports major 0" though not explicitly listed in the step, since the gate flip made "major 1" false | Falsifier: a reviewer who wants the message wording owned by a separate change. + +## 2026-09-11-3-vfs-foundation + +- Step 1: shared-vfs skeleton, value types, and canonical paths - `cargo nextest run -p shared-vfs` - 11 passed, 0 failed; clippy `-D warnings` and fmt clean. Review: clean. + - Decision: the virtual namespace requires a leading `/`; drive-letter-style paths (`C:/x`) are rejected as relative, with host-path translation deferred to the host backend | Falsifier: a later step requires identity-mount virtual paths of the form `C:/...` to canonicalize. + - Decision: case is preserved and comparison is case-sensitive (POSIX semantics) in the virtual namespace | Falsifier: a host-backend requirement mandates case-insensitive virtual-path comparison. + - Decision: `canonicalize`/`intern` carry targeted `#[allow(dead_code)]` until `Access` (a later step) becomes their caller | Falsifier: the next step wires `Access` and the allows remain. From 3ce38ff1f772a2280b66627d7cbc98cea6ae98a4 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 18:22:48 -0700 Subject: [PATCH 02/26] Define Vfs and VfsAccess traits and policy types This change defines the backend trait pair behind the virtual namespace: one trait for a backend that vends and releases identity-bound sessions, and one carrying all sixteen filesystem operations with defaults for ranged reads, string replacement, and content search. It adds the policy hook consulted before every operation, the operation and verdict types it matches on, and an allow-everything policy as the initial implementation. Execution identities are opaque and vended from a process-wide monotonic counter, so every operation can be attributed to one serial thread of execution. The default search matches literal text only and refuses regular expressions, since the crate is std-only and a regex engine must come from an overriding backend. - `Vfs` - acquire and release are the only entry points; every operation lives on the acquired access object, so storage is unreachable without an identity. Send is required and Sync is not: the handle serializes access. - `VfsAccess` - sixteen methods, with defaults for read_range, str_replace, grep, and the POSIX extras keeping the required set at eleven. - `Verdict` - Deny and Ask carry reason strings: Deny's flows back to the model as its recovery path, Ask's is the approval dialog text. - `ExecId` - an opaque single-field wrapper with no public constructor; vend draws from a process-global counter and has no caller until the handle arrives. - `read_range` - the default reads the whole file and slices, clipping at the end and returning empty when the offset is past it. - `str_replace` - the default rewrites the unique occurrence; zero or multiple matches are errors and leave the file unchanged. - `grep` - the default globs, reads, and line-scans with literal substring matching; regex queries return Unsupported, non-UTF-8 files and directories are skipped, and results cap at max_results with a truncation flag. - `AllowAll` - permits every operation. - `write_owned` - noted in the write documentation but not implemented in v1; the zero-copy move for the memory overlay waits for profiling. - `crates/shared-vfs/src/path.rs` - the dead-code allows on intern and canonicalize are removed now that the grep default canonicalizes. Design: new newtype @ crates/shared-vfs/src/traits.rs::ExecId boundary: pub Design: new global-state @ crates/shared-vfs/src/traits.rs::ExecId::vend Design: new speculative-abstraction @ crates/shared-vfs/src/traits.rs::VfsAccess boundary: pub Design: new speculative-abstraction @ crates/shared-vfs/src/traits.rs::Policy boundary: pub Design: new value-object @ crates/shared-vfs/src/traits.rs::Op boundary: pub Deferred: ExecId::vend carries #[allow(dead_code)] with no caller until the handle arrives in a later step Deferred: the defaulted write_owned method is noted but not implemented in v1, to be added when profiling calls for it Plan: vibe/2026-09-11-3-vfs-foundation.md --- crates/shared-vfs/src/lib.rs | 2 + crates/shared-vfs/src/path.rs | 4 - crates/shared-vfs/src/traits.rs | 553 ++++++++++++++++++++++++++++ vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 4 + 5 files changed, 560 insertions(+), 5 deletions(-) create mode 100644 crates/shared-vfs/src/traits.rs diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs index 8009a961d..b434d7fef 100644 --- a/crates/shared-vfs/src/lib.rs +++ b/crates/shared-vfs/src/lib.rs @@ -7,10 +7,12 @@ mod error; mod path; +mod traits; mod types; pub use error::VfsError; pub use path::{VfsPath, VfsPathBuf}; +pub use traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; pub use types::{Entry, FileType, GrepMatch, GrepQuery, GrepResults, Stat}; #[cfg(test)] diff --git a/crates/shared-vfs/src/path.rs b/crates/shared-vfs/src/path.rs index 924906018..b3d58d3df 100644 --- a/crates/shared-vfs/src/path.rs +++ b/crates/shared-vfs/src/path.rs @@ -27,8 +27,6 @@ impl Interner { } } - // Callers arrive with `Access` in a later step; only tests intern today. - #[allow(dead_code)] fn intern(&mut self, s: &str) -> u32 { if let Some(&id) = self.ids.get(s) { return id; @@ -129,8 +127,6 @@ impl fmt::Display for VfsPathBuf { /// trailing slash is dropped; the root canonicalizes to itself. Case is /// preserved and significant (POSIX semantics): paths differing only in /// case are distinct. Relative and empty paths are rejected. -// Callers arrive with `Access` in a later step; only tests canonicalize today. -#[allow(dead_code)] pub(crate) fn canonicalize(path: &str) -> Result { if path.is_empty() { return Err(VfsError::InvalidPath("empty path".into())); diff --git a/crates/shared-vfs/src/traits.rs b/crates/shared-vfs/src/traits.rs new file mode 100644 index 000000000..b4b05c0af --- /dev/null +++ b/crates/shared-vfs/src/traits.rs @@ -0,0 +1,553 @@ +//! The backend traits, the policy hook, and execution identity. +//! +//! `Vfs` is one backend behind the virtual namespace; `VfsAccess` is one +//! identity's session with it, carrying every filesystem operation. +//! `Policy` is the per-handle hook consulted before the claims check, and +//! `ExecId` is the identity every operation is attributed to. + +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::error::VfsError; +use crate::path::{VfsPath, VfsPathBuf, canonicalize}; +use crate::types::{Entry, GrepMatch, GrepQuery, GrepResults, Stat}; + +/// Identity of one serial thread of execution. Process-unique, vended +/// from a process-global monotonic counter. Opaque: no public constructor - +/// it must be nameable (it appears in [`Vfs::acquire`]), but only the +/// handle vends them. +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +pub struct ExecId(u64); + +impl ExecId { + /// Vends the next process-unique identity. + // The handle arrives in a later step; nothing vends identities today. + #[allow(dead_code)] + pub(crate) fn vend() -> Self { + static NEXT: AtomicU64 = AtomicU64::new(1); + Self(NEXT.fetch_add(1, Ordering::Relaxed)) + } +} + +/// One backend behind the virtual namespace. +/// +/// Sync by design: the Lua VM and the executor's single driver thread are +/// synchronous. Bytes at the operation level. `Send` is required, `Sync` +/// is not: the handle serializes access. The only way to touch storage is +/// to acquire an access object bound to an identity. +pub trait Vfs: Send { + /// Acquires an access object bound to `id`. Every operation on the + /// returned object is attributed to that identity: backends that + /// care can know who is touching what; the rest ignore it. + fn acquire(&mut self, id: ExecId) -> Result, VfsError>; + + /// Releases `id`. Also called from the access object's Drop, so + /// teardown paths (cancel, panic, early return) cannot skip it. + fn release(&mut self, id: ExecId) -> Result<(), VfsError>; + + /// Whether this backend rejects all mutations. + fn read_only(&self) -> bool { + false + } +} + +/// One identity's session with a backend. Holds the ExecId. +/// All filesystem operations live here - no access object, no ops. +/// Paths arrive validated, canonicalized, and interned; backends never +/// re-validate. +pub trait VfsAccess: Send { + /// Reads the file at `path` exactly as stored. + fn read(&self, path: &VfsPath) -> Result, VfsError>; + + /// Reads `len` bytes starting at byte `offset`. + /// + /// Default: read whole, slice. Backends that can seek (host + /// directory, SQLite) override and never materialize the file. + /// The handle's line-based ranges are built on this. + fn read_range(&self, path: &VfsPath, offset: u64, len: u64) -> Result, VfsError> { + let data = self.read(path)?; + let Ok(start) = usize::try_from(offset) else { + return Err(VfsError::Backend(format!( + "read_range offset {offset} exceeds the addressable size" + ))); + }; + let Ok(length) = usize::try_from(len) else { + return Err(VfsError::Backend(format!( + "read_range length {len} exceeds the addressable size" + ))); + }; + if start >= data.len() { + return Ok(Vec::new()); + } + let end = data.len().min(start.saturating_add(length)); + Ok(data[start..end].to_vec()) + } + + /// Creates or overwrites the file at `path`. + /// + /// Noted but not implemented in v1: a defaulted + /// `write_owned(&mut self, path: &VfsPath, contents: Vec)` + /// delegating to `write`, which the memory overlay would override to + /// move the buffer with zero copies. Add when profiling calls for it. + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError>; + + /// Appends to the file at `path`, creating it if absent. + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError>; + + /// Removes the file, link, or directory at `path`. + /// Absent is NotFound; a directory without `recursive` is an error. + /// On a symlink, removes the link, never the target. + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError>; + + /// A confirmed absence is `Ok(false)`; a backend failure is `Err`. + fn exists(&self, path: &VfsPath) -> Result; + + /// Returns stored paths matching `pattern`, sorted. + fn glob(&self, pattern: &str) -> Result, VfsError>; + + /// Lists the directory at `path`. + fn list(&self, path: &VfsPath) -> Result, VfsError>; + + /// Returns metadata for `path`. + fn stat(&self, path: &VfsPath) -> Result; + + /// Creates the directory at `path`. + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError>; + + /// Renames or moves, atomically where the backend allows. + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError>; + + /// Copies the file at `from` to `to`. + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError>; + + /// Replaces the unique occurrence of `old` with `new`. + /// Zero matches and multiple matches are both errors. + /// Default: read, count, replace, write. Override to push down. + fn str_replace(&mut self, path: &VfsPath, old: &str, new: &str) -> Result<(), VfsError> { + let bytes = self.read(path)?; + let text = String::from_utf8(bytes).map_err(|_| { + VfsError::Backend(format!("str_replace requires UTF-8 text: {path}")) + })?; + let count = text.matches(old).count(); + if count == 0 { + return Err(VfsError::Backend(format!( + "str_replace found no occurrence of {old:?} in {path}" + ))); + } + if count > 1 { + return Err(VfsError::Backend(format!( + "str_replace found {count} occurrences of {old:?} in {path}; exactly one is required" + ))); + } + let replaced = text.replacen(old, new, 1); + self.write(path, replaced.as_bytes()) + } + + /// Searches files under the query's root. + /// + /// Default: glob, read, line scan with literal substring matching. + /// Override for indexed backends. Regex queries return + /// [`VfsError::Unsupported`]: this crate is std-only, so a regex + /// engine must come from an overriding backend. Non-UTF-8 files and + /// directories are skipped. + fn grep(&self, query: &GrepQuery) -> Result { + if query.is_regex { + return Err(VfsError::Unsupported( + "the default grep matches literal text only; regex requires a backend override" + .into(), + )); + } + let base = match query.root.as_str() { + "/" => "", + root => root, + }; + let pattern = match &query.glob_filter { + Some(filter) => format!("{base}/**/{filter}"), + None => format!("{base}/**/*"), + }; + let mut matches = Vec::new(); + let mut truncated = false; + 'files: for path in self.glob(&pattern)? { + let vfs_path = canonicalize(&path)?; + let bytes = match self.read(&vfs_path) { + Ok(bytes) => bytes, + Err(VfsError::IsADirectory(_)) => continue, + Err(err) => return Err(err), + }; + let Ok(text) = String::from_utf8(bytes) else { + continue; + }; + for (index, line) in text.lines().enumerate() { + let hit = if query.case_insensitive { + line.to_lowercase() + .contains(&query.pattern.to_lowercase()) + } else { + line.contains(&query.pattern) + }; + if !hit { + continue; + } + if let Some(cap) = query.max_results { + if matches.len() >= cap { + truncated = true; + break 'files; + } + } + matches.push(GrepMatch { + path: path.clone(), + line_number: index + 1, + line: line.to_owned(), + }); + } + } + Ok(GrepResults { matches, truncated }) + } + + /// Creates a symbolic link at `link` naming `target`. + /// + /// POSIX extra; the default returns [`VfsError::Unsupported`]. + fn symlink(&mut self, target: &VfsPath, link: &VfsPath) -> Result<(), VfsError> { + let _ = target; + Err(VfsError::Unsupported(format!( + "symlink is not supported by this backend: {link}" + ))) + } + + /// Reads the target of the symbolic link at `path`. + /// + /// POSIX extra; the default returns [`VfsError::Unsupported`]. + fn read_link(&self, path: &VfsPath) -> Result { + Err(VfsError::Unsupported(format!( + "read_link is not supported by this backend: {path}" + ))) + } + + /// Changes the mode bits of `path`. + /// + /// POSIX extra; the default returns [`VfsError::Unsupported`]. + fn chmod(&mut self, path: &VfsPath, mode: u32) -> Result<(), VfsError> { + let _ = mode; + Err(VfsError::Unsupported(format!( + "chmod is not supported by this backend: {path}" + ))) + } +} + +/// What operation is being attempted - the policy matches on this. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum Op { + /// Reading a file's bytes. + Read, + /// Creating or overwriting a file. + Write, + /// Appending to a file. + Append, + /// Removing a file, link, or directory. + Delete, + /// Renaming or moving a path. + Rename, + /// Creating a directory. + Mkdir, + /// Copying a file. + Copy, + /// Searching file contents. + Grep, + /// Testing for existence. + Exists, + /// Matching paths against a pattern. + Glob, + /// Listing a directory. + List, + /// Reading metadata. + Stat, + /// Creating a symbolic link. + Symlink, + /// Reading a symbolic link's target. + ReadLink, + /// Changing mode bits. + Chmod, +} + +/// The policy's answer. Reasons are load-bearing in both directions: +/// Deny's string flows back to the model as the tool error (its +/// recovery path); Ask's string is what the user sees in the +/// approval dialog (what is being asked, and which rule fired). +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Verdict { + /// The operation may proceed. + Allow, + /// The operation is refused; the string is the model's recovery path. + Deny(String), + /// The operation needs user approval; the string is the dialog text. + Ask(String), +} + +/// One policy per VfsRef, consulted by Access on every operation, +/// before the claims check. Dynamic through shared state: the host +/// or UI holds the same Arc and changes behavior mid-run. +pub trait Policy: Send { + /// Decides whether `op` on `path` may proceed. + fn check(&self, op: Op, path: &VfsPath) -> Verdict; +} + +/// The v1 policy: every operation is allowed. +#[derive(Debug, Default)] +pub struct AllowAll; + +impl Policy for AllowAll { + fn check(&self, op: Op, path: &VfsPath) -> Verdict { + let _ = (op, path); + Verdict::Allow + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use super::{AllowAll, Op, Policy, Verdict, VfsAccess}; + use crate::error::VfsError; + use crate::path::{VfsPath, canonicalize}; + use crate::types::{Entry, GrepQuery, GrepResults, Stat}; + + /// Minimal in-memory backend exercising the trait defaults: the + /// required methods are direct map operations, and glob understands + /// the one pattern shape the default grep emits (`/**`). + struct StubBackend { + files: BTreeMap>, + } + + fn stub(files: &[(&str, &str)]) -> StubBackend { + StubBackend { + files: files + .iter() + .map(|(name, text)| ((*name).to_owned(), text.as_bytes().to_vec())) + .collect(), + } + } + + fn path(s: &str) -> Result { + canonicalize(s) + } + + fn query(root: &str, pattern: &str) -> Result { + Ok(GrepQuery { + pattern: pattern.to_owned(), + root: canonicalize(root)?.to_buf(), + is_regex: false, + case_insensitive: false, + glob_filter: None, + max_results: None, + }) + } + + impl VfsAccess for StubBackend { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + self.files + .get(path.as_str()) + .cloned() + .ok_or_else(|| VfsError::NotFound(path.to_string())) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.files.insert(path.to_string(), contents.to_vec()); + Ok(()) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.files + .entry(path.to_string()) + .or_default() + .extend_from_slice(contents); + Ok(()) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let _ = recursive; + self.files + .remove(path.as_str()) + .map(|_| ()) + .ok_or_else(|| VfsError::NotFound(path.to_string())) + } + + fn exists(&self, path: &VfsPath) -> Result { + Ok(self.files.contains_key(path.as_str())) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + let Some(index) = pattern.find("/**/") else { + return Ok(Vec::new()); + }; + let prefix = format!("{}/", &pattern[..index]); + let filter = &pattern[index + 4..]; + let suffix = filter.strip_prefix('*').unwrap_or(filter); + Ok(self + .files + .keys() + .filter(|name| name.starts_with(&prefix) && name.ends_with(suffix)) + .cloned() + .collect()) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + let _ = path; + Err(VfsError::Unsupported("the stub does not list".into())) + } + + fn stat(&self, path: &VfsPath) -> Result { + let _ = path; + Err(VfsError::Unsupported("the stub does not stat".into())) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let _ = (path, recursive); + Ok(()) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + let bytes = self + .files + .remove(from.as_str()) + .ok_or_else(|| VfsError::NotFound(from.to_string()))?; + self.files.insert(to.to_string(), bytes); + Ok(()) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + let bytes = self + .files + .get(from.as_str()) + .cloned() + .ok_or_else(|| VfsError::NotFound(from.to_string()))?; + self.files.insert(to.to_string(), bytes); + Ok(()) + } + } + + #[test] + fn the_default_read_range_slices_a_whole_read() -> Result<(), VfsError> { + let backend = stub(&[("/a.txt", "hello world")]); + let bytes = backend.read_range(&path("/a.txt")?, 6, 5)?; + assert_eq!(bytes, b"world"); + Ok(()) + } + + #[test] + fn the_default_read_range_clips_at_the_end_of_the_file() -> Result<(), VfsError> { + let backend = stub(&[("/a.txt", "hello")]); + assert_eq!(backend.read_range(&path("/a.txt")?, 2, 100)?, b"llo"); + assert!(backend.read_range(&path("/a.txt")?, 100, 5)?.is_empty()); + Ok(()) + } + + #[test] + fn the_default_str_replace_rewrites_the_unique_occurrence() -> Result<(), VfsError> { + let mut backend = stub(&[("/a.txt", "alpha beta gamma")]); + backend.str_replace(&path("/a.txt")?, "beta", "BETA")?; + assert_eq!(backend.read(&path("/a.txt")?)?, b"alpha BETA gamma"); + Ok(()) + } + + #[test] + fn the_default_str_replace_rejects_zero_matches() -> Result<(), VfsError> { + let mut backend = stub(&[("/a.txt", "alpha beta")]); + let result = backend.str_replace(&path("/a.txt")?, "missing", "x"); + assert!(matches!(result, Err(VfsError::Backend(_)))); + assert_eq!(backend.read(&path("/a.txt")?)?, b"alpha beta"); + Ok(()) + } + + #[test] + fn the_default_str_replace_rejects_multiple_matches() -> Result<(), VfsError> { + let mut backend = stub(&[("/a.txt", "foo and foo")]); + let result = backend.str_replace(&path("/a.txt")?, "foo", "bar"); + assert!(matches!(result, Err(VfsError::Backend(_)))); + assert_eq!(backend.read(&path("/a.txt")?)?, b"foo and foo"); + Ok(()) + } + + #[test] + fn the_default_grep_matches_literal_text_with_line_numbers() -> Result<(), VfsError> { + let backend = stub(&[ + ("/docs/a.md", "first hit line\nplain line\nsecond hit line"), + ("/docs/b.md", "nothing here"), + ]); + let results: GrepResults = backend.grep(&query("/docs", "hit")?)?; + assert!(!results.truncated); + assert_eq!(results.matches.len(), 2); + assert_eq!(results.matches[0].path, "/docs/a.md"); + assert_eq!(results.matches[0].line_number, 1); + assert_eq!(results.matches[0].line, "first hit line"); + assert_eq!(results.matches[1].line_number, 3); + Ok(()) + } + + #[test] + fn the_default_grep_honors_case_insensitive_matching() -> Result<(), VfsError> { + let backend = stub(&[("/a.txt", "MixedCase line")]); + let mut q = query("/", "mixedcase")?; + assert!(backend.grep(&q)?.matches.is_empty()); + q.case_insensitive = true; + assert_eq!(backend.grep(&q)?.matches.len(), 1); + Ok(()) + } + + #[test] + fn the_default_grep_scopes_the_search_to_the_glob_filter() -> Result<(), VfsError> { + let backend = stub(&[("/src/a.rs", "needle"), ("/src/b.txt", "needle")]); + let mut q = query("/src", "needle")?; + q.glob_filter = Some("*.rs".to_owned()); + let results = backend.grep(&q)?; + assert_eq!(results.matches.len(), 1); + assert_eq!(results.matches[0].path, "/src/a.rs"); + Ok(()) + } + + #[test] + fn the_default_grep_caps_results_and_reports_truncation() -> Result<(), VfsError> { + let backend = stub(&[("/a.txt", "hit\nhit\nhit")]); + let mut q = query("/", "hit")?; + q.max_results = Some(2); + let results = backend.grep(&q)?; + assert_eq!(results.matches.len(), 2); + assert!(results.truncated); + q.max_results = Some(10); + let results = backend.grep(&q)?; + assert_eq!(results.matches.len(), 3); + assert!(!results.truncated); + Ok(()) + } + + #[test] + fn the_default_grep_rejects_regex_without_a_backend_override() -> Result<(), VfsError> { + let backend = stub(&[("/a.txt", "hit")]); + let mut q = query("/", "h.t")?; + q.is_regex = true; + assert!(matches!(backend.grep(&q), Err(VfsError::Unsupported(_)))); + Ok(()) + } + + #[test] + fn unsupported_posix_defaults_return_the_right_error_kind() -> Result<(), VfsError> { + let mut backend = stub(&[("/a.txt", "x")]); + assert!(matches!( + backend.symlink(&path("/a.txt")?, &path("/b.txt")?), + Err(VfsError::Unsupported(_)) + )); + assert!(matches!( + backend.read_link(&path("/a.txt")?), + Err(VfsError::Unsupported(_)) + )); + assert!(matches!( + backend.chmod(&path("/a.txt")?, 0o644), + Err(VfsError::Unsupported(_)) + )); + Ok(()) + } + + #[test] + fn allow_all_permits_every_operation() -> Result<(), VfsError> { + let policy = AllowAll; + assert_eq!(policy.check(Op::Write, &path("/a.txt")?), Verdict::Allow); + Ok(()) + } +} diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index 5d31a72ac..1dcb32470 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -572,7 +572,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 2: Vfs and VfsAccess traits and policy types +### Step 2: Vfs and VfsAccess traits and policy types [completed] - Component: shared-vfs core - Define `trait Vfs` (acquire/release/read_only, Send, sync by design), `trait VfsAccess` (sixteen methods; defaults: read_range slices a whole read, str_replace is read-count-replace-write, grep is glob-read-line-scan, symlink/read_link/chmod return Unsupported), `trait Policy`, `enum Op`, `enum Verdict` (Deny and Ask carry reason strings), and `AllowAll`. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index 4b3acd792..686541d26 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -82,3 +82,7 @@ - Decision: the virtual namespace requires a leading `/`; drive-letter-style paths (`C:/x`) are rejected as relative, with host-path translation deferred to the host backend | Falsifier: a later step requires identity-mount virtual paths of the form `C:/...` to canonicalize. - Decision: case is preserved and comparison is case-sensitive (POSIX semantics) in the virtual namespace | Falsifier: a host-backend requirement mandates case-insensitive virtual-path comparison. - Decision: `canonicalize`/`intern` carry targeted `#[allow(dead_code)]` until `Access` (a later step) becomes their caller | Falsifier: the next step wires `Access` and the allows remain. +- Step 2: Vfs and VfsAccess traits and policy types - `cargo nextest run -p shared-vfs traits` - 12 passed, 0 failed. Review: clean. + - Decision: `Op` covers all sixteen access operations (the contract's `// ...` resolved to Exists/Glob/List/Stat/Symlink/ReadLink/Chmod) | Falsifier: a later step needs an op the policy cannot name. + - Decision: str_replace zero/multiple-match failures use `VfsError::Backend` with a descriptive message (no dedicated kind exists) | Falsifier: the facade or model recovery path needs to match on a distinct kind. + - Decision: default grep returns `Unsupported` for `is_regex` (std-only crate cannot ship a regex engine) and skips non-UTF-8 files | Falsifier: a caller requires regex semantics from the memory backend's default grep. From e32e031f96a1825086fc0ac8437f4212feefbacb Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 18:33:29 -0700 Subject: [PATCH 03/26] Add VfsRef handle and Access capability with claims The virtual filesystem gains its public entry points: a cloneable handle over one backend and a must-use capability through which every operation flows. Each acquisition vends a process-unique identity, and every operation canonicalizes its path, consults the installed policy, and registers a read or write claim before the backend is touched, so overlapping writers fail loudly instead of racing. Claims are keyed by canonical path so aliased spellings of one file collide, and dropping the capability releases them, which keeps cancellation and panic paths from leaking conflicts. Spawning a child retires the parent's claims, making the spawn the happens-before edge that keeps sequential fan-out legal. - `VfsRef::with_policy` installs the policy at construction next to the backend; `VfsRef::new` defaults to the allow-everything policy. This is the only seam through which a host interposes on operations. - `check_policy` collapses both the deny and ask verdicts into a permission-denied error carrying the policy's reason string. No approval prompt exists at this layer. - `Volume` holds the backend and the claims ledger as separately shareable arcs, so a later overlay can swap the backend while keeping one claims table. - `Claims` keeps the readers map, writers map, and live set behind one poison-safe mutex, and retired or released claims are deleted outright rather than tombstoned. - `gate` canonicalizes the path, consults the policy, then registers the claim, in that order, so a denied operation leaves no claim behind and every claim key is canonical. - `claim` fails a write when any other live identity claims the path and a read only when another identity writes it; an identity never conflicts with itself. The conflict error names the path, both identities, and both claim kinds. - `spawn` vends a fresh identity for the child and retires the parent's claims, and `impl Drop for Access` releases the identity's claims before a best-effort backend release. - `glob` registers its claim against the canonicalized pattern while the backend receives the pattern verbatim. - `read_range_numbered` emits one-based inclusive line ranges numbered absolutely from the start, padded to the widest emitted number. - `ExecId::vend` is now wired: acquisition and spawn call it, and its dead-code allowance is removed. - `VfsRef` wraps exactly one backend; there is no mount table, router, or overlay entry point yet. Design: new constructor-injection @ crates/shared-vfs/src/handle.rs::VfsRef boundary: pub Design: new shared-mutable-state @ crates/shared-vfs/src/handle.rs::Volume Design: new surface-growth @ crates/shared-vfs/src/handle.rs boundary: pub Plan: vibe/2026-09-11-3-vfs-foundation.md --- crates/shared-vfs/src/handle.rs | 1067 +++++++++++++++++++++++++++ crates/shared-vfs/src/lib.rs | 2 + crates/shared-vfs/src/traits.rs | 105 ++- vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 6 + 5 files changed, 1169 insertions(+), 13 deletions(-) create mode 100644 crates/shared-vfs/src/handle.rs diff --git a/crates/shared-vfs/src/handle.rs b/crates/shared-vfs/src/handle.rs new file mode 100644 index 000000000..dc39d4296 --- /dev/null +++ b/crates/shared-vfs/src/handle.rs @@ -0,0 +1,1067 @@ +//! The cloneable handle, the RAII capability, and the claims tables. +//! +//! [`VfsRef`] is the public handle: an `Arc`-shared volume pairing one +//! backend with the claims ledger, behind poison-safe locks. [`Access`] +//! is the RAII capability vended by [`VfsRef::acquire`]: it canonicalizes +//! paths at receipt, consults the handle's policy before the claims check +//! so a denied operation never registers a claim, registers claims, and +//! locks the backend's access object per call. Dropping an [`Access`] +//! releases the identity and its claims, so cancellation, panics, and +//! early returns cannot leak claims. + +use std::collections::{HashMap, HashSet}; +use std::fmt; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::error::VfsError; +use crate::path::{VfsPath, canonicalize}; +use crate::traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; +use crate::types::{Entry, GrepQuery, GrepResults, Stat}; + +/// Whether an operation claims read or write intent on its path. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ClaimKind { + Read, + Write, +} + +impl fmt::Display for ClaimKind { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Read => f.write_str("read"), + Self::Write => f.write_str("write"), + } + } +} + +/// The bookkeeping of who is touching what. Every entry is a live, +/// conflict-eligible claim; retired or released claims are deleted, +/// never stored. +struct Claims { + inner: Mutex, +} + +struct ClaimsTables { + readers: HashMap>, + writers: HashMap>, + live: HashSet, +} + +impl Claims { + fn new() -> Self { + Self { + inner: Mutex::new(ClaimsTables { + readers: HashMap::new(), + writers: HashMap::new(), + live: HashSet::new(), + }), + } + } + + /// Poison-safe lock: each guard scope is one complete table mutation, + /// so a panicking claimant cannot leave the tables half-updated. + fn tables(&self) -> MutexGuard<'_, ClaimsTables> { + self.inner.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Marks `id` as live. Only live identities hold claims; released + /// ones are forgotten entirely. + fn register_live(&self, id: ExecId) { + self.tables().live.insert(id); + } + + /// Registers `id`'s claim of `kind` on `path`, failing when another + /// live identity already holds a conflicting claim. A write conflicts + /// with any other identity in the path's readers or writers; a read + /// conflicts only with another identity in its writers; read-read + /// never conflicts. An identity never conflicts with itself. + fn claim(&self, path: VfsPath, id: ExecId, kind: ClaimKind) -> Result<(), VfsError> { + let mut tables = self.tables(); + if let Some(other) = other_claimant(&tables.writers, path, id) { + return Err(conflict(path, id, kind, other, ClaimKind::Write)); + } + if kind == ClaimKind::Write + && let Some(other) = other_claimant(&tables.readers, path, id) + { + return Err(conflict(path, id, kind, other, ClaimKind::Read)); + } + let map = match kind { + ClaimKind::Read => &mut tables.readers, + ClaimKind::Write => &mut tables.writers, + }; + let claimants = map.entry(path).or_default(); + if !claimants.contains(&id) { + claimants.push(id); + } + Ok(()) + } + + /// Deletes `id`'s claims but keeps the identity live: the spawn of a + /// child is the happens-before edge, so claims that predate the child + /// can never conflict again. + fn retire(&self, id: ExecId) { + delete_claims(&mut self.tables(), id); + } + + /// Deletes `id`'s claims and forgets the identity. + fn release(&self, id: ExecId) { + let mut tables = self.tables(); + tables.live.remove(&id); + delete_claims(&mut tables, id); + } +} + +/// Returns the first claimant of `path` in `map` other than `id`. +fn other_claimant( + map: &HashMap>, + path: VfsPath, + id: ExecId, +) -> Option { + map.get(&path)?.iter().find(|&&other| other != id).copied() +} + +/// Removes every claim held by `id`, dropping emptied path entries. +fn delete_claims(tables: &mut ClaimsTables, id: ExecId) { + tables.readers.retain(|_, ids| { + ids.retain(|&other| other != id); + !ids.is_empty() + }); + tables.writers.retain(|_, ids| { + ids.retain(|&other| other != id); + !ids.is_empty() + }); +} + +/// The conflict error names the path, both identities, and both claim +/// kinds: the executor maps it to a fatal run error, and the message is +/// the whole diagnosis. +fn conflict( + path: VfsPath, + id: ExecId, + kind: ClaimKind, + other: ExecId, + other_kind: ClaimKind, +) -> VfsError { + VfsError::Conflict(format!( + "{kind} on {path} by {id:?} conflicts with a {other_kind} claim by {other:?}" + )) +} + +/// One mounted filesystem instance: its backend and the ledger of who is +/// touching what. The two are separately `Arc`-shareable so `overlay()` +/// (a later step) can share the claims table while swapping the backend. +struct Volume { + backend: Arc>>, + claims: Arc, +} + +/// The cloneable handle over one backend and its claims ledger. +/// +/// Clones share the backend, the claims tables, and the policy: claims +/// registered through one clone conflict with operations through another. +#[derive(Clone)] +pub struct VfsRef { + volume: Arc, + policy: Arc, +} + +impl fmt::Debug for VfsRef { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VfsRef").finish_non_exhaustive() + } +} + +impl VfsRef { + /// Returns a handle over `backend` with the [`AllowAll`] policy. + pub fn new(backend: impl Vfs + 'static) -> VfsRef { + Self::with_policy(backend, AllowAll) + } + + /// Returns a handle over `backend` consulting `policy` on every + /// operation. The policy is dynamic through shared state: the host + /// holds the same `Arc` and changes behavior mid-run, and the next + /// operation sees it. + pub fn with_policy( + backend: impl Vfs + 'static, + policy: impl Policy + Sync + 'static, + ) -> VfsRef { + VfsRef { + volume: Arc::new(Volume { + backend: Arc::new(Mutex::new(Box::new(backend))), + claims: Arc::new(Claims::new()), + }), + policy: Arc::new(policy), + } + } + + /// Acquires the capability for a new serial thread of execution. + /// This is the only way in: every acquire vends a fresh [`ExecId`]. + /// + /// # Panics + /// Panics when the backend fails to acquire the identity. Backends + /// are expected to accept attribution; a refusal is a backend bug, + /// not a runtime condition. + pub fn acquire(&self) -> Access { + let id = ExecId::vend(); + let inner = self + .backend() + .acquire(id) + .unwrap_or_else(|err| panic!("the backend refused to acquire identity {id:?}: {err}")); + self.volume.claims.register_live(id); + Access { + id, + volume: self.volume.clone(), + policy: self.policy.clone(), + inner: Mutex::new(inner), + } + } + + /// Poison-safe lock on the backend. + fn backend(&self) -> MutexGuard<'_, Box> { + self.volume + .backend + .lock() + .unwrap_or_else(PoisonError::into_inner) + } +} + +/// The public capability. Holds an [`ExecId`] and the backend's access +/// object; every operation canonicalizes the path, checks the policy, +/// checks the claims tables, then locks the backend per call. +#[must_use = "an acquire dropped immediately is a bug: the capability carries the identity's claims"] +pub struct Access { + id: ExecId, + volume: Arc, + policy: Arc, + inner: Mutex>, +} + +impl Access { + /// Returns the capability for a new concurrent thread of execution. + /// The child gets a fresh [`ExecId`], and this capability's claims + /// are deleted from the tables: they predate the child by + /// construction, so a retired claim can never conflict again. The + /// spawn IS the happens-before edge - no fence call, no epochs. + /// + /// # Panics + /// Panics when the backend fails to acquire the child's identity; + /// see [`VfsRef::acquire`]. + pub fn spawn(&self) -> Access { + let id = ExecId::vend(); + let inner = self + .backend() + .acquire(id) + .unwrap_or_else(|err| panic!("the backend refused to acquire identity {id:?}: {err}")); + self.volume.claims.retire(self.id); + self.volume.claims.register_live(id); + Access { + id, + volume: self.volume.clone(), + policy: self.policy.clone(), + inner: Mutex::new(inner), + } + } + + /// Returns this capability's identity. + pub fn id(&self) -> ExecId { + self.id + } + + /// Reads the file at `path` exactly as stored. + /// + /// # Errors + /// Returns an error when the policy denies the read, when another + /// live identity holds a write claim on `path`, or when the backend + /// fails. + pub fn read(&self, path: &str) -> Result, VfsError> { + let path = self.gate(Op::Read, path, ClaimKind::Read)?; + self.inner().read(&path) + } + + /// Reads the file at `path` as UTF-8 text. + /// + /// # Errors + /// Returns an error when the file's contents are not UTF-8. + pub fn read_string(&self, path: &str) -> Result { + let bytes = self.read(path)?; + String::from_utf8(bytes) + .map_err(|_| VfsError::Backend(format!("read_string requires UTF-8 text: {path}"))) + } + + /// Reads lines `start..=end` of the file at `path`, 1-based and + /// inclusive, joined by `"\n"` with no trailing newline. An omitted + /// `end` means the last line; a given `end` clamps down to it; a + /// `start` past the last line reads as the empty string. + /// + /// # Errors + /// Returns an error when `start` is below 1 or `end` is before + /// `start`, when the file is missing, or when it is not UTF-8. + pub fn read_range( + &self, + path: &str, + start: usize, + end: Option, + ) -> Result { + self.with_line_range(path, start, end, |lines, _| lines.join("\n")) + } + + /// Reads lines `start..=end` as numbered lines, numbered absolutely + /// from `start`, each right-aligned to the width of the largest + /// emitted number and followed by `"| "`. Bounds behave exactly as + /// in [`Access::read_range`]. + /// + /// # Errors + /// Returns an error under the same conditions as + /// [`Access::read_range`]. + pub fn read_range_numbered( + &self, + path: &str, + start: usize, + end: Option, + ) -> Result { + self.with_line_range(path, start, end, |lines, first| { + let width = (first + lines.len() - 1).to_string().len(); + lines + .iter() + .enumerate() + .map(|(index, line)| format!("{:>width$}| {}", first + index, line)) + .collect::>() + .join("\n") + }) + } + + /// Creates or overwrites the file at `path`. + /// + /// # Errors + /// Returns an error when the policy denies the write, when another + /// live identity holds a claim on `path`, or when the backend fails. + pub fn write(&self, path: &str, contents: &[u8]) -> Result<(), VfsError> { + let path = self.gate(Op::Write, path, ClaimKind::Write)?; + self.inner().write(&path, contents) + } + + /// Appends to the file at `path`, creating it if absent. + /// + /// # Errors + /// Returns an error when the policy denies the append, when another + /// live identity holds a claim on `path`, or when the backend fails. + pub fn append(&self, path: &str, contents: &[u8]) -> Result<(), VfsError> { + let path = self.gate(Op::Append, path, ClaimKind::Write)?; + self.inner().append(&path, contents) + } + + /// Replaces the unique occurrence of `old` with `new` in the file at + /// `path`. Zero matches and multiple matches are both errors. + /// + /// # Errors + /// Returns an error when the policy denies the write, when another + /// live identity holds a claim on `path`, when the match count is not + /// exactly one, or when the backend fails. + pub fn str_replace(&self, path: &str, old: &str, new: &str) -> Result<(), VfsError> { + let path = self.gate(Op::Write, path, ClaimKind::Write)?; + self.inner().str_replace(&path, old, new) + } + + /// Removes the file, link, or directory at `path`. + /// + /// # Errors + /// Returns an error when the policy denies the delete, when another + /// live identity holds a claim on `path`, or when the backend fails. + pub fn remove(&self, path: &str, recursive: bool) -> Result<(), VfsError> { + let path = self.gate(Op::Delete, path, ClaimKind::Write)?; + self.inner().remove(&path, recursive) + } + + /// A confirmed absence is `Ok(false)`; a backend failure is `Err`. + /// + /// # Errors + /// Returns an error when the policy denies the check, when another + /// live identity holds a write claim on `path`, or when the backend + /// fails. + pub fn exists(&self, path: &str) -> Result { + let path = self.gate(Op::Exists, path, ClaimKind::Read)?; + self.inner().exists(&path) + } + + /// Returns stored paths matching `pattern`, sorted. + /// + /// # Errors + /// Returns an error when the policy denies the glob or when the + /// backend fails. + pub fn glob(&self, pattern: &str) -> Result, VfsError> { + // The claim key is the canonicalized pattern; the backend + // receives the pattern verbatim. + let _claimed = self.gate(Op::Glob, pattern, ClaimKind::Read)?; + self.inner().glob(pattern) + } + + /// Lists the directory at `path`. + /// + /// # Errors + /// Returns an error when the policy denies the list, when another + /// live identity holds a write claim on `path`, or when the backend + /// fails. + pub fn list(&self, path: &str) -> Result, VfsError> { + let path = self.gate(Op::List, path, ClaimKind::Read)?; + self.inner().list(&path) + } + + /// Returns metadata for `path`. + /// + /// # Errors + /// Returns an error when the policy denies the stat, when another + /// live identity holds a write claim on `path`, or when the backend + /// fails. + pub fn stat(&self, path: &str) -> Result { + let path = self.gate(Op::Stat, path, ClaimKind::Read)?; + self.inner().stat(&path) + } + + /// Creates the directory at `path`. + /// + /// # Errors + /// Returns an error when the policy denies the mkdir, when another + /// live identity holds a claim on `path`, or when the backend fails. + pub fn mkdir(&self, path: &str, recursive: bool) -> Result<(), VfsError> { + let path = self.gate(Op::Mkdir, path, ClaimKind::Write)?; + self.inner().mkdir(&path, recursive) + } + + /// Renames or moves, atomically where the backend allows. Both paths + /// are claimed as writes. + /// + /// # Errors + /// Returns an error when the policy denies the rename, when another + /// live identity holds a claim on either path, or when the backend + /// fails. + pub fn rename(&self, from: &str, to: &str) -> Result<(), VfsError> { + let from = self.gate(Op::Rename, from, ClaimKind::Write)?; + let to = self.gate(Op::Rename, to, ClaimKind::Write)?; + self.inner().rename(&from, &to) + } + + /// Copies the file at `from` to `to`. The source is claimed as a + /// read, the destination as a write. + /// + /// # Errors + /// Returns an error when the policy denies the copy, when another + /// live identity holds a conflicting claim on either path, or when + /// the backend fails. + pub fn copy(&self, from: &str, to: &str) -> Result<(), VfsError> { + let from = self.gate(Op::Copy, from, ClaimKind::Read)?; + let to = self.gate(Op::Copy, to, ClaimKind::Write)?; + self.inner().copy(&from, &to) + } + + /// Searches files under the query's root. + /// + /// # Errors + /// Returns an error when the policy denies the search, when another + /// live identity holds a write claim on the query's root, or when the + /// backend fails. + pub fn grep(&self, query: &GrepQuery) -> Result { + let root = canonicalize(query.root.as_str())?; + self.check_policy(Op::Grep, root)?; + self.volume.claims.claim(root, self.id, ClaimKind::Read)?; + self.inner().grep(query) + } + + /// Canonicalizes at receipt, consults the policy, then registers the + /// claim - in that order, so a denied operation never registers a + /// claim and every claim key is the canonical interned path. + fn gate(&self, op: Op, path: &str, claim: ClaimKind) -> Result { + let path = canonicalize(path)?; + self.check_policy(op, path)?; + self.volume.claims.claim(path, self.id, claim)?; + Ok(path) + } + + /// Consults the handle's policy. v1 maps `Ask` to `PermissionDenied`: + /// the approval dialog is a host concern above this layer, and the + /// reason string still names what was asked and which rule fired. + fn check_policy(&self, op: Op, path: VfsPath) -> Result<(), VfsError> { + match self.policy.check(op, &path) { + Verdict::Allow => Ok(()), + Verdict::Deny(reason) | Verdict::Ask(reason) => Err(VfsError::PermissionDenied(reason)), + } + } + + /// Reads the file and resolves one line range while its contents + /// remain live. + fn with_line_range( + &self, + path: &str, + start: usize, + end: Option, + render: impl FnOnce(&[&str], usize) -> String, + ) -> Result { + if start < 1 { + return Err(VfsError::Backend(format!( + "invalid line range for {path}: start {start} is below 1" + ))); + } + let contents = self.read_string(path)?; + let lines: Vec<&str> = contents.lines().collect(); + if start > lines.len() { + return Ok(String::new()); + } + let end = end.unwrap_or(lines.len()).min(lines.len()); + if end < start { + return Err(VfsError::Backend(format!( + "invalid line range for {path}: end {end} is before start {start}" + ))); + } + Ok(render(&lines[start - 1..end], start)) + } + + /// Poison-safe lock on the backend's access object, held per call. + fn inner(&self) -> MutexGuard<'_, Box> { + self.inner.lock().unwrap_or_else(PoisonError::into_inner) + } + + /// Poison-safe lock on the backend. + fn backend(&self) -> MutexGuard<'_, Box> { + self.volume + .backend + .lock() + .unwrap_or_else(PoisonError::into_inner) + } +} + +impl fmt::Debug for Access { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("Access") + .field("id", &self.id) + .finish_non_exhaustive() + } +} + +impl Drop for Access { + fn drop(&mut self) { + self.volume.claims.release(self.id); + // Releasing the identity at the backend is best-effort: the + // claims are already gone, so a backend failure here cannot + // leave a conflict behind. + let _ = self.backend().release(self.id); + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + + use super::{Access, VfsRef}; + use crate::error::VfsError; + use crate::path::VfsPath; + use crate::traits::{ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; + use crate::types::{Entry, Stat}; + + /// Minimal in-memory backend shared between the `Vfs` and the access + /// objects it vends. Releases are recorded so tests can observe the + /// identity lifecycle. + #[derive(Clone, Default)] + struct StubFs { + files: Arc>>>, + released: Arc>>, + } + + impl StubFs { + fn seeded(files: &[(&str, &str)]) -> StubFs { + let stub = StubFs::default(); + for (name, text) in files { + stub.files() + .insert((*name).to_owned(), text.as_bytes().to_vec()); + } + stub + } + + fn files(&self) -> MutexGuard<'_, BTreeMap>> { + self.files.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn released(&self) -> Vec { + self.released + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + } + + impl Vfs for StubFs { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + let _ = id; + Ok(Box::new(StubAccess { + files: Arc::clone(&self.files), + })) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + self.released + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(id); + Ok(()) + } + } + + struct StubAccess { + files: Arc>>>, + } + + impl StubAccess { + fn files(&self) -> MutexGuard<'_, BTreeMap>> { + self.files.lock().unwrap_or_else(PoisonError::into_inner) + } + } + + impl VfsAccess for StubAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + self.files() + .get(path.as_str()) + .cloned() + .ok_or_else(|| VfsError::NotFound(path.to_string())) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.files().insert(path.to_string(), contents.to_vec()); + Ok(()) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.files() + .entry(path.to_string()) + .or_default() + .extend_from_slice(contents); + Ok(()) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let _ = recursive; + self.files() + .remove(path.as_str()) + .map(|_| ()) + .ok_or_else(|| VfsError::NotFound(path.to_string())) + } + + fn exists(&self, path: &VfsPath) -> Result { + Ok(self.files().contains_key(path.as_str())) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + let prefix = pattern.split('*').next().unwrap_or(pattern); + Ok(self + .files() + .keys() + .filter(|name| name.starts_with(prefix)) + .cloned() + .collect()) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + let _ = path; + Err(VfsError::Unsupported("the stub does not list".into())) + } + + fn stat(&self, path: &VfsPath) -> Result { + let _ = path; + Err(VfsError::Unsupported("the stub does not stat".into())) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let _ = (path, recursive); + Ok(()) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + let bytes = self + .files() + .remove(from.as_str()) + .ok_or_else(|| VfsError::NotFound(from.to_string()))?; + self.files().insert(to.to_string(), bytes); + Ok(()) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + let bytes = self + .files() + .get(from.as_str()) + .cloned() + .ok_or_else(|| VfsError::NotFound(from.to_string()))?; + self.files().insert(to.to_string(), bytes); + Ok(()) + } + } + + fn handle(stub: &StubFs) -> VfsRef { + VfsRef::new(stub.clone()) + } + + /// Extracts the Conflict message or fails the test. + fn conflict_message(result: Result<(), VfsError>) -> String { + match result { + Err(VfsError::Conflict(message)) => message, + other => panic!("expected a conflict, got {other:?}"), + } + } + + #[test] + fn an_access_reads_and_writes_through_the_handle() -> Result<(), VfsError> { + let vfs = handle(&StubFs::default()); + let access = vfs.acquire(); + access.write("/notes/a.txt", b"hello")?; + assert_eq!(access.read("/notes/a.txt")?, b"hello"); + assert!(access.exists("/notes/a.txt")?); + Ok(()) + } + + #[test] + fn every_acquire_vends_a_process_unique_identity() { + let vfs = handle(&StubFs::default()); + let first = vfs.acquire(); + let second = vfs.acquire(); + assert_ne!(first.id(), second.id()); + } + + #[test] + fn one_identity_never_conflicts_with_itself() -> Result<(), VfsError> { + // Borrow semantics: a blocking call chain uses the parent's + // access, so sequential ops on one path by one identity stay + // legal - no new identity, no false conflict. + let vfs = handle(&StubFs::default()); + let access = vfs.acquire(); + access.write("/f.txt", b"one")?; + access.write("/f.txt", b"two")?; + access.append("/f.txt", b"!")?; + assert_eq!(access.read("/f.txt")?, b"two!"); + Ok(()) + } + + #[test] + fn a_write_conflicts_with_another_identitys_read_claim() -> Result<(), VfsError> { + let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); + let reader = vfs.acquire(); + let writer = vfs.acquire(); + reader.read("/f.txt")?; + let message = conflict_message(writer.write("/f.txt", b"new")); + assert!(message.contains("/f.txt"), "names the path: {message}"); + assert!( + message.contains("read"), + "names the standing claim kind: {message}" + ); + assert!( + message.contains("write"), + "names the attempted kind: {message}" + ); + assert!( + message.contains(&format!("{:?}", reader.id())), + "names the claimant: {message}" + ); + assert!( + message.contains(&format!("{:?}", writer.id())), + "names the attempter: {message}" + ); + Ok(()) + } + + #[test] + fn a_read_conflicts_with_another_identitys_write_claim() -> Result<(), VfsError> { + let vfs = handle(&StubFs::default()); + let writer = vfs.acquire(); + writer.write("/f.txt", b"x")?; + let reader = vfs.acquire(); + match reader.read("/f.txt") { + Err(VfsError::Conflict(_)) => {} + other => panic!("expected a conflict, got {other:?}"), + } + Ok(()) + } + + #[test] + fn two_writes_by_two_identities_conflict() -> Result<(), VfsError> { + let vfs = handle(&StubFs::default()); + let first = vfs.acquire(); + first.write("/f.txt", b"1")?; + let second = vfs.acquire(); + let message = conflict_message(second.write("/f.txt", b"2")); + assert!(message.contains("write claim"), "{message}"); + Ok(()) + } + + #[test] + fn reads_by_two_identities_never_conflict() -> Result<(), VfsError> { + let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); + let first = vfs.acquire(); + let second = vfs.acquire(); + first.read("/f.txt")?; + assert_eq!(second.read("/f.txt")?, b"data"); + Ok(()) + } + + #[test] + fn a_copy_conflicts_with_another_identitys_write_on_the_source() -> Result<(), VfsError> { + // Copy claims the source as a read, and a read booms on another + // live identity's write claim. + let vfs = handle(&StubFs::default()); + let writer = vfs.acquire(); + writer.write("/src.txt", b"data")?; + let copier = vfs.acquire(); + let message = conflict_message(copier.copy("/src.txt", "/dst.txt")); + assert!(message.contains("/src.txt"), "names the source: {message}"); + Ok(()) + } + + #[test] + fn a_copy_shares_the_source_with_another_identitys_read() -> Result<(), VfsError> { + // The source claim is a read, not a write: another identity's + // read claim on the source must not block the copy. Were the + // source claimed as a write, this copy would conflict. + let vfs = handle(&StubFs::seeded(&[("/src.txt", "data")])); + let reader = vfs.acquire(); + reader.read("/src.txt")?; + let copier = vfs.acquire(); + copier.copy("/src.txt", "/dst.txt")?; + assert_eq!(copier.read("/dst.txt")?, b"data"); + Ok(()) + } + + #[test] + fn a_copy_conflicts_with_another_identitys_claim_on_the_destination() -> Result<(), VfsError> { + // The destination is claimed as a write, so any other live + // identity's claim on it blocks the copy. + let vfs = handle(&StubFs::seeded(&[ + ("/src.txt", "data"), + ("/dst.txt", "old"), + ])); + let reader = vfs.acquire(); + reader.read("/dst.txt")?; + let copier = vfs.acquire(); + let message = conflict_message(copier.copy("/src.txt", "/dst.txt")); + assert!( + message.contains("/dst.txt"), + "names the destination: {message}" + ); + Ok(()) + } + + #[test] + fn a_rename_conflicts_with_a_claim_on_the_source_path() -> Result<(), VfsError> { + let vfs = handle(&StubFs::seeded(&[("/from.txt", "data")])); + let reader = vfs.acquire(); + reader.read("/from.txt")?; + let renamer = vfs.acquire(); + let message = conflict_message(renamer.rename("/from.txt", "/to.txt")); + assert!(message.contains("/from.txt"), "names the source: {message}"); + Ok(()) + } + + #[test] + fn a_rename_conflicts_with_a_claim_on_the_destination_path() -> Result<(), VfsError> { + // Both paths are claimed as writes; were the second gate dropped, + // this rename would sail through against the standing claim. + let vfs = handle(&StubFs::seeded(&[ + ("/from.txt", "data"), + ("/to.txt", "old"), + ])); + let reader = vfs.acquire(); + reader.read("/to.txt")?; + let renamer = vfs.acquire(); + let message = conflict_message(renamer.rename("/from.txt", "/to.txt")); + assert!( + message.contains("/to.txt"), + "names the destination: {message}" + ); + Ok(()) + } + + #[test] + fn dropping_an_access_releases_its_identity_and_claims() -> Result<(), VfsError> { + let stub = StubFs::default(); + let vfs = handle(&stub); + let first = vfs.acquire(); + let first_id = first.id(); + first.write("/f.txt", b"1")?; + drop(first); + assert!(stub.released().contains(&first_id)); + let second = vfs.acquire(); + second.write("/f.txt", b"2")?; + assert_eq!(second.read("/f.txt")?, b"2"); + Ok(()) + } + + #[test] + fn spawn_deletes_the_parents_claims() -> Result<(), VfsError> { + let vfs = handle(&StubFs::default()); + let parent = vfs.acquire(); + parent.write("/f.txt", b"1")?; + let child = parent.spawn(); + assert_ne!(parent.id(), child.id()); + // The parent's pre-spawn write claim is retired: the child can + // touch the same path without a false conflict. + child.write("/f.txt", b"2")?; + assert_eq!(child.read("/f.txt")?, b"2"); + Ok(()) + } + + #[test] + fn sequential_fanout_arms_stay_legal() -> Result<(), VfsError> { + // The pattern the claims model teaches: the parent spawns each + // arm in turn; a dropped arm releases its claims, so the next arm + // can merge onto the same path. + let vfs = handle(&StubFs::default()); + let parent = vfs.acquire(); + let arm_one = parent.spawn(); + arm_one.write("/evidence.md", b"one\n")?; + drop(arm_one); + let arm_two = parent.spawn(); + arm_two.append("/evidence.md", b"two\n")?; + assert_eq!(arm_two.read("/evidence.md")?, b"one\ntwo\n"); + Ok(()) + } + + #[test] + fn transfer_of_control_moves_the_claims_with_the_access() -> Result<(), VfsError> { + let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); + let original = vfs.acquire(); + original.read("/f.txt")?; + // Transfer of control moves the access object; the identity and + // its claims move with it. + let moved = original; + let other = vfs.acquire(); + let message = conflict_message(other.write("/f.txt", b"new")); + assert!(message.contains(&format!("{:?}", moved.id()))); + assert_eq!(moved.read("/f.txt")?, b"data"); + Ok(()) + } + + #[test] + fn alias_spellings_of_one_file_land_on_one_claim_key() -> Result<(), VfsError> { + let vfs = handle(&StubFs::seeded(&[("/a/b.txt", "x")])); + let reader = vfs.acquire(); + reader.read("/a/./b.txt")?; + let writer = vfs.acquire(); + let message = conflict_message(writer.write("/a//b.txt", b"y")); + assert!(message.contains("/a/b.txt"), "the canonical key: {message}"); + Ok(()) + } + + #[test] + fn claims_are_shared_across_handle_clones() -> Result<(), VfsError> { + let vfs = handle(&StubFs::default()); + let clone = vfs.clone(); + let first = vfs.acquire(); + first.write("/f.txt", b"1")?; + let second = clone.acquire(); + let message = conflict_message(second.write("/f.txt", b"2")); + assert!(message.contains("/f.txt"), "{message}"); + Ok(()) + } + + #[test] + fn a_denied_operation_never_registers_a_claim() -> Result<(), VfsError> { + /// A policy whose verdict flips through shared state mid-run. + struct FlipPolicy { + verdict: Arc>, + } + + impl Policy for FlipPolicy { + fn check(&self, op: Op, path: &VfsPath) -> Verdict { + let _ = (op, path); + self.verdict + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + } + + let verdict = Arc::new(Mutex::new(Verdict::Deny("writes are sealed".to_owned()))); + let vfs = VfsRef::with_policy( + StubFs::default(), + FlipPolicy { + verdict: Arc::clone(&verdict), + }, + ); + let denied = vfs.acquire(); + match denied.write("/f.txt", b"x") { + Err(VfsError::PermissionDenied(reason)) => { + assert_eq!(reason, "writes are sealed"); + } + other => panic!("expected a denial, got {other:?}"), + } + // The host flips the policy mid-run through shared state. + *verdict.lock().unwrap_or_else(PoisonError::into_inner) = Verdict::Allow; + let allowed = vfs.acquire(); + // Had the denied attempt registered a write claim, this write + // would conflict with it. + allowed.write("/f.txt", b"x")?; + assert_eq!(allowed.read("/f.txt")?, b"x"); + Ok(()) + } + + #[test] + fn read_range_slices_lines_one_based_and_inclusive() -> Result<(), VfsError> { + let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\nthree\n")])); + let access = vfs.acquire(); + assert_eq!(access.read_range("/f.txt", 2, None)?, "two\nthree"); + assert_eq!(access.read_range("/f.txt", 2, Some(99))?, "two\nthree"); + assert_eq!(access.read_range("/f.txt", 99, None)?, ""); + assert_eq!(access.read_range("/f.txt", 1, Some(1))?, "one"); + Ok(()) + } + + #[test] + fn read_range_rejects_invalid_bounds() { + let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\n")])); + let access = vfs.acquire(); + assert!(access.read_range("/f.txt", 0, None).is_err()); + assert!(access.read_range("/f.txt", 2, Some(1)).is_err()); + } + + #[test] + fn read_range_numbered_numbers_absolutely_from_start() -> Result<(), VfsError> { + let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\nthree\n")])); + let access = vfs.acquire(); + assert_eq!( + access.read_range_numbered("/f.txt", 1, None)?, + "1| one\n2| two\n3| three" + ); + assert_eq!( + access.read_range_numbered("/f.txt", 2, Some(3))?, + "2| two\n3| three" + ); + assert_eq!(access.read_range_numbered("/f.txt", 99, None)?, ""); + Ok(()) + } + + #[test] + fn read_range_numbered_pads_to_the_widest_emitted_number() -> Result<(), VfsError> { + let lines: Vec = (1..=10).map(|n| format!("line{n}")).collect(); + let text = lines.join("\n"); + let vfs = handle(&StubFs::seeded(&[("/f.txt", &text)])); + let access = vfs.acquire(); + assert_eq!( + access.read_range_numbered("/f.txt", 9, Some(10))?, + " 9| line9\n10| line10" + ); + Ok(()) + } + + #[test] + fn read_string_rejects_non_utf8() -> Result<(), VfsError> { + let vfs = handle(&StubFs::default()); + let access = vfs.acquire(); + access.write("/bin.dat", &[0xff, 0xfe])?; + match access.read_string("/bin.dat") { + Err(VfsError::Backend(_)) => {} + other => panic!("expected a UTF-8 failure, got {other:?}"), + } + Ok(()) + } + + #[test] + fn the_handle_and_capability_are_send_and_sync() { + fn assert_send_sync() {} + assert_send_sync::(); + assert_send_sync::(); + } +} diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs index b434d7fef..c52a483df 100644 --- a/crates/shared-vfs/src/lib.rs +++ b/crates/shared-vfs/src/lib.rs @@ -6,11 +6,13 @@ //! `/_promptforge` paths, no Store, no run concepts). mod error; +mod handle; mod path; mod traits; mod types; pub use error::VfsError; +pub use handle::{Access, VfsRef}; pub use path::{VfsPath, VfsPathBuf}; pub use traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; pub use types::{Entry, FileType, GrepMatch, GrepQuery, GrepResults, Stat}; diff --git a/crates/shared-vfs/src/traits.rs b/crates/shared-vfs/src/traits.rs index b4b05c0af..77ad05219 100644 --- a/crates/shared-vfs/src/traits.rs +++ b/crates/shared-vfs/src/traits.rs @@ -20,8 +20,6 @@ pub struct ExecId(u64); impl ExecId { /// Vends the next process-unique identity. - // The handle arrives in a later step; nothing vends identities today. - #[allow(dead_code)] pub(crate) fn vend() -> Self { static NEXT: AtomicU64 = AtomicU64::new(1); Self(NEXT.fetch_add(1, Ordering::Relaxed)) @@ -38,10 +36,18 @@ pub trait Vfs: Send { /// Acquires an access object bound to `id`. Every operation on the /// returned object is attributed to that identity: backends that /// care can know who is touching what; the rest ignore it. + /// + /// # Errors + /// + /// Returns an error when the backend cannot open a session. fn acquire(&mut self, id: ExecId) -> Result, VfsError>; /// Releases `id`. Also called from the access object's Drop, so /// teardown paths (cancel, panic, early return) cannot skip it. + /// + /// # Errors + /// + /// Returns an error when the backend cannot release the identity. fn release(&mut self, id: ExecId) -> Result<(), VfsError>; /// Whether this backend rejects all mutations. @@ -56,6 +62,11 @@ pub trait Vfs: Send { /// re-validate. pub trait VfsAccess: Send { /// Reads the file at `path` exactly as stored. + /// + /// # Errors + /// + /// Returns [`VfsError::NotFound`] when the file is absent, or a + /// backend error when the read fails. fn read(&self, path: &VfsPath) -> Result, VfsError>; /// Reads `len` bytes starting at byte `offset`. @@ -63,6 +74,11 @@ pub trait VfsAccess: Send { /// Default: read whole, slice. Backends that can seek (host /// directory, SQLite) override and never materialize the file. /// The handle's line-based ranges are built on this. + /// + /// # Errors + /// + /// Returns an error when the underlying read fails or the range + /// exceeds the addressable size. fn read_range(&self, path: &VfsPath, offset: u64, len: u64) -> Result, VfsError> { let data = self.read(path)?; let Ok(start) = usize::try_from(offset) else { @@ -88,45 +104,94 @@ pub trait VfsAccess: Send { /// `write_owned(&mut self, path: &VfsPath, contents: Vec)` /// delegating to `write`, which the memory overlay would override to /// move the buffer with zero copies. Add when profiling calls for it. + /// + /// # Errors + /// + /// Returns an error when the backend cannot write the contents. fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError>; /// Appends to the file at `path`, creating it if absent. + /// + /// # Errors + /// + /// Returns an error when the backend cannot append the contents. fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError>; /// Removes the file, link, or directory at `path`. /// Absent is NotFound; a directory without `recursive` is an error. /// On a symlink, removes the link, never the target. + /// + /// # Errors + /// + /// Returns [`VfsError::NotFound`] when the path is absent, or an + /// error when the removal fails. fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError>; /// A confirmed absence is `Ok(false)`; a backend failure is `Err`. + /// + /// # Errors + /// + /// Returns an error when the backend cannot determine existence. fn exists(&self, path: &VfsPath) -> Result; /// Returns stored paths matching `pattern`, sorted. + /// + /// # Errors + /// + /// Returns an error when the pattern is invalid or the backend fails. fn glob(&self, pattern: &str) -> Result, VfsError>; /// Lists the directory at `path`. + /// + /// # Errors + /// + /// Returns an error when the path is not a directory or the + /// backend fails. fn list(&self, path: &VfsPath) -> Result, VfsError>; /// Returns metadata for `path`. + /// + /// # Errors + /// + /// Returns [`VfsError::NotFound`] when the path is absent, or a + /// backend error when the stat fails. fn stat(&self, path: &VfsPath) -> Result; /// Creates the directory at `path`. + /// + /// # Errors + /// + /// Returns an error when the directory cannot be created. fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError>; /// Renames or moves, atomically where the backend allows. + /// + /// # Errors + /// + /// Returns an error when the rename fails; source and destination + /// are left unchanged. fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError>; /// Copies the file at `from` to `to`. + /// + /// # Errors + /// + /// Returns an error when the copy fails; source and destination + /// are left unchanged. fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError>; /// Replaces the unique occurrence of `old` with `new`. /// Zero matches and multiple matches are both errors. /// Default: read, count, replace, write. Override to push down. + /// + /// # Errors + /// + /// Returns an error when the file is not UTF-8, when the match + /// count is not exactly one, or when the read or write fails. fn str_replace(&mut self, path: &VfsPath, old: &str, new: &str) -> Result<(), VfsError> { let bytes = self.read(path)?; - let text = String::from_utf8(bytes).map_err(|_| { - VfsError::Backend(format!("str_replace requires UTF-8 text: {path}")) - })?; + let text = String::from_utf8(bytes) + .map_err(|_| VfsError::Backend(format!("str_replace requires UTF-8 text: {path}")))?; let count = text.matches(old).count(); if count == 0 { return Err(VfsError::Backend(format!( @@ -149,6 +214,11 @@ pub trait VfsAccess: Send { /// [`VfsError::Unsupported`]: this crate is std-only, so a regex /// engine must come from an overriding backend. Non-UTF-8 files and /// directories are skipped. + /// + /// # Errors + /// + /// Returns [`VfsError::Unsupported`] for regex queries, or an error + /// when the glob or a read fails. fn grep(&self, query: &GrepQuery) -> Result { if query.is_regex { return Err(VfsError::Unsupported( @@ -178,19 +248,18 @@ pub trait VfsAccess: Send { }; for (index, line) in text.lines().enumerate() { let hit = if query.case_insensitive { - line.to_lowercase() - .contains(&query.pattern.to_lowercase()) + line.to_lowercase().contains(&query.pattern.to_lowercase()) } else { line.contains(&query.pattern) }; if !hit { continue; } - if let Some(cap) = query.max_results { - if matches.len() >= cap { - truncated = true; - break 'files; - } + if let Some(cap) = query.max_results + && matches.len() >= cap + { + truncated = true; + break 'files; } matches.push(GrepMatch { path: path.clone(), @@ -205,6 +274,10 @@ pub trait VfsAccess: Send { /// Creates a symbolic link at `link` naming `target`. /// /// POSIX extra; the default returns [`VfsError::Unsupported`]. + /// + /// # Errors + /// + /// Returns [`VfsError::Unsupported`] unless a backend overrides. fn symlink(&mut self, target: &VfsPath, link: &VfsPath) -> Result<(), VfsError> { let _ = target; Err(VfsError::Unsupported(format!( @@ -215,6 +288,10 @@ pub trait VfsAccess: Send { /// Reads the target of the symbolic link at `path`. /// /// POSIX extra; the default returns [`VfsError::Unsupported`]. + /// + /// # Errors + /// + /// Returns [`VfsError::Unsupported`] unless a backend overrides. fn read_link(&self, path: &VfsPath) -> Result { Err(VfsError::Unsupported(format!( "read_link is not supported by this backend: {path}" @@ -224,6 +301,10 @@ pub trait VfsAccess: Send { /// Changes the mode bits of `path`. /// /// POSIX extra; the default returns [`VfsError::Unsupported`]. + /// + /// # Errors + /// + /// Returns [`VfsError::Unsupported`] unless a backend overrides. fn chmod(&mut self, path: &VfsPath, mode: u32) -> Result<(), VfsError> { let _ = mode; Err(VfsError::Unsupported(format!( diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index 1dcb32470..871dd4b1c 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -582,7 +582,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 3: handle, Access capability, and claims tables +### Step 3: handle, Access capability, and claims tables [completed] - Component: shared-vfs core - Implement `ExecId` (opaque, process-global monotonic counter, no public constructor), `Volume` (backend and claims as separately Arc-shareable), `Claims` (readers/writers maps from interned VfsPath to live ExecIds plus the live set; retired or released claims are deleted, never stored), `VfsRef` (`Arc`, poison-safe locking, `VfsRef::new` and `acquire()`), and `Access` (#[must_use]; canonicalizes at receipt, consults the Policy before the claims check so a denied operation never registers a claim, registers claims, locks the backend per call; `spawn()` vends a fresh ExecId and deletes the parent's claims; Drop releases the identity and its claims). diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index 686541d26..a6c0b60a0 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -86,3 +86,9 @@ - Decision: `Op` covers all sixteen access operations (the contract's `// ...` resolved to Exists/Glob/List/Stat/Symlink/ReadLink/Chmod) | Falsifier: a later step needs an op the policy cannot name. - Decision: str_replace zero/multiple-match failures use `VfsError::Backend` with a descriptive message (no dedicated kind exists) | Falsifier: the facade or model recovery path needs to match on a distinct kind. - Decision: default grep returns `Unsupported` for `is_regex` (std-only crate cannot ship a regex engine) and skips non-UTF-8 files | Falsifier: a caller requires regex semantics from the memory backend's default grep. +- Step 3: handle, Access capability, and claims tables - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p shared-vfs` - pass (48/48 tests). Review: 1 Important (copy/rename dual-path claim kinds untested), closed with 5 tests; drift review against component base clean. Verification fixes: fmt normalization in handle.rs tests; clippy `# Errors` docs on 19 trait methods and a collapsed nested if in step-2 traits.rs. + - Decision: added `VfsRef::with_policy` as the policy-installation seam (contract sketch shows only `new`) | Falsifier: step 7 installs ModePolicy through it with no further handle change. + - Decision: `Ask` maps to `PermissionDenied(reason)` in v1; the approval dialog is a plan non-goal | Falsifier: a later step needs Ask distinguishable and adds a `VfsError` kind (non_exhaustive permits). + - Decision: policy stored as `Arc` so `VfsRef`/`Access` are Send+Sync despite `Policy: Send` | Falsifier: a Send-but-not-Sync policy impl forces revisiting the bound. + - Decision: invalid line ranges return `VfsError::Backend` (no InvalidRange kind) | Falsifier: step 8's parity suite demands a dedicated kind. + - Decision: alias test uses lexical spellings (`/a/./b.txt` vs `/a//b.txt`); facade-relative spelling belongs to the step-8 Store facade since `canonicalize` rejects relative paths here | Falsifier: step 8's alias test covers facade-relative vs mount-absolute. From 23aaa83933bbe5ae44b3e8667449a0551de2a9b0 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 18:52:15 -0700 Subject: [PATCH 04/26] Add mount router, builder, and handle overlays The virtual filesystem handle can now serve several backends at once: each mounts at a path prefix, the longest matching prefix wins, and a backend's session is acquired only on first touch of its mount. Mounts install through a builder that freezes the table at build time, and routers nest because the router is itself a backend. A handle can also mount under another handle's router, so an overlay shares the base's conflict-detection table and policy while swapping only the storage view. Writes against read-only mounts are denied before anything is touched, and two-path operations that cross a mount boundary are rejected. - `Router` is crate-private and itself a `Vfs`, so routers nest and nobody outside the crate can hold one; the mount table is a `BTreeMap` fixed at construction and cheap to `Arc`-share with every session it vends. - `VfsRefBuilder` is the public way in: `mount` consumes and returns self and panics on a relative prefix or a duplicate mount, and `build` freezes the table into a handle under the `AllowAll` policy. - `VfsRef::overlay` mounts a backend over the base handle's namespace while sharing the base's claims table and policy, so conflicts are detected across both views of the same storage; an overlay at `/` panics because it would replace the base entirely. - `impl Vfs for VfsRef` lets a base handle mount under a child router, forwarding the caller's identity so the base's policy and claims apply to operations routed through it. - `RoutingAccess` resolves the longest-prefix mount per operation, strips the prefix so each backend sees a rooted mount-relative path, and acquires each backend's session lazily on first touch; its `Drop` releases the identity at every touched mount. - `check_writable` denies mutations on read-only mounts before any state is touched, so a denied operation never partially applies. - `one_mount` rejects cross-mount rename and copy with `Unsupported`, since backend atomicity guarantees stop at the mount boundary. - `glob` and `grep` rejoin the mount prefix on returned paths, so callers see full virtual paths while backends see mount-relative ones. - `HandleAccess` keeps the trait defaults for byte-range reads and the POSIX extras; the public capability exposes neither, so there is nothing to forward to. - `Router::release` and the mounted handle's `release` are no-ops; teardown flows through the routing session's `Drop`. Design: new registry @ crates/shared-vfs/src/router.rs::Router Design: new facade @ crates/shared-vfs/src/router.rs::VfsRefBuilder boundary: pub Design: new surface-growth @ crates/shared-vfs/src/handle.rs::VfsRef boundary: pub Design: extends value-object @ crates/shared-vfs/src/path.rs::VfsPathBuf Design: new pure-function @ crates/shared-vfs/src/router.rs::resolve deps: Mounts,str Design: new pure-function @ crates/shared-vfs/src/router.rs::strip_mount deps: str Design: new pure-function @ crates/shared-vfs/src/router.rs::rejoin deps: str Design: new pure-function @ crates/shared-vfs/src/router.rs::mount_matches deps: str Plan: vibe/2026-09-11-3-vfs-foundation.md --- crates/shared-vfs/src/handle.rs | 152 +++++- crates/shared-vfs/src/lib.rs | 2 + crates/shared-vfs/src/path.rs | 5 +- crates/shared-vfs/src/router.rs | 780 ++++++++++++++++++++++++++++ vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 5 + 6 files changed, 942 insertions(+), 4 deletions(-) create mode 100644 crates/shared-vfs/src/router.rs diff --git a/crates/shared-vfs/src/handle.rs b/crates/shared-vfs/src/handle.rs index dc39d4296..b9332f741 100644 --- a/crates/shared-vfs/src/handle.rs +++ b/crates/shared-vfs/src/handle.rs @@ -15,6 +15,7 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use crate::error::VfsError; use crate::path::{VfsPath, canonicalize}; +use crate::router::{Mounts, Router, VfsRefBuilder}; use crate::traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; use crate::types::{Entry, GrepQuery, GrepResults, Stat}; @@ -194,6 +195,48 @@ impl VfsRef { } } + /// Returns a builder for installing mounts. Mounts are fixed at + /// [`VfsRefBuilder::build`], so the table is immutable and cheap to + /// `Arc`-share thereafter. + #[must_use] + pub fn builder() -> VfsRefBuilder { + VfsRefBuilder::new() + } + + /// Returns a handle with `backend` mounted at `prefix` over this + /// handle's namespace. The claims table is shared: conflicts are + /// detected across both views of the same storage. + /// + /// # Panics + /// Panics when `prefix` is not an absolute virtual path or is the + /// root: an overlay at `/` would replace the base entirely, so use + /// [`VfsRef::new`] instead. + #[must_use] + pub fn overlay(&self, prefix: &str, backend: impl Vfs + 'static) -> VfsRef { + let canonical = canonicalize(prefix) + .unwrap_or_else(|err| panic!("invalid overlay prefix {prefix:?}: {err}")); + assert!( + canonical.as_str() != "/", + "an overlay at / would replace the base entirely; use VfsRef::new instead" + ); + // The base handle mounts at the root of the overlay's router: + // operations outside the overlay prefix route through the base's + // own policy and claims under the caller's identity. + let mut mounts = Mounts::new(); + let root = canonicalize("/") + .unwrap_or_else(|err| panic!("the namespace root is always valid: {err}")) + .to_buf(); + mounts.insert(root, Arc::new(Mutex::new(Box::new(self.clone())))); + mounts.insert(canonical.to_buf(), Arc::new(Mutex::new(Box::new(backend)))); + VfsRef { + volume: Arc::new(Volume { + backend: Arc::new(Mutex::new(Box::new(Router::new(mounts)))), + claims: Arc::clone(&self.volume.claims), + }), + policy: Arc::clone(&self.policy), + } + } + /// Acquires the capability for a new serial thread of execution. /// This is the only way in: every acquire vends a fresh [`ExecId`]. /// @@ -202,7 +245,18 @@ impl VfsRef { /// are expected to accept attribution; a refusal is a backend bug, /// not a runtime condition. pub fn acquire(&self) -> Access { - let id = ExecId::vend(); + self.acquire_with(ExecId::vend()) + } + + /// Acquires the capability under a given identity: how a mounted + /// handle forwards the caller's attribution. The identity registers + /// as live in this handle's claims table, so conflicts are detected + /// across both views of the same storage. + /// + /// # Panics + /// Panics when the backend fails to acquire the identity; see + /// [`VfsRef::acquire`]. + pub(crate) fn acquire_with(&self, id: ExecId) -> Access { let inner = self .backend() .acquire(id) @@ -216,6 +270,18 @@ impl VfsRef { } } + /// Builds a handle over a router with a fresh claims table and the + /// [`AllowAll`] policy: the builder's exit. + pub(crate) fn from_router(router: Router) -> VfsRef { + VfsRef { + volume: Arc::new(Volume { + backend: Arc::new(Mutex::new(Box::new(router))), + claims: Arc::new(Claims::new()), + }), + policy: Arc::new(AllowAll), + } + } + /// Poison-safe lock on the backend. fn backend(&self) -> MutexGuard<'_, Box> { self.volume @@ -546,6 +612,90 @@ impl Drop for Access { } } +/// A handle is itself a backend: mounting a base handle under a child +/// router - which is how [`VfsRef::overlay`] shares one claims table +/// across two views of the same storage - routes operations through the +/// base's policy and claims under the caller's identity. +impl Vfs for VfsRef { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + Ok(Box::new(HandleAccess(self.acquire_with(id)))) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + // The vended session's Drop releases the identity and its + // claims; nothing is registered at this level. + let _ = id; + Ok(()) + } + + fn read_only(&self) -> bool { + self.backend().read_only() + } +} + +/// The session vended by a mounted handle: forwards every operation +/// through the base handle's capability, so its policy and claims apply +/// under the caller's identity. Paths arrive canonical, so the +/// capability's canonicalization at receipt is an idempotent re-check. +/// +/// Byte-range reads and the POSIX extras keep their trait defaults: the +/// public capability exposes neither, so there is nothing to forward to. +struct HandleAccess(Access); + +impl VfsAccess for HandleAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + self.0.read(path.as_str()) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.0.write(path.as_str(), contents) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.0.append(path.as_str(), contents) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.0.remove(path.as_str(), recursive) + } + + fn exists(&self, path: &VfsPath) -> Result { + self.0.exists(path.as_str()) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + self.0.glob(pattern) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + self.0.list(path.as_str()) + } + + fn stat(&self, path: &VfsPath) -> Result { + self.0.stat(path.as_str()) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.0.mkdir(path.as_str(), recursive) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.0.rename(from.as_str(), to.as_str()) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.0.copy(from.as_str(), to.as_str()) + } + + fn str_replace(&mut self, path: &VfsPath, old: &str, new: &str) -> Result<(), VfsError> { + self.0.str_replace(path.as_str(), old, new) + } + + fn grep(&self, query: &GrepQuery) -> Result { + self.0.grep(query) + } +} + #[cfg(test)] mod tests { use std::collections::BTreeMap; diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs index c52a483df..a1abc1c42 100644 --- a/crates/shared-vfs/src/lib.rs +++ b/crates/shared-vfs/src/lib.rs @@ -8,12 +8,14 @@ mod error; mod handle; mod path; +mod router; mod traits; mod types; pub use error::VfsError; pub use handle::{Access, VfsRef}; pub use path::{VfsPath, VfsPathBuf}; +pub use router::VfsRefBuilder; pub use traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; pub use types::{Entry, FileType, GrepMatch, GrepQuery, GrepResults, Stat}; diff --git a/crates/shared-vfs/src/path.rs b/crates/shared-vfs/src/path.rs index b3d58d3df..76085d30b 100644 --- a/crates/shared-vfs/src/path.rs +++ b/crates/shared-vfs/src/path.rs @@ -94,8 +94,9 @@ impl fmt::Display for VfsPath { } /// Owned canonical virtual path, for places that outlive an interned -/// reference or arrive owned (grep roots, symlink targets). -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +/// reference or arrive owned (grep roots, symlink targets). Ordered for +/// the mount table's `BTreeMap`. +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct VfsPathBuf(String); impl VfsPathBuf { diff --git a/crates/shared-vfs/src/router.rs b/crates/shared-vfs/src/router.rs new file mode 100644 index 000000000..00e9a1a43 --- /dev/null +++ b/crates/shared-vfs/src/router.rs @@ -0,0 +1,780 @@ +//! The mount table and its routing access. +//! +//! A `Router` is itself a [`Vfs`], so routers nest; the public concept +//! is "a `VfsRef` with these mounts," expressed through +//! [`VfsRefBuilder`]. Mounts are fixed at build, so the table is +//! immutable and cheap to `Arc`-share. The routing access resolves the +//! longest-prefix mount per operation, strips the mount prefix so each +//! backend sees a rooted path within its own mount, and acquires each +//! backend's access lazily on first touch. + +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::fmt; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::error::VfsError; +use crate::handle::VfsRef; +use crate::path::{VfsPath, VfsPathBuf, canonicalize}; +use crate::traits::{ExecId, Vfs, VfsAccess}; +use crate::types::{Entry, GrepQuery, GrepResults, Stat}; + +/// One mounted backend behind a shared lock. +type Mounted = Arc>>; + +/// The mount table: backends at canonical prefixes, longest prefix +/// wins. Immutable after construction, so it is cheap to `Arc`-share +/// with every routing access the router vends. +pub(crate) type Mounts = BTreeMap; + +/// Poison-safe lock on a mounted backend. +fn lock(backend: &Mounted) -> MutexGuard<'_, Box> { + backend.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Whether the mount at `prefix` serves `path`: the root mount serves +/// everything; any other mount serves itself and its descendants. +fn mount_matches(prefix: &str, path: &str) -> bool { + if prefix == "/" { + return true; + } + path == prefix + || path + .strip_prefix(prefix) + .is_some_and(|rest| rest.starts_with('/')) +} + +/// Resolves the longest-prefix mount serving `path`. +fn resolve<'m>(mounts: &'m Mounts, path: &str) -> Option<(&'m VfsPathBuf, &'m Mounted)> { + mounts + .iter() + .filter(|(prefix, _)| mount_matches(prefix.as_str(), path)) + .max_by_key(|(prefix, _)| prefix.as_str().len()) +} + +/// The path a mounted backend sees: the mount prefix stripped, rooted +/// at the mount. The mount itself is its own root. +fn strip_mount<'a>(prefix: &str, path: &'a str) -> &'a str { + if prefix == "/" { + return path; + } + match path.strip_prefix(prefix) { + Some("") => "/", + Some(rest) => rest, + None => path, // unreachable: resolve() matched the prefix + } +} + +/// Restores the mount prefix on a path a backend returned, so callers +/// see full virtual paths. +fn rejoin(prefix: &str, path: &str) -> String { + if prefix == "/" { + return path.to_owned(); + } + if path == "/" { + return prefix.to_owned(); + } + format!("{prefix}{path}") +} + +/// The mount table. Backends install at prefixes; longest prefix wins. +/// A Router is itself a [`Vfs`], so routers nest. Crate-private: the +/// public concept is "a `VfsRef` with these mounts," expressed through +/// [`VfsRefBuilder`]; privacy enforces mounts-fixed-at-construction, +/// since nobody outside the crate can hold one. +pub(crate) struct Router { + mounts: Arc, +} + +impl Router { + pub(crate) fn new(mounts: Mounts) -> Router { + Router { + mounts: Arc::new(mounts), + } + } +} + +impl Vfs for Router { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + Ok(Box::new(RoutingAccess { + id, + mounts: Arc::clone(&self.mounts), + acquired: RefCell::new(BTreeMap::new()), + })) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + // The routing access's Drop releases the identity at every + // touched mount; the router itself holds no per-identity state. + let _ = id; + Ok(()) + } + + // read_only() keeps the default: a router mixes mounts, and the + // routing access enforces each mount's own flag per operation. +} + +/// One identity's session with the router. Resolves the longest-prefix +/// mount per operation and acquires each backend's access lazily on +/// first touch of that mount. +struct RoutingAccess { + id: ExecId, + mounts: Arc, + /// Per-mount sessions, keyed by mount prefix. `RefCell` because + /// read-only trait methods take `&self`; the enclosing capability + /// serializes every call, so the cell is never contended. + acquired: RefCell>>, +} + +impl RoutingAccess { + /// Runs `op` against the session of the mount serving `path`, + /// acquiring that session on first touch. + fn with_mount( + &self, + path: VfsPath, + op: impl FnOnce(&mut dyn VfsAccess) -> Result, + ) -> Result { + let (prefix, backend) = resolve(&self.mounts, path.as_str()) + .ok_or_else(|| VfsError::NotFound(format!("no mount serves {path}")))?; + let mut acquired = self.acquired.borrow_mut(); + if !acquired.contains_key(prefix) { + let session = lock(backend).acquire(self.id)?; + acquired.insert(prefix.clone(), session); + } + let Some(session) = acquired.get_mut(prefix) else { + unreachable!("the session was just acquired") + }; + op(session.as_mut()) + } + + /// The mount-relative path the serving backend sees. + fn strip(&self, path: VfsPath) -> Result { + let (prefix, _) = resolve(&self.mounts, path.as_str()) + .ok_or_else(|| VfsError::NotFound(format!("no mount serves {path}")))?; + canonicalize(strip_mount(prefix.as_str(), path.as_str())) + } + + /// Rejects mutations on read-only mounts before anything is + /// touched: a denied operation never partially applies. + fn check_writable(&self, path: VfsPath) -> Result<(), VfsError> { + let (prefix, backend) = resolve(&self.mounts, path.as_str()) + .ok_or_else(|| VfsError::NotFound(format!("no mount serves {path}")))?; + if lock(backend).read_only() { + return Err(VfsError::PermissionDenied(format!( + "the mount at {prefix} is read-only, so {path} cannot be mutated" + ))); + } + Ok(()) + } + + /// Two-path operations require one mount: backend atomicity + /// guarantees stop at the mount boundary. + fn one_mount(&self, from: VfsPath, to: VfsPath, op: &str) -> Result<(), VfsError> { + let from_prefix = resolve(&self.mounts, from.as_str()) + .ok_or_else(|| VfsError::NotFound(format!("no mount serves {from}")))? + .0; + let to_prefix = resolve(&self.mounts, to.as_str()) + .ok_or_else(|| VfsError::NotFound(format!("no mount serves {to}")))? + .0; + if from_prefix != to_prefix { + return Err(VfsError::Unsupported(format!( + "{op} across mounts is unsupported: {from} and {to} are served by different mounts" + ))); + } + Ok(()) + } +} + +impl VfsAccess for RoutingAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.read(&stripped)) + } + + fn read_range(&self, path: &VfsPath, offset: u64, len: u64) -> Result, VfsError> { + // Delegated, not defaulted, so backends that can seek never + // materialize the file. + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.read_range(&stripped, offset, len)) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.check_writable(*path)?; + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.write(&stripped, contents)) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.check_writable(*path)?; + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.append(&stripped, contents)) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.check_writable(*path)?; + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.remove(&stripped, recursive)) + } + + fn exists(&self, path: &VfsPath) -> Result { + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.exists(&stripped)) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + // Patterns are not paths, but they route like one: + // canonicalizing resolves dot segments and rejects escapes past + // the namespace root; wildcards are ordinary segments. + let canonical = canonicalize(pattern)?; + let (prefix, _) = resolve(&self.mounts, canonical.as_str()) + .ok_or_else(|| VfsError::NotFound(format!("no mount serves {canonical}")))?; + let scoped = strip_mount(prefix.as_str(), canonical.as_str()).to_owned(); + let mut matches = self.with_mount(canonical, |session| session.glob(&scoped))?; + for path in &mut matches { + *path = rejoin(prefix.as_str(), path); + } + Ok(matches) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.list(&stripped)) + } + + fn stat(&self, path: &VfsPath) -> Result { + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.stat(&stripped)) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.check_writable(*path)?; + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.mkdir(&stripped, recursive)) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.check_writable(*from)?; + self.check_writable(*to)?; + self.one_mount(*from, *to, "rename")?; + let from_stripped = self.strip(*from)?; + let to_stripped = self.strip(*to)?; + self.with_mount(*from, |session| { + session.rename(&from_stripped, &to_stripped) + }) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.check_writable(*to)?; + self.one_mount(*from, *to, "copy")?; + let from_stripped = self.strip(*from)?; + let to_stripped = self.strip(*to)?; + self.with_mount(*from, |session| session.copy(&from_stripped, &to_stripped)) + } + + fn str_replace(&mut self, path: &VfsPath, old: &str, new: &str) -> Result<(), VfsError> { + // Delegated so backends can push down; the read-only check here + // covers the backend's default read-plus-write as well. + self.check_writable(*path)?; + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.str_replace(&stripped, old, new)) + } + + fn grep(&self, query: &GrepQuery) -> Result { + let root = canonicalize(query.root.as_str())?; + let (prefix, _) = resolve(&self.mounts, root.as_str()) + .ok_or_else(|| VfsError::NotFound(format!("no mount serves {root}")))?; + let mut scoped = query.clone(); + scoped.root = canonicalize(strip_mount(prefix.as_str(), root.as_str()))?.to_buf(); + let mut results = self.with_mount(root, |session| session.grep(&scoped))?; + for hit in &mut results.matches { + hit.path = rejoin(prefix.as_str(), &hit.path); + } + Ok(results) + } + + fn symlink(&mut self, target: &VfsPath, link: &VfsPath) -> Result<(), VfsError> { + self.check_writable(*link)?; + let stripped = self.strip(*link)?; + // The target is a stored name, not resolved: it passes verbatim. + self.with_mount(*link, |session| session.symlink(target, &stripped)) + } + + fn read_link(&self, path: &VfsPath) -> Result { + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.read_link(&stripped)) + } + + fn chmod(&mut self, path: &VfsPath, mode: u32) -> Result<(), VfsError> { + self.check_writable(*path)?; + let stripped = self.strip(*path)?; + self.with_mount(*path, |session| session.chmod(&stripped, mode)) + } +} + +impl Drop for RoutingAccess { + fn drop(&mut self) { + // Release the identity at every touched mount. Sessions drop + // first: a mounted handle's session releases through its own + // capability's Drop, and the backend-level release below is its + // no-op counterpart. Plain backends get their one release here. + for (prefix, session) in std::mem::take(self.acquired.get_mut()) { + drop(session); + if let Some(backend) = self.mounts.get(&prefix) { + let _ = lock(backend).release(self.id); + } + } + } +} + +/// Mount installation for [`VfsRef`]. Mounts are fixed at +/// [`VfsRefBuilder::build`], so the table is immutable and cheap to +/// `Arc`-share thereafter. +pub struct VfsRefBuilder { + mounts: Mounts, +} + +impl fmt::Debug for VfsRefBuilder { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("VfsRefBuilder").finish_non_exhaustive() + } +} + +impl VfsRefBuilder { + pub(crate) fn new() -> VfsRefBuilder { + VfsRefBuilder { + mounts: BTreeMap::new(), + } + } + /// Mounts `backend` at `prefix`, consuming and returning the + /// builder. The root prefix `/` serves the whole namespace; a + /// longer prefix shadows a shorter one (longest prefix wins). + /// + /// # Panics + /// Panics when `prefix` is not an absolute virtual path or a mount + /// already sits at `prefix`: both are construction-time bugs. + #[must_use] + pub fn mount(mut self, prefix: &str, backend: impl Vfs + 'static) -> VfsRefBuilder { + let canonical = canonicalize(prefix) + .unwrap_or_else(|err| panic!("invalid mount prefix {prefix:?}: {err}")); + let key = canonical.to_buf(); + assert!( + !self.mounts.contains_key(&key), + "a mount already sits at {prefix:?}" + ); + self.mounts + .insert(key, Arc::new(Mutex::new(Box::new(backend)))); + self + } + + /// Freezes the mount table into a handle with the [`AllowAll`] + /// policy. + /// + /// [`AllowAll`]: crate::AllowAll + #[must_use] + pub fn build(self) -> VfsRef { + VfsRef::from_router(Router::new(self.mounts)) + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + + use crate::error::VfsError; + use crate::handle::VfsRef; + use crate::path::VfsPath; + use crate::traits::{ExecId, Vfs, VfsAccess}; + use crate::types::{Entry, Stat}; + + /// A recording in-memory stub. Files are keyed by the exact paths + /// the backend is handed, so tests observe prefix stripping + /// directly; every served path is recorded; the acquire count pins + /// lazy per-mount acquisition. + #[derive(Clone, Default)] + struct StubFs { + files: Arc>>>, + seen: Arc>>, + acquires: Arc>, + read_only: bool, + } + + impl StubFs { + fn seeded(files: &[(&str, &str)]) -> StubFs { + let stub = StubFs::default(); + for (name, text) in files { + stub.files() + .insert((*name).to_owned(), text.as_bytes().to_vec()); + } + stub + } + + fn with_read_only(mut self) -> StubFs { + self.read_only = true; + self + } + + fn files(&self) -> MutexGuard<'_, BTreeMap>> { + self.files.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn seen(&self) -> Vec { + self.seen + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + + fn acquire_count(&self) -> usize { + *self.acquires.lock().unwrap_or_else(PoisonError::into_inner) + } + } + + impl Vfs for StubFs { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + let _ = id; + *self.acquires.lock().unwrap_or_else(PoisonError::into_inner) += 1; + Ok(Box::new(StubAccess { + files: Arc::clone(&self.files), + seen: Arc::clone(&self.seen), + })) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + let _ = id; + Ok(()) + } + + fn read_only(&self) -> bool { + self.read_only + } + } + + struct StubAccess { + files: Arc>>>, + seen: Arc>>, + } + + impl StubAccess { + fn files(&self) -> MutexGuard<'_, BTreeMap>> { + self.files.lock().unwrap_or_else(PoisonError::into_inner) + } + + fn record(&self, path: VfsPath) { + self.seen + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(path.to_string()); + } + } + + impl VfsAccess for StubAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + self.record(*path); + self.files() + .get(path.as_str()) + .cloned() + .ok_or_else(|| VfsError::NotFound(path.to_string())) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.record(*path); + self.files().insert(path.to_string(), contents.to_vec()); + Ok(()) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.record(*path); + self.files() + .entry(path.to_string()) + .or_default() + .extend_from_slice(contents); + Ok(()) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let _ = recursive; + self.record(*path); + self.files() + .remove(path.as_str()) + .map(|_| ()) + .ok_or_else(|| VfsError::NotFound(path.to_string())) + } + + fn exists(&self, path: &VfsPath) -> Result { + self.record(*path); + Ok(self.files().contains_key(path.as_str())) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + let prefix = pattern.split('*').next().unwrap_or(pattern); + let mut matches: Vec = self + .files() + .keys() + .filter(|name| name.starts_with(prefix)) + .cloned() + .collect(); + matches.sort(); + Ok(matches) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + let _ = path; + Err(VfsError::Unsupported("the stub does not list".into())) + } + + fn stat(&self, path: &VfsPath) -> Result { + let _ = path; + Err(VfsError::Unsupported("the stub does not stat".into())) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let _ = (path, recursive); + Ok(()) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.record(*from); + self.record(*to); + let bytes = self + .files() + .remove(from.as_str()) + .ok_or_else(|| VfsError::NotFound(from.to_string()))?; + self.files().insert(to.to_string(), bytes); + Ok(()) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.record(*from); + self.record(*to); + let bytes = self + .files() + .get(from.as_str()) + .cloned() + .ok_or_else(|| VfsError::NotFound(from.to_string()))?; + self.files().insert(to.to_string(), bytes); + Ok(()) + } + } + + #[test] + fn the_longest_prefix_mount_wins_and_acquires_lazily() -> Result<(), VfsError> { + let outer = StubFs::default(); + let inner = StubFs::default(); + let untouched = StubFs::default(); + let vfs = VfsRef::builder() + .mount("/a", outer.clone()) + .mount("/a/b", inner.clone()) + .mount("/elsewhere", untouched.clone()) + .build(); + let access = vfs.acquire(); + access.write("/a/b/f.txt", b"inner")?; + access.write("/a/f.txt", b"outer")?; + // Each backend keyed the file by its mount-relative path. + assert!(inner.files().contains_key("/f.txt")); + assert!(outer.files().contains_key("/f.txt")); + assert_eq!(access.read("/a/b/f.txt")?, b"inner"); + assert_eq!(access.read("/a/f.txt")?, b"outer"); + // Lazy per-mount acquire: the untouched mount never opened one. + assert_eq!(inner.acquire_count(), 1); + assert_eq!(outer.acquire_count(), 1); + assert_eq!(untouched.acquire_count(), 0); + Ok(()) + } + + #[test] + fn a_longer_mount_shadows_the_same_prefix_of_a_shorter_one() -> Result<(), VfsError> { + let base = StubFs::seeded(&[("/f.txt", "base-f"), ("/mnt/f.txt", "base-mnt")]); + let shadow = StubFs::seeded(&[("/f.txt", "shadow-mnt")]); + let vfs = VfsRef::builder() + .mount("/", base.clone()) + .mount("/mnt", shadow.clone()) + .build(); + let access = vfs.acquire(); + // The shadow mount owns everything under /mnt. + assert_eq!(access.read("/mnt/f.txt")?, b"shadow-mnt"); + // The base still owns the rest of the namespace. + assert_eq!(access.read("/f.txt")?, b"base-f"); + // The base's own /mnt/f.txt is unreachable through the handle. + assert_eq!( + base.files().get("/mnt/f.txt").map(Vec::as_slice), + Some(b"base-mnt".as_slice()) + ); + Ok(()) + } + + #[test] + fn a_mounted_handle_applies_its_own_claims_under_the_callers_identity() -> Result<(), VfsError> + { + // Nesting: a base handle mounted under a child router. + let base_storage = StubFs::default(); + let base = VfsRef::new(base_storage.clone()); + let local = StubFs::default(); + let child = VfsRef::builder() + .mount("/base", base.clone()) + .mount("/local", local.clone()) + .build(); + let writer = child.acquire(); + writer.write("/base/f.txt", b"nested")?; + // The child router stripped its mount prefix: the base backend + // keyed the file at its own root. + assert!(base_storage.files().contains_key("/f.txt")); + // A second child identity conflicts on the same path: the claim + // registered through the mounted handle is visible. + let reader = child.acquire(); + match reader.read("/base/f.txt") { + Err(VfsError::Conflict(_)) => {} + other => panic!("expected a conflict, got {other:?}"), + } + // The local mount routes to its own backend. + writer.write("/local/g.txt", b"local")?; + assert!(local.files().contains_key("/g.txt")); + Ok(()) + } + + #[test] + fn writes_to_a_read_only_mount_are_denied_without_partial_application() -> Result<(), VfsError> + { + let ro = StubFs::seeded(&[("/a.txt", "keep")]).with_read_only(); + let rw = StubFs::default(); + let vfs = VfsRef::builder() + .mount("/", rw.clone()) + .mount("/ro", ro.clone()) + .build(); + let access = vfs.acquire(); + // Reads are not gated. + assert_eq!(access.read("/ro/a.txt")?, b"keep"); + // A write is denied with a clear read-only error. + match access.write("/ro/new.txt", b"x") { + Err(VfsError::PermissionDenied(message)) => { + assert!(message.contains("read-only"), "names the cause: {message}"); + } + other => panic!("expected a read-only denial, got {other:?}"), + } + assert!(!ro.files().contains_key("/new.txt")); + // A rename wholly inside the mount is denied before the source + // is touched. + match access.rename("/ro/a.txt", "/ro/b.txt") { + Err(VfsError::PermissionDenied(_)) => {} + other => panic!("expected a read-only denial, got {other:?}"), + } + assert_eq!(access.read("/ro/a.txt")?, b"keep"); + assert!(!ro.files().contains_key("/b.txt")); + // A copy whose destination is read-only is denied; the source + // is untouched. + access.write("/x.txt", b"data")?; + match access.copy("/x.txt", "/ro/x.txt") { + Err(VfsError::PermissionDenied(_)) => {} + other => panic!("expected a read-only denial, got {other:?}"), + } + assert!(!ro.files().contains_key("/x.txt")); + assert_eq!(access.read("/x.txt")?, b"data"); + Ok(()) + } + + #[test] + fn traversal_that_escapes_the_namespace_root_is_rejected() { + let vfs = VfsRef::builder().mount("/mnt", StubFs::default()).build(); + let access = vfs.acquire(); + assert!(matches!( + access.read("/mnt/../../etc/passwd"), + Err(VfsError::InvalidPath(_)) + )); + assert!(matches!( + access.glob("/mnt/../../*"), + Err(VfsError::InvalidPath(_)) + )); + } + + #[test] + fn a_path_that_climbs_out_of_its_mount_is_not_served_by_that_mount() -> Result<(), VfsError> { + // No root mount: the only storage lives at /mnt. + let storage = StubFs::seeded(&[("/f.txt", "inside")]); + let vfs = VfsRef::builder().mount("/mnt", storage.clone()).build(); + let access = vfs.acquire(); + // Dot segments within the mount resolve within the mount: the + // backend sees the clean mount-relative path. + assert_eq!(access.read("/mnt/sub/../f.txt")?, b"inside"); + assert!(storage.seen().contains(&"/f.txt".to_owned())); + // A path that climbs out of the mount re-roots absolutely: it + // canonicalizes to /secret.txt, no mount serves it, and the + // mount's backend never sees the traversal spelling. + match access.read("/mnt/../secret.txt") { + Err(VfsError::NotFound(_)) => {} + other => panic!("expected no serving mount, got {other:?}"), + } + assert!( + storage.seen().iter().all(|path| !path.contains("..")), + "the backend never sees a traversal: {:?}", + storage.seen() + ); + Ok(()) + } + + #[test] + fn glob_routes_to_the_serving_mount_and_restores_the_prefix() -> Result<(), VfsError> { + let inner = StubFs::seeded(&[("/x.txt", "x")]); + let vfs = VfsRef::builder() + .mount("/a", StubFs::default()) + .mount("/a/b", inner.clone()) + .build(); + let access = vfs.acquire(); + let matches = access.glob("/a/b/*.txt")?; + assert_eq!(matches, vec!["/a/b/x.txt".to_owned()]); + Ok(()) + } + + #[test] + fn one_handle_serves_several_mounts_and_an_overlay_simultaneously() -> Result<(), VfsError> { + let store = StubFs::default(); + let scratch = StubFs::default(); + let extra = StubFs::default(); + let base = VfsRef::builder() + .mount("/store", store.clone()) + .mount("/scratch", scratch.clone()) + .build(); + let overlay = base.overlay("/overlay", extra.clone()); + + let writer = overlay.acquire(); + writer.write("/store/doc.md", b"store")?; + writer.write("/scratch/tmp.txt", b"scratch")?; + writer.write("/overlay/x.txt", b"overlay")?; + // Dropping releases the writer's claims into the shared table. + drop(writer); + + // The base handle serves its own mounts from the same storage. + let reader = base.acquire(); + assert_eq!(reader.read("/store/doc.md")?, b"store"); + assert_eq!(reader.read("/scratch/tmp.txt")?, b"scratch"); + // The overlay mount exists only in the overlay's view. + assert!(matches!( + reader.read("/overlay/x.txt"), + Err(VfsError::NotFound(_)) + )); + Ok(()) + } + + #[test] + fn an_overlay_shares_the_bases_claims_table() -> Result<(), VfsError> { + let base = VfsRef::builder().mount("/store", StubFs::default()).build(); + let overlay = base.overlay("/overlay", StubFs::default()); + let first = base.acquire(); + first.write("/store/shared.txt", b"1")?; + // A write claim registered through the base conflicts with a + // write attempted through the overlay: one claims table. + let second = overlay.acquire(); + match second.write("/store/shared.txt", b"2") { + Err(VfsError::Conflict(message)) => { + assert!(message.contains("/store/shared.txt"), "{message}"); + } + other => panic!("expected a conflict, got {other:?}"), + } + Ok(()) + } + + #[test] + #[should_panic(expected = "invalid mount prefix")] + fn the_builder_rejects_a_relative_mount_prefix() { + let _ = VfsRef::builder().mount("relative", StubFs::default()); + } +} diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index 871dd4b1c..735cc6a75 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -593,7 +593,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 4: router, builder, and overlays +### Step 4: router, builder, and overlays [completed] - Component: shared-vfs router - Implement crate-private `Router` (BTreeMap mount table, longest-prefix dispatch, lazy per-mount acquire on first touch), `impl Vfs for Router` (routers nest), `impl Vfs for VfsRef` (forwards acquire with the given ExecId, so a base handle mounts under a child router), `VfsRefBuilder` (mount consumes and returns self; build() freezes the table), and `VfsRef::overlay()` (shares the claims table, swaps only the backend view). diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index a6c0b60a0..e4793878c 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -92,3 +92,8 @@ - Decision: policy stored as `Arc` so `VfsRef`/`Access` are Send+Sync despite `Policy: Send` | Falsifier: a Send-but-not-Sync policy impl forces revisiting the bound. - Decision: invalid line ranges return `VfsError::Backend` (no InvalidRange kind) | Falsifier: step 8's parity suite demands a dedicated kind. - Decision: alias test uses lexical spellings (`/a/./b.txt` vs `/a//b.txt`); facade-relative spelling belongs to the step-8 Store facade since `canonicalize` rejects relative paths here | Falsifier: step 8's alias test covers facade-relative vs mount-absolute. +- Step 4: router, builder, and overlays - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p shared-vfs` - pass (58/58 tests). Review: clean (with component-base drift check). Verification fixes: fmt normalization; clippy `#[must_use]` on builder/overlay/build, `VfsPath` by value in private helpers, let-else rewrite. + - Decision: backends see mount-relative rooted paths (router strips on dispatch, rejoins on glob/grep results) | Falsifier: a contract passage requiring backends to see full virtual paths. + - Decision: cross-mount rename/copy return `Unsupported` instead of read-plus-write | Falsifier: a caller needing atomic cross-mount moves. + - Decision: `Router::release` and mounted-handle `release` are no-ops; teardown flows through the routing session's `Drop` | Falsifier: a backend requiring explicit release independent of `Drop`. + - Decision: `overlay()` shares the base's policy `Arc` as well as its claims table | Falsifier: a requirement that overlays carry independent policy. From 69faa2f8e1b56fa5e4d8b00b3f5c179bfd017f35 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 19:06:46 -0700 Subject: [PATCH 05/26] Add in-memory VFS backend with bounded glob matching Adds the generic in-memory backend for the virtual filesystem, carrying the former in-memory store semantics onto the new trait surface: bytes keyed by canonical path, writes that materialize their ancestor directories, and strict removals where absent is an error and a non-empty directory needs the recursive flag. Identity attribution on acquire and release is accepted as a no-op because every session shares the one map and conflict enforcement lives in the claims layer above. Adds a small glob grammar with validation plus a bounded, recursion-free matcher, so a hostile pattern cannot drive exponential time or blow the stack. The backend is exported publicly and its zero value is a meaningful empty filesystem. - `MemoryBackend` stores files in an ordered map and directories in an ordered set behind a poison-safe shared lock, so listings come out sorted and clones share one storage. - `MemoryBackend::acquire` accepts the identity as a no-op and every session shares the one map, because conflict enforcement lives in the claims model above the backend. - `crates/shared-vfs/src/glob.rs` validates patterns against a grammar of literals, single-star within a segment, and double-star as a whole segment, then matches with an iterative dynamic program that is linear and recursion-free; patterns over 1024 bytes are refused outright. - `MemoryAccess::write` and `MemoryAccess::append` materialize ancestor directories, so no mkdir is needed before a write; both validate the destination before mutating. - `MemoryAccess::remove` is strict: absent is NotFound, a non-empty directory without the recursive flag is DirectoryNotEmpty, and the namespace root cannot be removed. - `MemoryAccess::rename` finishes validation before any mutation so a failed rename changes nothing; a directory moves with its whole subtree and cannot move onto the root or into its own descendant. - `MemoryAccess::glob` returns sorted results that include directories, compiling the pattern once and reusing the tokens across every key while the lock is held. - `MemoryAccess::stat` reports kind and size with mode and timestamps as None rather than fabricating values. - `MemoryBackend::release` does nothing; the backend holds no resources and teardown flows through drop. Design: new surface-growth @ crates/shared-vfs/src/memory.rs::MemoryBackend boundary: pub Design: new shared-mutable-state @ crates/shared-vfs/src/memory.rs::MemoryBackend Design: new pure-function @ crates/shared-vfs/src/glob.rs::validate_glob_grammar deps: str Design: new pure-function @ crates/shared-vfs/src/glob.rs::compile_glob deps: u8 Design: new pure-function @ crates/shared-vfs/src/glob.rs::tokenize_glob deps: u8 Design: new pure-function @ crates/shared-vfs/src/glob.rs::matches_tokens deps: GlobToken,u8 Design: new pure-function @ crates/shared-vfs/src/glob.rs::glob_match deps: u8 Design: new pure-function @ crates/shared-vfs/src/memory.rs::ancestors deps: str Plan: vibe/2026-09-11-3-vfs-foundation.md --- crates/shared-vfs/src/glob.rs | 222 ++++++++ crates/shared-vfs/src/lib.rs | 3 + crates/shared-vfs/src/memory.rs | 841 ++++++++++++++++++++++++++++ vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 4 + 5 files changed, 1071 insertions(+), 1 deletion(-) create mode 100644 crates/shared-vfs/src/glob.rs create mode 100644 crates/shared-vfs/src/memory.rs diff --git a/crates/shared-vfs/src/glob.rs b/crates/shared-vfs/src/glob.rs new file mode 100644 index 000000000..0a7da2462 --- /dev/null +++ b/crates/shared-vfs/src/glob.rs @@ -0,0 +1,222 @@ +//! Glob-pattern grammar and a bounded, recursion-free matcher. +//! +//! The backend validates a caller-supplied glob against one grammar and +//! then matches stored paths with a bounded iterative dynamic program, so +//! a hostile pattern cannot drive exponential time or blow the stack. + +/// The largest glob pattern, in bytes, a backend will attempt to match. +/// +/// The recursion-free matcher is linear, but an unbounded pattern is still +/// a cheap denial-of-service lever, so an over-long pattern is refused +/// outright. +pub(crate) const MAX_GLOB_PATTERN_BYTES: usize = 1024; + +/// One unit of a validated glob pattern. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum GlobToken { + /// A literal byte that must match exactly. + Literal(u8), + /// `*`: zero or more bytes, none of them `/` (stays within one segment). + Star, + /// `**` not bounded by a `/`: zero or more bytes of any kind. + DoubleStar, + /// `**/`: zero or more whole path segments (empty, or any run ending `/`). + DoubleStarSlash, +} + +/// Validates the glob grammar, rejecting unsupported forms. +/// +/// The grammar is: literal bytes, `*` (within a segment), and `**` +/// occupying a whole segment (`**`, `**/...`, `.../**`, `.../**/...`). +/// There is no escape syntax, so a backslash is unsupported and runs of +/// three or more `*` are rejected rather than silently reinterpreted. +pub(crate) fn validate_glob_grammar(pattern: &str) -> Result<(), &'static str> { + if pattern.contains('\\') { + return Err("pattern does not support backslash escapes"); + } + let bytes = pattern.as_bytes(); + let mut index = 0; + while index < bytes.len() { + if bytes[index] != b'*' { + index += 1; + continue; + } + let run_start = index; + while index < bytes.len() && bytes[index] == b'*' { + index += 1; + } + let run_len = index - run_start; + if run_len > 2 { + return Err("more than two consecutive '*' are not supported"); + } + if run_len == 2 { + let before_ok = run_start == 0 || bytes[run_start - 1] == b'/'; + let after_ok = index == bytes.len() || bytes[index] == b'/'; + if !before_ok || !after_ok { + return Err("'**' must occupy a whole path segment"); + } + } + } + Ok(()) +} + +/// Compiles an already-grammar-validated pattern into reusable +/// [`GlobToken`]s. Callers that match one pattern against many paths +/// compile once and reuse the tokens via [`matches_tokens`], so the +/// per-path tokenization cost is not repeated for every key. +pub(crate) fn compile_glob(pattern: &[u8]) -> Vec { + debug_assert!( + std::str::from_utf8(pattern).is_ok_and(|text| validate_glob_grammar(text).is_ok()), + "compile_glob requires validated grammar" + ); + tokenize_glob(pattern) +} + +/// Tokenizes an already-grammar-validated pattern into [`GlobToken`]s. +fn tokenize_glob(pattern: &[u8]) -> Vec { + let mut tokens = Vec::with_capacity(pattern.len()); + let mut index = 0; + while index < pattern.len() { + match pattern[index] { + b'*' => { + if pattern.get(index + 1) == Some(&b'*') { + if pattern.get(index + 2) == Some(&b'/') { + tokens.push(GlobToken::DoubleStarSlash); + index += 3; + } else { + tokens.push(GlobToken::DoubleStar); + index += 2; + } + } else { + tokens.push(GlobToken::Star); + index += 1; + } + } + byte => { + tokens.push(GlobToken::Literal(byte)); + index += 1; + } + } + } + tokens +} + +/// Matches `text` against a glob `pattern` where `*` stays within a +/// segment and `**` spans `/`. Bounded iterative dynamic programming over +/// reachable text positions, with no recursion and no suffix backtracking: +/// `O(tokens * text_len)` time and `O(text_len)` space. Compiles and +/// matches in one shot; retained as the parity reference for +/// [`matches_tokens`] and only needed in tests. +#[cfg(test)] +pub(crate) fn glob_match(pattern: &[u8], text: &[u8]) -> bool { + matches_tokens(&tokenize_glob(pattern), text) +} + +/// Matches `text` against already-compiled glob `tokens` (see +/// [`compile_glob`]). Split from the one-shot matcher so one compiled +/// pattern can be reused across many paths without re-tokenizing per path. +pub(crate) fn matches_tokens(tokens: &[GlobToken], text: &[u8]) -> bool { + let len = text.len(); + // `reachable[j]` is true when some prefix of the pattern consumed so + // far matches exactly `text[..j]`. + let mut reachable = vec![false; len + 1]; + reachable[0] = true; + let mut next = vec![false; len + 1]; + + for &token in tokens { + next.fill(false); + match token { + GlobToken::Literal(byte) => { + for j in 0..len { + if reachable[j] && text[j] == byte { + next[j + 1] = true; + } + } + } + GlobToken::Star => { + // Zero or more non-`/` bytes: sweep left to right, carrying + // reachability forward across each non-slash byte. + let mut carry = false; + for j in 0..=len { + let here = reachable[j] || carry; + next[j] = here; + carry = here && j < len && text[j] != b'/'; + } + } + GlobToken::DoubleStar => { + // Zero or more bytes of any kind: once any position is + // reachable, every later position is too. + let mut seen = false; + for j in 0..=len { + seen |= reachable[j]; + next[j] = seen; + } + } + GlobToken::DoubleStarSlash => { + // Empty, or any run ending in `/` (whole path segments). + let mut seen = false; + for j in 0..=len { + let mut here = reachable[j]; + if seen && j > 0 && text[j - 1] == b'/' { + here = true; + } + next[j] = here; + if reachable[j] { + seen = true; + } + } + } + } + std::mem::swap(&mut reachable, &mut next); + } + reachable[len] +} + +#[cfg(test)] +mod tests { + use super::{compile_glob, glob_match, matches_tokens}; + + #[test] + fn compiled_and_one_shot_match_expected_results_across_many_paths() { + // A pattern compiled once and reused across many keys must produce + // the pinned result for every key. The one-shot matcher is checked + // against the same independent expectations. + let paths = [ + "a.txt", + "src/a.rs", + "src/b.rs", + "src/deep/c.rs", + "src/deep/deeper/d.rs", + "notes/today.md", + ]; + for (pattern, expected) in [ + ("*.txt", [true, false, false, false, false, false]), + ("src/*.rs", [false, true, true, false, false, false]), + ("src/**/*.rs", [false, true, true, true, true, false]), + ("**/*.md", [false, false, false, false, false, true]), + ("src/**", [false, true, true, true, true, false]), + ("no*match", [false, false, false, false, false, false]), + ] { + let tokens = compile_glob(pattern.as_bytes()); + for (path, expected) in paths.iter().zip(expected) { + assert_eq!( + matches_tokens(&tokens, path.as_bytes()), + expected, + "compiled pattern {pattern:?} produced the wrong result for {path:?}", + ); + assert_eq!( + glob_match(pattern.as_bytes(), path.as_bytes()), + expected, + "one-shot pattern {pattern:?} produced the wrong result for {path:?}", + ); + } + } + } + + #[cfg(debug_assertions)] + #[test] + #[should_panic(expected = "compile_glob requires validated grammar")] + fn compile_glob_rejects_unvalidated_direct_input_in_debug_builds() { + let _ = compile_glob(b"bad/***/pattern"); + } +} diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs index a1abc1c42..c1a8714e2 100644 --- a/crates/shared-vfs/src/lib.rs +++ b/crates/shared-vfs/src/lib.rs @@ -6,7 +6,9 @@ //! `/_promptforge` paths, no Store, no run concepts). mod error; +mod glob; mod handle; +mod memory; mod path; mod router; mod traits; @@ -14,6 +16,7 @@ mod types; pub use error::VfsError; pub use handle::{Access, VfsRef}; +pub use memory::MemoryBackend; pub use path::{VfsPath, VfsPathBuf}; pub use router::VfsRefBuilder; pub use traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; diff --git a/crates/shared-vfs/src/memory.rs b/crates/shared-vfs/src/memory.rs new file mode 100644 index 000000000..b564a5018 --- /dev/null +++ b/crates/shared-vfs/src/memory.rs @@ -0,0 +1,841 @@ +//! The generic in-memory backend. +//! +//! [`MemoryBackend`] carries the former MemStore semantics onto the VFS +//! trait surface: bytes keyed by canonical path, writes that materialize +//! their ancestor directories (no `mkdir` needed before a write), and +//! strict removals (absent is `NotFound`; a non-empty directory without +//! `recursive` is an error). `ExecId` attribution is accepted as a no-op: +//! every session shares the one map. It holds no resources and drops with +//! the run. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use crate::error::VfsError; +use crate::glob::{MAX_GLOB_PATTERN_BYTES, compile_glob, matches_tokens, validate_glob_grammar}; +use crate::path::VfsPath; +use crate::traits::{ExecId, Vfs, VfsAccess}; +use crate::types::{Entry, FileType, Stat}; + +/// The storage one backend shares with every session it vends. `BTreeMap` +/// and `BTreeSet` keep listing and glob results ordered without a sort +/// step. +#[derive(Debug)] +struct Tree { + /// File contents by canonical path. + files: BTreeMap>, + /// Every directory, including each file's ancestors. The root is + /// always present. + dirs: BTreeSet, +} + +impl Default for Tree { + fn default() -> Self { + Tree { + files: BTreeMap::new(), + dirs: BTreeSet::from(["/".to_owned()]), + } + } +} + +/// The ancestor directories of `path`, rootward: `/a/b/c` yields `/a` +/// then `/a/b`. The root itself is always present, so it is never yielded. +fn ancestors(path: &str) -> Vec<&str> { + let mut result = Vec::new(); + let mut rest = path; + while let Some(pos) = rest.rfind('/') { + if pos == 0 { + break; + } + result.push(&path[..pos]); + rest = &path[..pos]; + } + result.reverse(); + result +} + +impl Tree { + /// Whether `path` is a file or a directory. + fn contains(&self, path: &str) -> bool { + self.files.contains_key(path) || self.dirs.contains(path) + } + + /// Whether `path` has any descendants. + fn has_children(&self, path: &str) -> bool { + let prefix = format!("{path}/"); + self.files.keys().any(|key| key.starts_with(&prefix)) + || self.dirs.iter().any(|key| key.starts_with(&prefix)) + } + + /// The immediate child names of the directory at `path`, sorted. + fn children(&self, path: &str) -> Vec { + let prefix = if path == "/" { + "/".to_owned() + } else { + format!("{path}/") + }; + let mut names = BTreeSet::new(); + for key in self.files.keys().chain(self.dirs.iter()) { + if let Some(rest) = key.strip_prefix(prefix.as_str()) + && !rest.is_empty() + && !rest.contains('/') + { + names.insert(rest.to_owned()); + } + } + names.into_iter().collect() + } + + /// Metadata for a path known to exist, or `None`. Times and mode are + /// `None`: the memory backend does not track them, and an invented + /// mtime would be nondeterministic. + fn stat_of(&self, path: &str) -> Option { + if let Some(bytes) = self.files.get(path) { + return Some(Stat { + file_type: FileType::File, + size: bytes.len() as u64, + mode: None, + modified: None, + created: None, + }); + } + if self.dirs.contains(path) { + return Some(Stat { + file_type: FileType::Directory, + size: 0, + mode: None, + modified: None, + created: None, + }); + } + None + } + + /// Validates that `path` can receive a file: no directory already sits + /// at `path` and no ancestor is a file. Failure-atomic: callers run + /// this before any mutation. + fn check_file_destination(&self, path: &str) -> Result<(), VfsError> { + if self.dirs.contains(path) { + return Err(VfsError::IsADirectory(path.to_owned())); + } + if let Some(ancestor) = ancestors(path) + .into_iter() + .find(|ancestor| self.files.contains_key(*ancestor)) + { + return Err(VfsError::NotADirectory(ancestor.to_owned())); + } + Ok(()) + } + + /// Inserts every missing ancestor directory of `path`. + fn create_ancestors(&mut self, path: &str) { + for ancestor in ancestors(path) { + self.dirs.insert(ancestor.to_owned()); + } + } +} + +/// An in-memory [`Vfs`] backend. +/// +/// Files live in a [`BTreeMap`] keyed by canonical path, so listing and +/// glob results are ordered without a sort step. Clones share the same +/// storage. The zero value (`Default`) is a meaningful empty backend. +/// +/// # Examples +/// ``` +/// use shared_vfs::{MemoryBackend, VfsRef}; +/// +/// let vfs = VfsRef::new(MemoryBackend::new()); +/// let access = vfs.acquire(); +/// access.write("/notes.md", b"todo")?; +/// assert_eq!(access.read("/notes.md")?, b"todo"); +/// # Ok::<(), shared_vfs::VfsError>(()) +/// ``` +#[derive(Debug, Default, Clone)] +#[non_exhaustive] +pub struct MemoryBackend { + tree: Arc>, +} + +impl MemoryBackend { + /// Creates an empty in-memory backend. + #[must_use] + pub fn new() -> MemoryBackend { + MemoryBackend::default() + } +} + +impl Vfs for MemoryBackend { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + // Attribution is accepted as a no-op: every session shares the + // one map, and the claims model above the backend enforces + // conflicts. + let _ = id; + Ok(Box::new(MemoryAccess { + tree: Arc::clone(&self.tree), + })) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + let _ = id; + Ok(()) + } +} + +/// One identity's session with a [`MemoryBackend`]. The identity is +/// dropped on the floor: the map is shared and attribution is a no-op. +struct MemoryAccess { + tree: Arc>, +} + +impl MemoryAccess { + /// Poison-safe lock on the storage, held per call. + fn tree(&self) -> MutexGuard<'_, Tree> { + self.tree.lock().unwrap_or_else(PoisonError::into_inner) + } +} + +impl VfsAccess for MemoryAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + let tree = self.tree(); + if let Some(bytes) = tree.files.get(path.as_str()) { + return Ok(bytes.clone()); + } + if tree.dirs.contains(path.as_str()) { + return Err(VfsError::IsADirectory(path.to_string())); + } + Err(VfsError::NotFound(path.to_string())) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + let mut tree = self.tree(); + tree.check_file_destination(path.as_str())?; + tree.create_ancestors(path.as_str()); + tree.files.insert(path.to_string(), contents.to_vec()); + Ok(()) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + let mut tree = self.tree(); + if let Some(bytes) = tree.files.get_mut(path.as_str()) { + bytes.extend_from_slice(contents); + return Ok(()); + } + tree.check_file_destination(path.as_str())?; + tree.create_ancestors(path.as_str()); + tree.files.insert(path.to_string(), contents.to_vec()); + Ok(()) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let mut tree = self.tree(); + let key = path.as_str(); + if tree.files.remove(key).is_some() { + return Ok(()); + } + if !tree.dirs.contains(key) { + return Err(VfsError::NotFound(path.to_string())); + } + if key == "/" { + return Err(VfsError::PermissionDenied( + "the namespace root cannot be removed".into(), + )); + } + if tree.has_children(key) && !recursive { + return Err(VfsError::DirectoryNotEmpty(path.to_string())); + } + let prefix = format!("{key}/"); + tree.files.retain(|file, _| !file.starts_with(&prefix)); + tree.dirs.retain(|dir| !dir.starts_with(&prefix)); + tree.dirs.remove(key); + Ok(()) + } + + fn exists(&self, path: &VfsPath) -> Result { + Ok(self.tree().contains(path.as_str())) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + if pattern.len() > MAX_GLOB_PATTERN_BYTES { + return Err(VfsError::InvalidPath(format!( + "glob pattern exceeds {MAX_GLOB_PATTERN_BYTES} bytes" + ))); + } + if let Err(reason) = validate_glob_grammar(pattern) { + return Err(VfsError::InvalidPath(format!( + "invalid glob pattern {pattern:?}: {reason}" + ))); + } + // Compile once, then reuse the tokens across every key, so the + // per-key tokenization cost is not repeated while the storage + // lock is held. Matching itself is bounded and non-backtracking. + let tokens = compile_glob(pattern.as_bytes()); + let tree = self.tree(); + let mut matches: Vec = tree + .files + .keys() + .chain(tree.dirs.iter()) + .filter(|key| matches_tokens(&tokens, key.as_bytes())) + .cloned() + .collect(); + matches.sort_unstable(); + Ok(matches) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + let tree = self.tree(); + let key = path.as_str(); + if tree.files.contains_key(key) { + return Err(VfsError::NotADirectory(path.to_string())); + } + if !tree.dirs.contains(key) { + return Err(VfsError::NotFound(path.to_string())); + } + let mut entries = Vec::new(); + for name in tree.children(key) { + let full = if key == "/" { + format!("/{name}") + } else { + format!("{key}/{name}") + }; + let Some(stat) = tree.stat_of(&full) else { + continue; // unreachable: children() only yields existing paths + }; + entries.push(Entry { + name, + stat, + description: None, + }); + } + Ok(entries) + } + + fn stat(&self, path: &VfsPath) -> Result { + self.tree() + .stat_of(path.as_str()) + .ok_or_else(|| VfsError::NotFound(path.to_string())) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + let mut tree = self.tree(); + let key = path.as_str(); + if tree.contains(key) { + return Err(VfsError::AlreadyExists(path.to_string())); + } + if let Some(file) = ancestors(key) + .into_iter() + .find(|ancestor| tree.files.contains_key(*ancestor)) + { + return Err(VfsError::NotADirectory(file.to_owned())); + } + let missing_ancestors: Vec<&str> = ancestors(key) + .into_iter() + .filter(|ancestor| !tree.dirs.contains(*ancestor)) + .collect(); + if !recursive && !missing_ancestors.is_empty() { + return Err(VfsError::NotFound(format!( + "the parent of {path} does not exist" + ))); + } + tree.create_ancestors(key); + tree.dirs.insert(key.to_owned()); + Ok(()) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + let mut tree = self.tree(); + let source = from.as_str(); + let dest = to.as_str(); + if source == "/" { + return Err(VfsError::PermissionDenied( + "the namespace root cannot be renamed".into(), + )); + } + if dest.starts_with(&format!("{source}/")) { + return Err(VfsError::InvalidPath(format!( + "cannot rename {source} into its own descendant {dest}" + ))); + } + if let Some(bytes) = tree.files.get(source).cloned() { + tree.check_file_destination(dest)?; + tree.create_ancestors(dest); + tree.files.remove(source); + tree.files.insert(dest.to_owned(), bytes); + return Ok(()); + } + if !tree.dirs.contains(source) { + return Err(VfsError::NotFound(from.to_string())); + } + // A directory moves with its whole subtree. Validation finishes + // before any mutation, so a failed rename changes nothing. + if dest == "/" { + return Err(VfsError::PermissionDenied( + "a directory cannot be renamed onto the namespace root".into(), + )); + } + if tree.files.contains_key(dest) { + return Err(VfsError::NotADirectory(to.to_string())); + } + if tree.dirs.contains(dest) && tree.has_children(dest) { + return Err(VfsError::DirectoryNotEmpty(to.to_string())); + } + if let Some(ancestor) = ancestors(dest) + .into_iter() + .find(|ancestor| tree.files.contains_key(*ancestor)) + { + return Err(VfsError::NotADirectory(ancestor.to_owned())); + } + let prefix = format!("{source}/"); + let moved_files: Vec = tree + .files + .keys() + .filter(|key| key.starts_with(&prefix)) + .cloned() + .collect(); + let moved_dirs: Vec = tree + .dirs + .iter() + .filter(|key| key.starts_with(&prefix)) + .cloned() + .collect(); + for key in moved_files { + if let Some(bytes) = tree.files.remove(&key) { + tree.files.insert(format!("{dest}{}", &key[source.len()..]), bytes); + } + } + for key in moved_dirs { + tree.dirs.remove(&key); + tree.dirs.insert(format!("{dest}{}", &key[source.len()..])); + } + tree.dirs.remove(source); + tree.dirs.remove(dest); + tree.create_ancestors(dest); + tree.dirs.insert(dest.to_owned()); + Ok(()) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + let mut tree = self.tree(); + let Some(bytes) = tree.files.get(from.as_str()).cloned() else { + if tree.dirs.contains(from.as_str()) { + return Err(VfsError::IsADirectory(from.to_string())); + } + return Err(VfsError::NotFound(from.to_string())); + }; + tree.check_file_destination(to.as_str())?; + tree.create_ancestors(to.as_str()); + tree.files.insert(to.to_string(), bytes); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::MemoryBackend; + use crate::error::VfsError; + use crate::path::{VfsPath, canonicalize}; + use crate::traits::{ExecId, Vfs, VfsAccess}; + use crate::types::FileType; + + fn path(s: &str) -> Result { + canonicalize(s) + } + + /// Returns a session on a backend pre-populated through the write + /// path, so seeding exercises the same code the tests do. + fn seeded(files: &[(&str, &str)]) -> Result, VfsError> { + let mut backend = MemoryBackend::new(); + let mut access = backend.acquire(ExecId::vend())?; + for (name, text) in files { + access.write(&path(name)?, text.as_bytes())?; + } + Ok(access) + } + + #[test] + fn read_returns_the_exact_bytes_stored() -> Result<(), VfsError> { + let mut backend = MemoryBackend::new(); + let mut access = backend.acquire(ExecId::vend())?; + let bytes = [0x00_u8, 0xff, 0x00, 0x7f]; + access.write(&path("/bin.dat")?, &bytes)?; + assert_eq!(access.read(&path("/bin.dat")?)?, bytes); + Ok(()) + } + + #[test] + fn read_of_an_absent_path_is_not_found() -> Result<(), VfsError> { + let access = seeded(&[])?; + assert!(matches!( + access.read(&path("/missing.txt")?), + Err(VfsError::NotFound(_)) + )); + Ok(()) + } + + #[test] + fn read_of_a_directory_is_is_a_directory() -> Result<(), VfsError> { + let access = seeded(&[("/dir/f.txt", "x")])?; + assert!(matches!( + access.read(&path("/dir")?), + Err(VfsError::IsADirectory(_)) + )); + Ok(()) + } + + #[test] + fn read_range_slices_bytes_and_clips_at_the_end() -> Result<(), VfsError> { + let access = seeded(&[("/f.txt", "hello world")])?; + assert_eq!(access.read_range(&path("/f.txt")?, 6, 5)?, b"world"); + assert_eq!(access.read_range(&path("/f.txt")?, 6, 100)?, b"world"); + assert!(access.read_range(&path("/f.txt")?, 100, 5)?.is_empty()); + Ok(()) + } + + #[test] + fn write_creates_overwrites_and_materializes_ancestor_directories() -> Result<(), VfsError> { + let mut access = seeded(&[])?; + access.write(&path("/a/b/f.txt")?, b"one")?; + assert_eq!(access.read(&path("/a/b/f.txt")?)?, b"one"); + // MemStore semantics: no mkdir was needed; the ancestors exist. + assert!(access.exists(&path("/a")?)?); + assert!(access.exists(&path("/a/b")?)?); + access.write(&path("/a/b/f.txt")?, b"two")?; + assert_eq!(access.read(&path("/a/b/f.txt")?)?, b"two"); + Ok(()) + } + + #[test] + fn write_at_a_directory_path_is_rejected_without_touching_the_tree() -> Result<(), VfsError> { + let mut access = seeded(&[("/dir/f.txt", "x")])?; + assert!(matches!( + access.write(&path("/dir")?, b"y"), + Err(VfsError::IsADirectory(_)) + )); + assert_eq!(access.read(&path("/dir/f.txt")?)?, b"x"); + Ok(()) + } + + #[test] + fn append_creates_when_absent_and_extends_when_present() -> Result<(), VfsError> { + let mut access = seeded(&[])?; + access.append(&path("/log.txt")?, b"first\n")?; + access.append(&path("/log.txt")?, b"second")?; + assert_eq!(access.read(&path("/log.txt")?)?, b"first\nsecond"); + Ok(()) + } + + #[test] + fn remove_of_an_absent_path_is_not_found() -> Result<(), VfsError> { + let mut access = seeded(&[])?; + assert!(matches!( + access.remove(&path("/gone.txt")?, false), + Err(VfsError::NotFound(_)) + )); + Ok(()) + } + + #[test] + fn remove_of_a_file_removes_it() -> Result<(), VfsError> { + let mut access = seeded(&[("/f.txt", "x")])?; + access.remove(&path("/f.txt")?, false)?; + assert!(!access.exists(&path("/f.txt")?)?); + Ok(()) + } + + #[test] + fn remove_of_a_nonempty_directory_without_recursive_is_an_error() -> Result<(), VfsError> { + let mut access = seeded(&[("/dir/f.txt", "x")])?; + assert!(matches!( + access.remove(&path("/dir")?, false), + Err(VfsError::DirectoryNotEmpty(_)) + )); + // The failed removal changed nothing. + assert_eq!(access.read(&path("/dir/f.txt")?)?, b"x"); + assert!(access.exists(&path("/dir")?)?); + Ok(()) + } + + #[test] + fn remove_of_an_empty_directory_without_recursive_succeeds() -> Result<(), VfsError> { + let mut access = seeded(&[])?; + access.mkdir(&path("/empty")?, false)?; + access.remove(&path("/empty")?, false)?; + assert!(!access.exists(&path("/empty")?)?); + Ok(()) + } + + #[test] + fn remove_with_recursive_deletes_the_whole_subtree() -> Result<(), VfsError> { + let mut access = seeded(&[ + ("/d/a.txt", "a"), + ("/d/sub/b.txt", "b"), + ("/keep.txt", "k"), + ])?; + access.remove(&path("/d")?, true)?; + assert!(!access.exists(&path("/d")?)?); + assert!(!access.exists(&path("/d/sub")?)?); + assert!(!access.exists(&path("/d/sub/b.txt")?)?); + assert_eq!(access.read(&path("/keep.txt")?)?, b"k"); + Ok(()) + } + + #[test] + fn the_namespace_root_cannot_be_removed() -> Result<(), VfsError> { + let mut access = seeded(&[("/f.txt", "x")])?; + assert!(matches!( + access.remove(&path("/")?, true), + Err(VfsError::PermissionDenied(_)) + )); + assert_eq!(access.read(&path("/f.txt")?)?, b"x"); + Ok(()) + } + + #[test] + fn exists_distinguishes_files_directories_and_absence() -> Result<(), VfsError> { + let access = seeded(&[("/dir/f.txt", "x")])?; + assert!(access.exists(&path("/dir/f.txt")?)?); + assert!(access.exists(&path("/dir")?)?); + assert!(access.exists(&path("/")?)?); + assert!(!access.exists(&path("/dir/missing.txt")?)?); + Ok(()) + } + + #[test] + fn glob_matches_star_within_a_segment_and_double_star_across() -> Result<(), VfsError> { + let access = seeded(&[ + ("/src/a.rs", ""), + ("/src/b.rs", ""), + ("/src/deep/c.rs", ""), + ("/notes/today.md", ""), + ])?; + assert_eq!( + access.glob("/src/*.rs")?, + vec!["/src/a.rs".to_owned(), "/src/b.rs".to_owned()] + ); + assert_eq!( + access.glob("/src/**/*.rs")?, + vec![ + "/src/a.rs".to_owned(), + "/src/b.rs".to_owned(), + "/src/deep/c.rs".to_owned(), + ] + ); + assert_eq!(access.glob("/**/*.md")?, vec!["/notes/today.md".to_owned()]); + Ok(()) + } + + #[test] + fn glob_results_are_sorted_and_include_directories() -> Result<(), VfsError> { + let access = seeded(&[("/d/b.txt", ""), ("/d/a.txt", ""), ("/d/sub/c.txt", "")])?; + assert_eq!( + access.glob("/d/*")?, + vec![ + "/d/a.txt".to_owned(), + "/d/b.txt".to_owned(), + "/d/sub".to_owned(), + ] + ); + Ok(()) + } + + #[test] + fn glob_rejects_invalid_patterns() -> Result<(), VfsError> { + let access = seeded(&[("/f.txt", "x")])?; + assert!(matches!( + access.glob("/a/***/b"), + Err(VfsError::InvalidPath(_)) + )); + assert!(matches!( + access.glob("/a\\b"), + Err(VfsError::InvalidPath(_)) + )); + Ok(()) + } + + #[test] + fn list_returns_sorted_entries_with_stats() -> Result<(), VfsError> { + let access = seeded(&[("/d/b.txt", "bb"), ("/d/a.txt", "a"), ("/d/sub/c.txt", "c")])?; + let entries = access.list(&path("/d")?)?; + let names: Vec<&str> = entries.iter().map(|entry| entry.name.as_str()).collect(); + assert_eq!(names, vec!["a.txt", "b.txt", "sub"]); + assert_eq!(entries[0].stat.file_type, FileType::File); + assert_eq!(entries[0].stat.size, 1); + assert_eq!(entries[2].stat.file_type, FileType::Directory); + assert!(entries.iter().all(|entry| entry.description.is_none())); + Ok(()) + } + + #[test] + fn list_of_a_file_or_an_absent_path_is_an_error() -> Result<(), VfsError> { + let access = seeded(&[("/f.txt", "x")])?; + assert!(matches!( + access.list(&path("/f.txt")?), + Err(VfsError::NotADirectory(_)) + )); + assert!(matches!( + access.list(&path("/missing")?), + Err(VfsError::NotFound(_)) + )); + Ok(()) + } + + #[test] + fn stat_reports_kinds_and_sizes_without_fabricated_times() -> Result<(), VfsError> { + let access = seeded(&[("/dir/f.txt", "hello")])?; + let file = access.stat(&path("/dir/f.txt")?)?; + assert_eq!(file.file_type, FileType::File); + assert_eq!(file.size, 5); + assert!(file.mode.is_none() && file.modified.is_none() && file.created.is_none()); + let dir = access.stat(&path("/dir")?)?; + assert_eq!(dir.file_type, FileType::Directory); + assert!(matches!( + access.stat(&path("/missing")?), + Err(VfsError::NotFound(_)) + )); + Ok(()) + } + + #[test] + fn mkdir_creates_directories_and_rejects_existing_paths() -> Result<(), VfsError> { + let mut access = seeded(&[("/f.txt", "x")])?; + access.mkdir(&path("/new")?, false)?; + assert!(access.exists(&path("/new")?)?); + assert!(matches!( + access.mkdir(&path("/new")?, false), + Err(VfsError::AlreadyExists(_)) + )); + assert!(matches!( + access.mkdir(&path("/f.txt")?, false), + Err(VfsError::AlreadyExists(_)) + )); + Ok(()) + } + + #[test] + fn mkdir_without_recursive_requires_an_existing_parent() -> Result<(), VfsError> { + let mut access = seeded(&[])?; + assert!(matches!( + access.mkdir(&path("/a/b")?, false), + Err(VfsError::NotFound(_)) + )); + access.mkdir(&path("/a/b")?, true)?; + assert!(access.exists(&path("/a")?)?); + assert!(access.exists(&path("/a/b")?)?); + Ok(()) + } + + #[test] + fn mkdir_through_a_file_parent_is_not_a_directory() -> Result<(), VfsError> { + let mut access = seeded(&[("/f.txt", "x")])?; + assert!(matches!( + access.mkdir(&path("/f.txt/g")?, true), + Err(VfsError::NotADirectory(_)) + )); + Ok(()) + } + + #[test] + fn rename_moves_a_file_and_leaves_nothing_behind() -> Result<(), VfsError> { + let mut access = seeded(&[("/from.txt", "data")])?; + access.rename(&path("/from.txt")?, &path("/sub/to.txt")?)?; + assert!(!access.exists(&path("/from.txt")?)?); + assert_eq!(access.read(&path("/sub/to.txt")?)?, b"data"); + Ok(()) + } + + #[test] + fn rename_moves_a_directory_subtree() -> Result<(), VfsError> { + let mut access = seeded(&[("/d/a.txt", "a"), ("/d/sub/b.txt", "b")])?; + access.rename(&path("/d")?, &path("/moved")?)?; + assert!(!access.exists(&path("/d")?)?); + assert_eq!(access.read(&path("/moved/a.txt")?)?, b"a"); + assert_eq!(access.read(&path("/moved/sub/b.txt")?)?, b"b"); + Ok(()) + } + + #[test] + fn renaming_a_directory_onto_the_root_is_rejected() -> Result<(), VfsError> { + let mut access = seeded(&[("/d/a.txt", "a"), ("/other.txt", "o")])?; + assert!(matches!( + access.rename(&path("/d")?, &path("/")?), + Err(VfsError::PermissionDenied(_)) + )); + // The failed rename changed nothing: the subtree is intact. + assert_eq!(access.read(&path("/d/a.txt")?)?, b"a"); + assert_eq!(access.read(&path("/other.txt")?)?, b"o"); + let root: Vec = access + .list(&path("/")?)? + .into_iter() + .map(|entry| entry.name) + .collect(); + assert_eq!(root, vec!["d".to_owned(), "other.txt".to_owned()]); + Ok(()) + } + + #[test] + fn a_failed_rename_leaves_source_and_destination_unchanged() -> Result<(), VfsError> { + let mut access = seeded(&[("/dst.txt", "old")])?; + assert!(matches!( + access.rename(&path("/missing.txt")?, &path("/dst.txt")?), + Err(VfsError::NotFound(_)) + )); + assert_eq!(access.read(&path("/dst.txt")?)?, b"old"); + // Renaming a directory into its own descendant is rejected. + access.mkdir(&path("/d")?, false)?; + assert!(matches!( + access.rename(&path("/d")?, &path("/d/inner")?), + Err(VfsError::InvalidPath(_)) + )); + assert!(access.exists(&path("/d")?)?); + Ok(()) + } + + #[test] + fn copy_duplicates_a_files_bytes() -> Result<(), VfsError> { + let mut access = seeded(&[("/src.txt", "data")])?; + access.copy(&path("/src.txt")?, &path("/dst.txt")?)?; + assert_eq!(access.read(&path("/src.txt")?)?, b"data"); + assert_eq!(access.read(&path("/dst.txt")?)?, b"data"); + Ok(()) + } + + #[test] + fn copy_rejects_directories_and_a_failed_copy_changes_nothing() -> Result<(), VfsError> { + let mut access = seeded(&[("/d/f.txt", "x"), ("/dst.txt", "old")])?; + assert!(matches!( + access.copy(&path("/d")?, &path("/dst.txt")?), + Err(VfsError::IsADirectory(_)) + )); + assert_eq!(access.read(&path("/dst.txt")?)?, b"old"); + assert!(matches!( + access.copy(&path("/missing.txt")?, &path("/dst.txt")?), + Err(VfsError::NotFound(_)) + )); + assert_eq!(access.read(&path("/dst.txt")?)?, b"old"); + Ok(()) + } + + #[test] + fn acquire_and_release_accept_attribution_as_a_no_op() -> Result<(), VfsError> { + let mut backend = MemoryBackend::new(); + let mut first = backend.acquire(ExecId::vend())?; + first.write(&path("/f.txt")?, b"shared")?; + // A second identity's session sees the same map. + let second = backend.acquire(ExecId::vend())?; + assert_eq!(second.read(&path("/f.txt")?)?, b"shared"); + drop(first); + drop(second); + backend.release(ExecId::vend())?; + Ok(()) + } + + #[test] + fn the_default_is_a_meaningful_empty_backend() -> Result<(), VfsError> { + let mut backend = MemoryBackend::default(); + let access = backend.acquire(ExecId::vend())?; + assert!(access.exists(&path("/")?)?); + assert!(!access.exists(&path("/anything")?)?); + assert!(access.list(&path("/")?)?.is_empty()); + Ok(()) + } +} diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index 735cc6a75..bd1b023e1 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -604,7 +604,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 5: memory backend +### Step 5: memory backend [completed] - Component: shared-vfs backends - Implement the generic in-memory backend in shared-vfs carrying former MemStore semantics (Default where a zero value is meaningful; acquire/release accept ExecId attribution as a no-op). diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index e4793878c..a801d9c33 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -97,3 +97,7 @@ - Decision: cross-mount rename/copy return `Unsupported` instead of read-plus-write | Falsifier: a caller needing atomic cross-mount moves. - Decision: `Router::release` and mounted-handle `release` are no-ops; teardown flows through the routing session's `Drop` | Falsifier: a backend requiring explicit release independent of `Drop`. - Decision: `overlay()` shares the base's policy `Arc` as well as its claims table | Falsifier: a requirement that overlays carry independent policy. +- Step 5: memory backend - `cargo nextest run -p shared-vfs` - 91 passed, 0 failed; clippy `-D warnings` clean. Review: 1 Critical (renaming a directory onto the namespace root bypassed the DirectoryNotEmpty guard and rewrote subtree keys to unreachable `//...` paths - silent data loss), closed by rejecting `dest == "/"` in the directory branch with a regression test. + - Decision: writes materialize ancestor directories instead of requiring mkdir (MemStore flat-map semantics carried over) | Falsifier: the Store facade or a host backend needs POSIX ENOENT-on-missing-parent behavior. + - Decision: glob results include directories, not just files | Falsifier: a caller (engine adapter, grep default) misbehaves when directories match. + - Decision: backend named `MemoryBackend` (plan pins no name; parallels `HostBackend`) | Falsifier: a later plan step or review pins a different name. From f41d5869325bd8b3461dd8e756ec7b0ec8173b51 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 19:17:43 -0700 Subject: [PATCH 06/26] Add thin host filesystem backend to shared-vfs The virtual namespace gains a backend that serves host OS directories directly through standard filesystem calls. It offers two construction modes: identity, where a virtual path is the host path, and rooted, where every resolved path is containment-checked against a canonicalized root directory. Writes, copies, and renames are failure-atomic through a sibling temp file plus rename, and a read-only flag rejects every mutation before anything is touched. The backend is deliberately thin; resolver hardening and symlink policy are left to a later hardening pass. - `HostBackend::rooted` canonicalizes and validates the directory at construction, returning an error rather than panicking, so every later containment check compares against an already-canonical root. - `HostBackend::acquire` accepts the execution identity as a no-op because the host filesystem holds no per-identity state; conflict enforcement lives above the backend. - `atomic_write` creates a sibling temp file, syncs it, and renames it over the destination, removing the temp file when any step fails. - `HostAccess::exists` returns Ok(false) only for a confirmed absence; permission denials and other I/O failures surface as errors. - `walk_root` starts the glob walk at the literal prefix's parent directory, so a wildcard-free pattern matches the literal file itself. - `HostAccess::rename` rejects renaming the mounted root, onto the mounted root, or into its own descendant before any syscall. - `contain` canonicalizes only the nearest existing ancestor of a candidate, so a dangling symlink inside the root is not itself resolved; writing through one follows host symlink semantics. Design: new encapsulated-invariant @ crates/shared-vfs/src/host.rs::HostBackend boundary: pub Design: new global-state @ crates/shared-vfs/src/host.rs::TEMP_COUNTER Design: new pure-function @ crates/shared-vfs/src/host.rs::map_io deps: std::io::Error,str Plan: vibe/2026-09-11-3-vfs-foundation.md --- crates/shared-vfs/src/host.rs | 912 ++++++++++++++++++++++++++++ crates/shared-vfs/src/lib.rs | 2 + crates/shared-vfs/src/memory.rs | 9 +- vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 4 + 5 files changed, 922 insertions(+), 7 deletions(-) create mode 100644 crates/shared-vfs/src/host.rs diff --git a/crates/shared-vfs/src/host.rs b/crates/shared-vfs/src/host.rs new file mode 100644 index 000000000..8c4d343aa --- /dev/null +++ b/crates/shared-vfs/src/host.rs @@ -0,0 +1,912 @@ +//! The host filesystem backend, stage 1 (thin). +//! +//! [`HostBackend`] serves host-OS directories behind the virtual +//! namespace over direct `std::fs` calls. Two constructors: +//! [`HostBackend::identity`] (the virtual path IS the host path) and +//! [`HostBackend::rooted`] (chroot-style, with lexical plus +//! canonicalize containment). Writes, copies, and renames are +//! failure-atomic: a sibling temp file plus rename, so a failed +//! operation leaves source, destination, and accounting unchanged. +//! +//! Stage 2 hardening (the Bashkit RealFs resolver trio, symlink +//! policies, Windows long paths and device names) is deferred. The +//! known stage 1 limitation: containment canonicalizes the nearest +//! existing ancestor, so a dangling symlink inside the root is not +//! itself resolved; writing through one follows the host's own +//! symlink semantics. + +use std::fs::{self, File}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use crate::error::VfsError; +use crate::glob::{MAX_GLOB_PATTERN_BYTES, compile_glob, matches_tokens, validate_glob_grammar}; +use crate::path::{VfsPath, canonicalize}; +use crate::traits::{ExecId, Vfs, VfsAccess}; +use crate::types::{Entry, FileType, Stat}; + +/// Maps an I/O failure to the error kind the trait surface promises. +fn map_io(path: &str, err: &std::io::Error) -> VfsError { + let message = format!("{path}: {err}"); + match err.kind() { + std::io::ErrorKind::NotFound => VfsError::NotFound(message), + std::io::ErrorKind::PermissionDenied => VfsError::PermissionDenied(message), + std::io::ErrorKind::AlreadyExists => VfsError::AlreadyExists(message), + std::io::ErrorKind::IsADirectory => VfsError::IsADirectory(message), + std::io::ErrorKind::NotADirectory => VfsError::NotADirectory(message), + std::io::ErrorKind::DirectoryNotEmpty => VfsError::DirectoryNotEmpty(message), + _ => VfsError::Backend(message), + } +} + +/// How virtual paths reach host paths: verbatim, or contained under a +/// canonicalized root. +#[derive(Debug, Clone)] +enum HostRoot { + /// The virtual path is the host path (modulo the Windows drive + /// letter spelling). + Identity, + /// Chroot-style: the virtual root is this canonical host directory. + Rooted(PathBuf), +} + +/// Translates an identity-mode virtual path to a host path. On Windows +/// the virtual spelling of `C:\Users\x` is `/C:/Users/x`: a leading +/// slash before a drive letter is stripped. +#[cfg(windows)] +fn identity_to_host(virtual_path: &str) -> PathBuf { + let bytes = virtual_path.as_bytes(); + if bytes.len() >= 3 && bytes[0] == b'/' && bytes[1].is_ascii_alphabetic() && bytes[2] == b':' { + return PathBuf::from(&virtual_path[1..]); + } + PathBuf::from(virtual_path) +} + +/// Translates an identity-mode virtual path to a host path. +#[cfg(not(windows))] +fn identity_to_host(virtual_path: &str) -> PathBuf { + PathBuf::from(virtual_path) +} + +/// Translates a host path back to its identity-mode virtual spelling: +/// forward slashes, and on Windows a leading slash before a drive +/// letter (`C:\Users\x` becomes `/C:/Users/x`). +#[cfg(windows)] +fn identity_to_virtual(host: &Path) -> String { + let spelled = host.to_string_lossy().replace('\\', "/"); + let bytes = spelled.as_bytes(); + if bytes.len() >= 2 && bytes[0].is_ascii_alphabetic() && bytes[1] == b':' { + return format!("/{spelled}"); + } + spelled +} + +/// Translates a host path back to its identity-mode virtual spelling. +#[cfg(not(windows))] +fn identity_to_virtual(host: &Path) -> String { + host.to_string_lossy().into_owned() +} + +/// Joins a canonical virtual path onto a host root. The virtual path is +/// canonical (dot segments resolved at receipt, forward slashes), so +/// the join cannot escape lexically. +fn join_virtual(root: &Path, virtual_path: &str) -> PathBuf { + let mut host = root.to_path_buf(); + for segment in virtual_path.split('/').filter(|s| !s.is_empty()) { + host.push(segment); + } + host +} + +/// Canonicalize containment: the candidate's nearest existing ancestor +/// is canonicalized and must sit under the (already canonical) root; +/// the missing tail is re-appended lexically. This catches link escapes +/// for existing paths while still resolving paths yet to be created. +fn contain(root: &Path, candidate: &Path, original: VfsPath) -> Result { + let denied = || VfsError::PermissionDenied(format!("{original} escapes the mounted root")); + let mut ancestor = candidate; + let mut tail: Vec<&std::ffi::OsStr> = Vec::new(); + loop { + if ancestor.exists() { + let canonical = + fs::canonicalize(ancestor).map_err(|err| map_io(original.as_str(), &err))?; + if !canonical.starts_with(root) { + return Err(denied()); + } + let mut resolved = canonical; + for component in tail.iter().rev() { + resolved.push(component); + } + return Ok(resolved); + } + let Some(parent) = ancestor.parent() else { + return Err(denied()); + }; + let Some(name) = ancestor.file_name() else { + return Err(denied()); + }; + tail.push(name); + ancestor = parent; + } +} + +/// Uniquifies failure-atomic temp file names within the process. +static TEMP_COUNTER: AtomicU64 = AtomicU64::new(0); + +/// Writes `contents` to `dest` failure-atomically: a sibling temp file +/// plus rename, so a failed write leaves the destination unchanged and +/// no temp file behind. +fn atomic_write(dest: &Path, contents: &[u8]) -> Result<(), VfsError> { + let display = dest.to_string_lossy().into_owned(); + let Some(parent) = dest.parent() else { + return Err(VfsError::InvalidPath(format!( + "{display} has no parent directory" + ))); + }; + let Some(name) = dest.file_name() else { + return Err(VfsError::InvalidPath(format!("{display} has no file name"))); + }; + let temp = parent.join(format!( + ".{}.vfs-tmp-{}-{}", + name.to_string_lossy(), + std::process::id(), + TEMP_COUNTER.fetch_add(1, Ordering::Relaxed) + )); + let outcome = (|| { + let mut file = File::create(&temp).map_err(|err| map_io(&display, &err))?; + file.write_all(contents) + .map_err(|err| map_io(&display, &err))?; + file.sync_all().map_err(|err| map_io(&display, &err))?; + drop(file); + fs::rename(&temp, dest).map_err(|err| map_io(&display, &err)) + })(); + if outcome.is_err() { + let _ = fs::remove_file(&temp); + } + outcome +} + +/// Creates the destination's ancestor directories, matching the memory +/// backend's materialize-on-write semantics. +fn create_parent(host: &Path, path: VfsPath) -> Result<(), VfsError> { + if let Some(parent) = host.parent() { + fs::create_dir_all(parent).map_err(|err| map_io(path.as_str(), &err))?; + } + Ok(()) +} + +/// The directory the walk must start from: the literal leading +/// segments of the pattern. A wildcard-free pattern can match only the +/// literal path itself, so the walk starts from its parent directory +/// for the literal file to be among the collected candidates, matching +/// the memory backend's parity. +fn walk_root(pattern: &str) -> &str { + let literal = match pattern.find('*') { + Some(star) => &pattern[..star], + None => pattern, + }; + match literal.rfind('/') { + Some(0) | None => "/", + Some(slash) => &literal[..slash], + } +} + +/// Collects every path under `dir`, files and directories, without +/// descending into links: a linked directory is collected, not +/// followed, so the walk can neither leave the mounted root nor loop. +fn walk(dir: &Path, found: &mut Vec) -> Result<(), VfsError> { + let entries = fs::read_dir(dir).map_err(|err| map_io(&dir.to_string_lossy(), &err))?; + for entry in entries { + let entry = entry.map_err(|err| map_io(&dir.to_string_lossy(), &err))?; + let path = entry.path(); + let file_type = entry + .file_type() + .map_err(|err| map_io(&dir.to_string_lossy(), &err))?; + found.push(path.clone()); + if file_type.is_dir() { + walk(&path, found)?; + } + } + Ok(()) +} + +/// Maps a host file type to the named POSIX kinds. +#[cfg(unix)] +fn file_type_of(file_type: fs::FileType) -> FileType { + use std::os::unix::fs::FileTypeExt; + if file_type.is_dir() { + FileType::Directory + } else if file_type.is_symlink() { + FileType::Symlink + } else if file_type.is_file() { + FileType::File + } else if file_type.is_fifo() { + FileType::Fifo + } else if file_type.is_socket() { + FileType::Socket + } else if file_type.is_char_device() { + FileType::CharDevice + } else if file_type.is_block_device() { + FileType::BlockDevice + } else { + FileType::File + } +} + +/// Maps a host file type to the named POSIX kinds. Windows distinguishes +/// only files, directories, and symlinks through `std`. +#[cfg(not(unix))] +fn file_type_of(file_type: fs::FileType) -> FileType { + if file_type.is_dir() { + FileType::Directory + } else if file_type.is_symlink() { + FileType::Symlink + } else { + FileType::File + } +} + +/// POSIX mode bits where the host tracks them. +#[cfg(unix)] +fn mode_of(metadata: &fs::Metadata) -> Option { + use std::os::unix::fs::PermissionsExt; + Some(metadata.permissions().mode()) +} + +/// POSIX mode bits where the host tracks them: not on Windows. +#[cfg(not(unix))] +fn mode_of(_: &fs::Metadata) -> Option { + None +} + +/// Builds metadata without fabricating fields the host does not track. +fn stat_of(metadata: &fs::Metadata) -> Stat { + Stat { + file_type: file_type_of(metadata.file_type()), + size: metadata.len(), + mode: mode_of(metadata), + modified: metadata.modified().ok(), + created: metadata.created().ok(), + } +} + +/// A host filesystem backend behind the virtual namespace. +/// +/// Stage 1 (thin): direct `std::fs` operations, lexical plus +/// canonicalize containment for [`HostBackend::rooted`], and +/// failure-atomic writes, copies, and renames. `ExecId` attribution is +/// accepted as a no-op: the host filesystem holds no per-identity +/// state. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct HostBackend { + root: HostRoot, + read_only: bool, +} + +impl HostBackend { + /// A backend whose virtual paths ARE host paths: virtual + /// `/a/b` is host `/a/b` (on Windows, virtual `/C:/a/b` is host + /// `C:\a\b`). No containment applies. + #[must_use] + pub fn identity() -> HostBackend { + HostBackend { + root: HostRoot::Identity, + read_only: false, + } + } + + /// A chroot-style backend: the virtual root is `dir`, canonicalized + /// and validated as a directory at construction. Every resolved + /// path is containment-checked against the canonical root. + /// + /// # Errors + /// + /// Returns [`VfsError::NotFound`] when `dir` is absent, or + /// [`VfsError::NotADirectory`] when it is not a directory. + pub fn rooted(dir: impl AsRef) -> Result { + let display = dir.as_ref().to_string_lossy().into_owned(); + let canonical = fs::canonicalize(dir.as_ref()).map_err(|err| map_io(&display, &err))?; + if !canonical.is_dir() { + return Err(VfsError::NotADirectory(display)); + } + Ok(HostBackend { + root: HostRoot::Rooted(canonical), + read_only: false, + }) + } + + /// Sets whether the backend rejects all mutations. The flag is a + /// property of the mount, orthogonal to policy. + #[must_use] + pub fn with_read_only(mut self, read_only: bool) -> HostBackend { + self.read_only = read_only; + self + } +} + +impl Vfs for HostBackend { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + // Attribution is accepted as a no-op: the host filesystem holds + // no per-identity state, and the claims model above the backend + // enforces conflicts. + let _ = id; + Ok(Box::new(HostAccess { + root: self.root.clone(), + read_only: self.read_only, + })) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + let _ = id; + Ok(()) + } + + fn read_only(&self) -> bool { + self.read_only + } +} + +/// One identity's session with a [`HostBackend`]. The identity is +/// dropped on the floor: attribution is a no-op. +struct HostAccess { + root: HostRoot, + read_only: bool, +} + +impl HostAccess { + /// Resolves a canonical virtual path to its host path, applying + /// containment in rooted mode. + fn resolve(&self, path: VfsPath) -> Result { + match &self.root { + HostRoot::Identity => Ok(identity_to_host(path.as_str())), + HostRoot::Rooted(root) => { + let candidate = join_virtual(root, path.as_str()); + contain(root, &candidate, path) + } + } + } + + /// Translates a host path back to its virtual spelling. + fn to_virtual(&self, host: &Path) -> String { + match &self.root { + HostRoot::Identity => identity_to_virtual(host), + HostRoot::Rooted(root) => { + let relative = host.strip_prefix(root).unwrap_or(host); + let mut virtual_path = String::new(); + for component in relative.components() { + virtual_path.push('/'); + virtual_path.push_str(&component.as_os_str().to_string_lossy()); + } + if virtual_path.is_empty() { + "/".to_owned() + } else { + virtual_path + } + } + } + } + + /// Rejects mutations on a read-only backend before anything is + /// touched: a denied operation never partially applies. + fn check_writable(&self, path: VfsPath) -> Result<(), VfsError> { + if self.read_only { + return Err(VfsError::PermissionDenied(format!( + "the host backend is read-only, so {path} cannot be mutated" + ))); + } + Ok(()) + } +} + +impl VfsAccess for HostAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + let host = self.resolve(*path)?; + if host.is_dir() { + return Err(VfsError::IsADirectory(path.to_string())); + } + fs::read(&host).map_err(|err| map_io(path.as_str(), &err)) + } + + fn read_range(&self, path: &VfsPath, offset: u64, len: u64) -> Result, VfsError> { + // Seek, never materialize: the host can position directly. + let host = self.resolve(*path)?; + if host.is_dir() { + return Err(VfsError::IsADirectory(path.to_string())); + } + let mut file = File::open(&host).map_err(|err| map_io(path.as_str(), &err))?; + file.seek(SeekFrom::Start(offset)) + .map_err(|err| map_io(path.as_str(), &err))?; + let mut buffer = Vec::new(); + file.take(len) + .read_to_end(&mut buffer) + .map_err(|err| map_io(path.as_str(), &err))?; + Ok(buffer) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.check_writable(*path)?; + let host = self.resolve(*path)?; + if host.is_dir() { + return Err(VfsError::IsADirectory(path.to_string())); + } + create_parent(&host, *path)?; + atomic_write(&host, contents) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.check_writable(*path)?; + let host = self.resolve(*path)?; + if host.is_dir() { + return Err(VfsError::IsADirectory(path.to_string())); + } + create_parent(&host, *path)?; + let mut file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&host) + .map_err(|err| map_io(path.as_str(), &err))?; + file.write_all(contents) + .map_err(|err| map_io(path.as_str(), &err)) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.check_writable(*path)?; + if path.as_str() == "/" { + return Err(VfsError::PermissionDenied( + "the mounted root cannot be removed".into(), + )); + } + let host = self.resolve(*path)?; + let metadata = fs::symlink_metadata(&host).map_err(|err| map_io(path.as_str(), &err))?; + // symlink_metadata does not follow links: a symlink is removed + // as a link, never its target. + if metadata.is_dir() { + if recursive { + fs::remove_dir_all(&host) + } else { + fs::remove_dir(&host) + } + } else { + fs::remove_file(&host) + } + .map_err(|err| map_io(path.as_str(), &err)) + } + + fn exists(&self, path: &VfsPath) -> Result { + let host = self.resolve(*path)?; + // symlink_metadata counts a dangling link as existing. Only a + // confirmed absence is Ok(false); every other failure (a + // denied permission, a genuine I/O error) surfaces as Err, as + // the trait contract requires. + match fs::symlink_metadata(&host) { + Ok(_) => Ok(true), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(err) => Err(map_io(path.as_str(), &err)), + } + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + if pattern.len() > MAX_GLOB_PATTERN_BYTES { + return Err(VfsError::InvalidPath(format!( + "glob pattern exceeds {MAX_GLOB_PATTERN_BYTES} bytes" + ))); + } + if let Err(reason) = validate_glob_grammar(pattern) { + return Err(VfsError::InvalidPath(format!( + "invalid glob pattern {pattern:?}: {reason}" + ))); + } + let tokens = compile_glob(pattern.as_bytes()); + let root = self.resolve(canonicalize(walk_root(pattern))?)?; + if !root.is_dir() { + return Ok(Vec::new()); + } + let mut found = Vec::new(); + walk(&root, &mut found)?; + let mut matches: Vec = found + .iter() + .map(|host| self.to_virtual(host)) + .filter(|virtual_path| matches_tokens(&tokens, virtual_path.as_bytes())) + .collect(); + matches.sort_unstable(); + Ok(matches) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + let host = self.resolve(*path)?; + let entries = fs::read_dir(&host).map_err(|err| map_io(path.as_str(), &err))?; + let mut result = Vec::new(); + for entry in entries { + let entry = entry.map_err(|err| map_io(path.as_str(), &err))?; + // DirEntry::metadata does not follow symlinks. + let metadata = entry + .metadata() + .map_err(|err| map_io(path.as_str(), &err))?; + result.push(Entry { + name: entry.file_name().to_string_lossy().into_owned(), + stat: stat_of(&metadata), + description: None, + }); + } + result.sort_by(|a, b| a.name.cmp(&b.name)); + Ok(result) + } + + fn stat(&self, path: &VfsPath) -> Result { + let host = self.resolve(*path)?; + let metadata = fs::symlink_metadata(&host).map_err(|err| map_io(path.as_str(), &err))?; + Ok(stat_of(&metadata)) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.check_writable(*path)?; + let host = self.resolve(*path)?; + if fs::symlink_metadata(&host).is_ok() { + return Err(VfsError::AlreadyExists(path.to_string())); + } + if recursive { + fs::create_dir_all(&host) + } else { + fs::create_dir(&host) + } + .map_err(|err| map_io(path.as_str(), &err)) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.check_writable(*from)?; + if from.as_str() == "/" { + return Err(VfsError::PermissionDenied( + "the mounted root cannot be renamed".into(), + )); + } + if to.as_str() == "/" { + return Err(VfsError::PermissionDenied( + "a path cannot be renamed onto the mounted root".into(), + )); + } + if to.as_str().starts_with(&format!("{}/", from.as_str())) { + return Err(VfsError::InvalidPath(format!( + "cannot rename {from} into its own descendant {to}" + ))); + } + let host_from = self.resolve(*from)?; + let host_to = self.resolve(*to)?; + // Validation finishes before the rename syscall, so a failed + // rename changes nothing; the rename itself is atomic. + fs::symlink_metadata(&host_from).map_err(|err| map_io(from.as_str(), &err))?; + create_parent(&host_to, *to)?; + fs::rename(&host_from, &host_to).map_err(|err| map_io(from.as_str(), &err)) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.check_writable(*to)?; + let host_from = self.resolve(*from)?; + let host_to = self.resolve(*to)?; + if host_from.is_dir() { + return Err(VfsError::IsADirectory(from.to_string())); + } + let bytes = fs::read(&host_from).map_err(|err| map_io(from.as_str(), &err))?; + if host_to.is_dir() { + return Err(VfsError::IsADirectory(to.to_string())); + } + create_parent(&host_to, *to)?; + atomic_write(&host_to, &bytes) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::path::{Path, PathBuf}; + use std::sync::atomic::{AtomicU64, Ordering}; + + use super::{HostBackend, identity_to_virtual, map_io}; + use crate::error::VfsError; + use crate::path::{VfsPath, canonicalize}; + use crate::traits::{ExecId, Vfs, VfsAccess}; + use crate::types::FileType; + + fn path(s: &str) -> Result { + canonicalize(s) + } + + /// A unique temporary directory that removes itself on drop. + struct TempDir(PathBuf); + + impl TempDir { + fn new() -> Result { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let dir = std::env::temp_dir().join(format!( + "shared-vfs-host-test-{}-{}", + std::process::id(), + COUNTER.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&dir).map_err(|err| map_io("the temporary directory", &err))?; + Ok(TempDir(dir)) + } + + fn path(&self) -> &Path { + &self.0 + } + } + + impl Drop for TempDir { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.0); + } + } + + /// Acquires a session on a rooted backend over `dir`. + fn rooted_access(dir: &Path) -> Result, VfsError> { + let mut backend = HostBackend::rooted(dir)?; + backend.acquire(ExecId::vend()) + } + + /// Asserts no failure-atomic temp file survived under `dir`. + fn assert_no_temp_files_left(dir: &Path) -> Result<(), VfsError> { + for entry in fs::read_dir(dir).map_err(|err| map_io("listing the temp dir", &err))? { + let entry = entry.map_err(|err| map_io("listing the temp dir", &err))?; + assert!( + !entry.file_name().to_string_lossy().contains(".vfs-tmp-"), + "a temp file survived: {}", + entry.path().display() + ); + } + Ok(()) + } + + /// Creates a directory link, returning false when the host refuses. + /// Windows uses a junction (no privilege required, unlike + /// `symlink_dir`); Unix uses a plain symlink. + #[cfg(windows)] + fn make_dir_link(link: &Path, target: &Path) -> bool { + std::process::Command::new("cmd") + .arg("/c") + .arg("mklink") + .arg("/J") + .arg(link) + .arg(target) + .status() + .is_ok_and(|status| status.success()) + } + + /// Creates a directory link, returning false when the host refuses. + #[cfg(unix)] + fn make_dir_link(link: &Path, target: &Path) -> bool { + std::os::unix::fs::symlink(target, link).is_ok() + } + + #[test] + fn a_rooted_backend_round_trips_files_and_directories() -> Result<(), VfsError> { + let temp = TempDir::new()?; + let mut access = rooted_access(temp.path())?; + // Writes materialize ancestor directories, as the memory + // backend does: no mkdir is needed first. + access.write(&path("/a/b/f.txt")?, b"hello")?; + assert_eq!(access.read(&path("/a/b/f.txt")?)?, b"hello"); + access.append(&path("/a/b/f.txt")?, b" world")?; + assert_eq!(access.read(&path("/a/b/f.txt")?)?, b"hello world"); + // The seek override serves byte ranges without a whole read. + assert_eq!(access.read_range(&path("/a/b/f.txt")?, 6, 5)?, b"world"); + assert!(access.exists(&path("/a")?)?); + assert!(access.exists(&path("/a/b")?)?); + assert!(!access.exists(&path("/a/missing.txt")?)?); + let stat = access.stat(&path("/a/b/f.txt")?)?; + assert_eq!(stat.file_type, FileType::File); + assert_eq!(stat.size, 11); + // The host tracks modification times; honesty permits Some here. + assert!(stat.modified.is_some()); + let entries = access.list(&path("/a/b")?)?; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].name, "f.txt"); + assert_eq!(entries[0].stat.file_type, FileType::File); + assert!(entries[0].description.is_none()); + access.mkdir(&path("/a/c")?, false)?; + assert!(matches!( + access.mkdir(&path("/a/c")?, false), + Err(VfsError::AlreadyExists(_)) + )); + access.rename(&path("/a/b/f.txt")?, &path("/a/b/g.txt")?)?; + assert!(!access.exists(&path("/a/b/f.txt")?)?); + access.copy(&path("/a/b/g.txt")?, &path("/a/c/h.txt")?)?; + assert_eq!(access.read(&path("/a/c/h.txt")?)?, b"hello world"); + assert_eq!( + access.glob("/a/**/*.txt")?, + vec!["/a/b/g.txt".to_owned(), "/a/c/h.txt".to_owned()] + ); + // The mounted root itself cannot be removed. + assert!(matches!( + access.remove(&path("/")?, true), + Err(VfsError::PermissionDenied(_)) + )); + access.remove(&path("/a")?, true)?; + assert!(!access.exists(&path("/a")?)?); + Ok(()) + } + + #[test] + fn a_rooted_backend_rejects_links_that_escape_the_mount_root() -> Result<(), VfsError> { + let outside = TempDir::new()?; + fs::write(outside.path().join("secret.txt"), b"classified") + .map_err(|err| map_io("seeding the outside file", &err))?; + let root = TempDir::new()?; + if !make_dir_link(&root.path().join("link"), outside.path()) { + // The host refused the link (privileges); there is nothing + // to escape through, so the test vacuously passes. + return Ok(()); + } + let mut access = rooted_access(root.path())?; + assert!( + matches!( + access.read(&path("/link/secret.txt")?), + Err(VfsError::PermissionDenied(_)) + ), + "a read through the escaping link must be denied" + ); + assert!( + matches!( + access.write(&path("/link/new.txt")?, b"x"), + Err(VfsError::PermissionDenied(_)) + ), + "a write through the escaping link must be denied" + ); + assert_eq!( + fs::read(outside.path().join("secret.txt")) + .map_err(|err| map_io("reading the outside file", &err))?, + b"classified" + ); + assert!(!outside.path().join("new.txt").exists()); + Ok(()) + } + + #[test] + fn a_failed_write_leaves_the_destination_unchanged_and_no_temp_file_behind() + -> Result<(), VfsError> { + let temp = TempDir::new()?; + let mut access = rooted_access(temp.path())?; + // A write over an existing directory fails before the temp + // file is created; the directory survives. + access.mkdir(&path("/dir")?, false)?; + assert!(matches!( + access.write(&path("/dir")?, b"x"), + Err(VfsError::IsADirectory(_)) + )); + assert!(temp.path().join("dir").is_dir()); + // A write through a file ancestor fails; the file is unchanged. + access.write(&path("/f.txt")?, b"original")?; + assert!(access.write(&path("/f.txt/g.txt")?, b"x").is_err()); + assert_eq!(access.read(&path("/f.txt")?)?, b"original"); + assert_no_temp_files_left(temp.path())?; + Ok(()) + } + + #[test] + fn a_failed_copy_leaves_source_and_destination_unchanged() -> Result<(), VfsError> { + let temp = TempDir::new()?; + let mut access = rooted_access(temp.path())?; + access.write(&path("/dst.txt")?, b"old")?; + assert!(matches!( + access.copy(&path("/missing.txt")?, &path("/dst.txt")?), + Err(VfsError::NotFound(_)) + )); + assert_eq!(access.read(&path("/dst.txt")?)?, b"old"); + assert_no_temp_files_left(temp.path())?; + Ok(()) + } + + #[test] + fn a_failed_rename_leaves_source_and_destination_unchanged() -> Result<(), VfsError> { + let temp = TempDir::new()?; + let mut access = rooted_access(temp.path())?; + access.write(&path("/dst.txt")?, b"old")?; + assert!(matches!( + access.rename(&path("/missing.txt")?, &path("/dst.txt")?), + Err(VfsError::NotFound(_)) + )); + assert_eq!(access.read(&path("/dst.txt")?)?, b"old"); + // Renaming a directory into its own descendant is rejected. + access.mkdir(&path("/d")?, false)?; + assert!(matches!( + access.rename(&path("/d")?, &path("/d/inner")?), + Err(VfsError::InvalidPath(_)) + )); + assert!(access.exists(&path("/d")?)?); + assert_no_temp_files_left(temp.path())?; + Ok(()) + } + + #[test] + fn a_read_only_backend_rejects_every_mutation_but_serves_reads() -> Result<(), VfsError> { + let temp = TempDir::new()?; + fs::write(temp.path().join("keep.txt"), b"keep") + .map_err(|err| map_io("seeding the kept file", &err))?; + let mut backend = HostBackend::rooted(temp.path())?.with_read_only(true); + assert!(Vfs::read_only(&backend)); + let mut access = backend.acquire(ExecId::vend())?; + assert_eq!(access.read(&path("/keep.txt")?)?, b"keep"); + assert!(access.exists(&path("/keep.txt")?)?); + assert_eq!(access.stat(&path("/keep.txt")?)?.size, 4); + for result in [ + access.write(&path("/keep.txt")?, b"x"), + access.write(&path("/new.txt")?, b"x"), + access.append(&path("/keep.txt")?, b"x"), + access.remove(&path("/keep.txt")?, false), + access.mkdir(&path("/dir")?, false), + access.rename(&path("/keep.txt")?, &path("/moved.txt")?), + access.copy(&path("/keep.txt")?, &path("/copy.txt")?), + ] { + assert!( + matches!(result, Err(VfsError::PermissionDenied(_))), + "expected a read-only denial, got {result:?}" + ); + } + assert_eq!(access.read(&path("/keep.txt")?)?, b"keep"); + assert!(!temp.path().join("new.txt").exists()); + Ok(()) + } + + #[test] + fn an_identity_backend_maps_virtual_paths_directly_to_host_paths() -> Result<(), VfsError> { + let temp = TempDir::new()?; + let host_file = temp.path().join("identity.txt"); + let virtual_spelling = identity_to_virtual(&host_file); + let mut backend = HostBackend::identity(); + let mut access = backend.acquire(ExecId::vend())?; + access.write(&path(&virtual_spelling)?, b"direct")?; + assert_eq!( + fs::read(&host_file).map_err(|err| map_io("reading the host file", &err))?, + b"direct" + ); + assert_eq!(access.read(&path(&virtual_spelling)?)?, b"direct"); + assert!(access.exists(&path(&virtual_spelling)?)?); + Ok(()) + } + + #[test] + fn a_literal_glob_pattern_matches_the_file_itself() -> Result<(), VfsError> { + let temp = TempDir::new()?; + let mut access = rooted_access(temp.path())?; + access.write(&path("/d/a.txt")?, b"x")?; + // A wildcard-free pattern matches the literal path itself, at + // parity with the memory backend. + assert_eq!(access.glob("/d/a.txt")?, vec!["/d/a.txt".to_owned()]); + assert_eq!(access.glob("/d/missing.txt")?, Vec::::new()); + // A literal directory matches itself as well. + assert_eq!(access.glob("/d")?, vec!["/d".to_owned()]); + Ok(()) + } + + #[test] + #[cfg(unix)] + fn exists_surfaces_backend_failures_as_errors() -> Result<(), VfsError> { + let temp = TempDir::new()?; + let mut access = rooted_access(temp.path())?; + access.write(&path("/f.txt")?, b"x")?; + // A lookup through a file ancestor fails with ENOTDIR, not + // ENOENT: an indeterminate path must not report as absent. + assert!(matches!( + access.exists(&path("/f.txt/g.txt")?), + Err(VfsError::NotADirectory(_)) + )); + assert!(!access.exists(&path("/missing.txt")?)?); + Ok(()) + } + + #[test] + fn the_rooted_constructor_requires_an_existing_directory() -> Result<(), VfsError> { + let temp = TempDir::new()?; + fs::write(temp.path().join("f.txt"), b"x") + .map_err(|err| map_io("seeding the file", &err))?; + assert!(matches!( + HostBackend::rooted(temp.path().join("f.txt")), + Err(VfsError::NotADirectory(_)) + )); + assert!(matches!( + HostBackend::rooted(temp.path().join("missing")), + Err(VfsError::NotFound(_)) + )); + Ok(()) + } +} diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs index c1a8714e2..9ec946bc4 100644 --- a/crates/shared-vfs/src/lib.rs +++ b/crates/shared-vfs/src/lib.rs @@ -8,6 +8,7 @@ mod error; mod glob; mod handle; +mod host; mod memory; mod path; mod router; @@ -16,6 +17,7 @@ mod types; pub use error::VfsError; pub use handle::{Access, VfsRef}; +pub use host::HostBackend; pub use memory::MemoryBackend; pub use path::{VfsPath, VfsPathBuf}; pub use router::VfsRefBuilder; diff --git a/crates/shared-vfs/src/memory.rs b/crates/shared-vfs/src/memory.rs index b564a5018..443db5ab5 100644 --- a/crates/shared-vfs/src/memory.rs +++ b/crates/shared-vfs/src/memory.rs @@ -400,7 +400,8 @@ impl VfsAccess for MemoryAccess { .collect(); for key in moved_files { if let Some(bytes) = tree.files.remove(&key) { - tree.files.insert(format!("{dest}{}", &key[source.len()..]), bytes); + tree.files + .insert(format!("{dest}{}", &key[source.len()..]), bytes); } } for key in moved_dirs { @@ -566,11 +567,7 @@ mod tests { #[test] fn remove_with_recursive_deletes_the_whole_subtree() -> Result<(), VfsError> { - let mut access = seeded(&[ - ("/d/a.txt", "a"), - ("/d/sub/b.txt", "b"), - ("/keep.txt", "k"), - ])?; + let mut access = seeded(&[("/d/a.txt", "a"), ("/d/sub/b.txt", "b"), ("/keep.txt", "k")])?; access.remove(&path("/d")?, true)?; assert!(!access.exists(&path("/d")?)?); assert!(!access.exists(&path("/d/sub")?)?); diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index bd1b023e1..87e5c939c 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -614,7 +614,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 6: host backend, stage 1 thin +### Step 6: host backend, stage 1 thin [completed] - Component: shared-vfs backends - Implement `HostBackend::identity()` (virtual path is the host path) and `HostBackend::rooted(dir)` (chroot-style) in shared-vfs over direct std::fs: lexical plus canonicalize containment for rooted, failure-atomic writes (sibling temp file plus rename; a failed write, copy, or rename leaves source, destination, and accounting unchanged), and the read_only flag rejecting all mutations. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index a801d9c33..7de525f02 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -101,3 +101,7 @@ - Decision: writes materialize ancestor directories instead of requiring mkdir (MemStore flat-map semantics carried over) | Falsifier: the Store facade or a host backend needs POSIX ENOENT-on-missing-parent behavior. - Decision: glob results include directories, not just files | Falsifier: a caller (engine adapter, grep default) misbehaves when directories match. - Decision: backend named `MemoryBackend` (plan pins no name; parallels `HostBackend`) | Falsifier: a later plan step or review pins a different name. +- Step 6: host backend, stage 1 thin - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p shared-vfs` - pass (100/100 tests). Review: 2 Important, both closed (`exists` mapped all errors to Ok(false), now NotFound-only; literal glob patterns never matched because walk_root did not resolve wildcard-free patterns to their parent). Verification fix: fmt drift in memory.rs. + - Decision: `rooted()` returns `Result` (canonicalization can fail) rather than panicking, matching the RealFs oracle's `open` | Falsifier: the contract's mount example shows `HostBackend::rooted("C:/work")` used without unwrap. + - Decision: read-only is set via `with_read_only(bool)` builder, not a `read_only()` setter, to avoid shadowing the `Vfs::read_only` trait method | Falsifier: a later step needing mid-run flips of the flag (the contract assigns mode flips to Policy, not the backend flag). + - Decision: containment canonicalizes the nearest existing ancestor and re-appends the missing tail; dangling-symlink resolution is documented as deferred to stage 2 | Falsifier: a write through a dangling symlink inside the root escaping containment. From 0df2a7e663e48da97029edc014bb3e0a84b516f5 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 19:27:08 -0700 Subject: [PATCH 07/26] Add promptforge-vfs policy crate with mode gate Introduce a new crate carrying the promptforge-specific virtual filesystem policy: the run-scoped store mount layout, a stock handle constructor, and the editor mode gate. The gate refuses mutations according to the current mode while reads always flow, and the mode lives in a shared cell the UI flips mid-run so the next operation observes the change without executor involvement. The crate depends only on the shared machinery crate, keeping policy apart from generic mechanism. - `STORE_MOUNT`: fixes the run-scoped store mount path as a public constant so hosts and callers never hardcode it. - `empty()`: builds a router with a fresh memory backend at the store mount, empty of content but not of mounts, so callers can seed before the run and extract after. - `ModePolicy`: Ask answers every mutation with an approval verdict naming the rule that fired, Plan permits mutations only to paths ending in `.md` case-sensitively, Agent allows everything, and reads are never gated. - `ModeHandle`: the UI's cloneable half of the shared mode cell; setting it takes effect on the very next operation, and one-way versus reversible control is just who still holds the handle. - `is_mutation` and `is_markdown`: the gate's two classification helpers, both pure; the markdown rule is deliberately case-sensitive because virtual paths are POSIX-strict. Design: new value-object @ crates/promptforge-vfs/src/lib.rs::Mode boundary: pub Design: new shared-mutable-state @ crates/promptforge-vfs/src/lib.rs::ModePolicy boundary: pub Design: new shared-mutable-state @ crates/promptforge-vfs/src/lib.rs::ModeHandle boundary: pub Design: new pure-function @ crates/promptforge-vfs/src/lib.rs::is_mutation deps: Op Design: new pure-function @ crates/promptforge-vfs/src/lib.rs::is_markdown deps: VfsPath Plan: vibe/2026-09-11-3-vfs-foundation.md --- Cargo.lock | 7 + Cargo.toml | 1 + crates/promptforge-vfs/Cargo.toml | 15 ++ crates/promptforge-vfs/src/lib.rs | 230 ++++++++++++++++++++++++++++ vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 4 + 6 files changed, 258 insertions(+), 1 deletion(-) create mode 100644 crates/promptforge-vfs/Cargo.toml create mode 100644 crates/promptforge-vfs/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 8a0c98c35..6dbcd6ed4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4941,6 +4941,13 @@ dependencies = [ "thiserror 2.0.19", ] +[[package]] +name = "promptforge-vfs" +version = "0.3.0" +dependencies = [ + "shared-vfs", +] + [[package]] name = "promptforge-web-search" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 27fa738c9..420e7176a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ gateway-logging = { path = "crates/gateway-logging", version = "0.3.0" } shared-loopback = { path = "crates/shared-loopback", version = "0.3.0" } shared-protocol = { path = "crates/shared-protocol", version = "0.3.0" } shared-sidecar = { path = "crates/shared-sidecar", version = "0.3.0" } +shared-vfs = { path = "crates/shared-vfs", version = "0.3.0" } gateway-routing = { path = "crates/gateway-routing", version = "0.3.0" } promptforge-lua = { path = "crates/promptforge-lua", version = "0.3.0" } promptforge-model-client = { path = "crates/promptforge-model-client", version = "0.3.0" } diff --git a/crates/promptforge-vfs/Cargo.toml b/crates/promptforge-vfs/Cargo.toml new file mode 100644 index 000000000..57b950a73 --- /dev/null +++ b/crates/promptforge-vfs/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "promptforge-vfs" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +description = "PromptForge virtual filesystem policy: the /_promptforge mount layout, the empty() stock handle, and the mode policy" + +[dependencies] +shared-vfs.workspace = true + +[lints] +workspace = true diff --git a/crates/promptforge-vfs/src/lib.rs b/crates/promptforge-vfs/src/lib.rs new file mode 100644 index 000000000..6d96c135b --- /dev/null +++ b/crates/promptforge-vfs/src/lib.rs @@ -0,0 +1,230 @@ +//! PromptForge policy over the shared virtual filesystem machinery. +//! +//! This crate carries promptforge policy, never generic machinery: the +//! `/_promptforge` mount layout, the [`empty`] stock handle, and +//! [`ModePolicy`], the editor mode gate. Generic machinery (traits, +//! claims, routing, backends) lives in `shared-vfs` below. + +use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; + +use shared_vfs::{MemoryBackend, Op, Policy, Verdict, VfsPath, VfsRef}; + +/// The mount prefix of the run-scoped store. Hosts seed before `run()` +/// and extract after through this mount; callers never hardcode the +/// path - [`empty`] installs it and the Store facade scopes to it. +pub const STORE_MOUNT: &str = "/_promptforge/store"; + +/// The stock handle: a router with a fresh memory backend at +/// [`STORE_MOUNT`]. Empty of content, not of mounts, so callers can +/// seed before `run()` and extract after. +#[must_use] +pub fn empty() -> VfsRef { + VfsRef::builder() + .mount(STORE_MOUNT, MemoryBackend::new()) + .build() +} + +/// The editor mode: what the model may mutate right now. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Mode { + /// Every mutation is refused pending user approval; reads flow. + Ask, + /// Mutations are allowed only to markdown paths; reads flow. + Plan, + /// Every operation is allowed. + Agent, +} + +/// The mode gate: one policy per handle, consulted on every operation +/// before the claims check. Modes gate mutations, never reads. The +/// current mode lives behind a shared `Arc`: the UI holds the +/// [`ModeHandle`] and flips modes mid-run, and the next operation sees +/// it - no executor involvement. One-way vs reversible is just who +/// still holds the handle. +#[derive(Debug)] +pub struct ModePolicy { + mode: Arc>, +} + +/// The UI's half of the mode gate. Clones share the one cell. +#[derive(Debug, Clone)] +pub struct ModeHandle { + mode: Arc>, +} + +impl ModePolicy { + /// Returns a policy starting in `mode`. + #[must_use] + pub fn new(mode: Mode) -> ModePolicy { + ModePolicy { + mode: Arc::new(Mutex::new(mode)), + } + } + + /// Returns the UI's half: flipping it mid-run takes effect on the + /// next operation. + #[must_use] + pub fn handle(&self) -> ModeHandle { + ModeHandle { + mode: Arc::clone(&self.mode), + } + } +} + +impl ModeHandle { + /// Sets the current mode. + pub fn set(&self, mode: Mode) { + *lock(&self.mode) = mode; + } + + /// Returns the current mode. + #[must_use] + pub fn mode(&self) -> Mode { + *lock(&self.mode) + } +} + +/// Poison-safe lock on the shared mode cell. +fn lock(mode: &Arc>) -> MutexGuard<'_, Mode> { + mode.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// Whether the operation mutates storage. Modes gate mutations, never +/// reads. +fn is_mutation(op: Op) -> bool { + matches!( + op, + Op::Write + | Op::Append + | Op::Delete + | Op::Rename + | Op::Mkdir + | Op::Copy + | Op::Symlink + | Op::Chmod + ) +} + +/// Plan mode's markdown rule: a `.md` suffix, case-sensitively - the +/// virtual namespace is POSIX-shaped and strict, so `.MD` is not +/// markdown and the case-insensitive suggestion does not apply. +#[expect( + clippy::case_sensitive_file_extension_comparisons, + reason = "virtual paths are POSIX-strict; case-insensitive extension matching is a host-OS notion" +)] +fn is_markdown(path: VfsPath) -> bool { + path.as_str().ends_with(".md") +} + +impl Policy for ModePolicy { + fn check(&self, op: Op, path: &VfsPath) -> Verdict { + if !is_mutation(op) { + return Verdict::Allow; + } + match *lock(&self.mode) { + Mode::Agent => Verdict::Allow, + Mode::Ask => Verdict::Ask(format!( + "{op:?} on {path} needs user approval: the Ask mode refuses all mutations" + )), + // Copy is checked per path, source and destination alike, + // so Plan refuses to even read a non-markdown source. The + // policy cannot tell the two apart; conservative refusal + // is the safe side. + Mode::Plan if is_markdown(*path) => Verdict::Allow, + Mode::Plan => Verdict::Deny(format!( + "{op:?} on {path} is refused: the Plan mode allows mutations only to markdown paths" + )), + } + } +} + +#[cfg(test)] +mod tests { + use shared_vfs::{VfsError, VfsRef}; + + use super::{Mode, ModePolicy, STORE_MOUNT, empty}; + + #[test] + fn empty_carries_the_store_mount() -> Result<(), VfsError> { + let vfs = empty(); + let access = vfs.acquire(); + let path = format!("{STORE_MOUNT}/paper.md"); + access.write(&path, b"# draft")?; + assert_eq!(access.read(&path)?, b"# draft"); + // Empty of content, not of mounts: the mount exists and serves. + assert!(access.exists(&path)?); + Ok(()) + } + + #[test] + fn empty_serves_nothing_outside_the_store_mount() { + let vfs = empty(); + let access = vfs.acquire(); + assert!(matches!( + access.read("/elsewhere.txt"), + Err(VfsError::NotFound(_)) + )); + } + + #[test] + fn a_mode_flip_through_the_shared_handle_is_visible_on_the_next_operation() + -> Result<(), VfsError> { + let policy = ModePolicy::new(Mode::Ask); + let handle = policy.handle(); + let vfs = VfsRef::with_policy(empty(), policy); + let access = vfs.acquire(); + let path = format!("{STORE_MOUNT}/notes.md"); + match access.write(&path, b"x") { + Err(VfsError::PermissionDenied(reason)) => { + assert!( + reason.contains("Ask"), + "names the rule that fired: {reason}" + ); + } + other => panic!("expected an approval denial, got {other:?}"), + } + // The UI flips the mode mid-run through the shared handle; the + // very next operation sees it. + handle.set(Mode::Agent); + access.write(&path, b"x")?; + assert_eq!(access.read(&path)?, b"x"); + Ok(()) + } + + #[test] + fn plan_mode_allows_mutations_only_to_markdown_paths() -> Result<(), VfsError> { + let policy = ModePolicy::new(Mode::Plan); + let vfs = VfsRef::with_policy(empty(), policy); + let access = vfs.acquire(); + let markdown = format!("{STORE_MOUNT}/notes.md"); + let binary = format!("{STORE_MOUNT}/data.bin"); + access.write(&markdown, b"# ok")?; + match access.write(&binary, b"x") { + Err(VfsError::PermissionDenied(reason)) => { + assert!( + reason.contains("Plan"), + "names the rule that fired: {reason}" + ); + } + other => panic!("expected a denial, got {other:?}"), + } + // The denied write never partially applied. + assert!(!access.exists(&binary)?); + Ok(()) + } + + #[test] + fn modes_gate_mutations_never_reads() -> Result<(), VfsError> { + let policy = ModePolicy::new(Mode::Agent); + let handle = policy.handle(); + let vfs = VfsRef::with_policy(empty(), policy); + let path = format!("{STORE_MOUNT}/paper.md"); + vfs.acquire().write(&path, b"text")?; + // Even in Ask, the strictest mode, reads flow. + handle.set(Mode::Ask); + let access = vfs.acquire(); + assert_eq!(access.read(&path)?, b"text"); + assert!(access.exists(&path)?); + Ok(()) + } +} diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index 87e5c939c..b224ebf8b 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -624,7 +624,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 7: promptforge-vfs policy crate +### Step 7: promptforge-vfs policy crate [completed] - Component: promptforge-vfs - Create `crates/promptforge-vfs` (depends on shared-vfs only): the `/_promptforge/store` mount layout, the `empty()` stock constructor (a router with a fresh memory backend at the store mount; empty of content, not of mounts), and `ModePolicy` (Ask denies all mutations, Plan allows mutations only to markdown paths, Agent allows all; modes gate mutations, never reads) behind a UI-flippable shared Arc so a mode change mid-run takes effect on the next operation. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index 7de525f02..e4ef180da 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -105,3 +105,7 @@ - Decision: `rooted()` returns `Result` (canonicalization can fail) rather than panicking, matching the RealFs oracle's `open` | Falsifier: the contract's mount example shows `HostBackend::rooted("C:/work")` used without unwrap. - Decision: read-only is set via `with_read_only(bool)` builder, not a `read_only()` setter, to avoid shadowing the `Vfs::read_only` trait method | Falsifier: a later step needing mid-run flips of the flag (the contract assigns mode flips to Policy, not the backend flag). - Decision: containment canonicalizes the nearest existing ancestor and re-appends the missing tail; dangling-symlink resolution is documented as deferred to stage 2 | Falsifier: a write through a dangling symlink inside the root escaping containment. +- Step 7: promptforge-vfs policy crate - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p promptforge-vfs` - pass (5/5 tests). Review: clean (with component-base drift check). Verification fix: fmt normalization of two test assertions. + - Decision: Ask mode returns `Verdict::Ask` (not `Deny`) for mutations; the capability layer maps both to `PermissionDenied`, and the Ask string is the approval-dialog text per the contract | Falsifier: a later integration step asserts `Verdict::Deny` from `ModePolicy::check` in Ask mode. + - Decision: Plan's markdown rule is a case-sensitive `.md` suffix (clippy's case-insensitive suggestion explicitly `#[expect]`-overridden), matching the POSIX-strict virtual namespace | Falsifier: a host needs `.MD`/`.markdown` admitted in Plan mode. + - Decision: `ModeHandle` is a separate cloneable UI half (`policy.handle()`), per "one-way vs reversible is just who still holds the mode handle" | Falsifier: a caller needs to flip modes holding only the `ModePolicy`. From c45272145c671bc1a0b4cab545103f5cfc600ee4 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 19:39:11 -0700 Subject: [PATCH 08/26] Replace the store trait and backends with a VFS facade The store crate inverts from a public trait with pluggable backends into a concrete facade that borrows the caller's VFS capability, so every store operation is attributed to a live identity and participates in the claims model. Write-race detection moves out of a fanout-specific registry into identity claims, which also cover appends and any concurrency shape rather than only fanout arms. The caller-facing contract is preserved: validated logical paths, verbatim reads, anchor edits, numbered line ranges, idempotent delete, and the two-wildcard glob grammar, all reported in the existing error vocabulary. Poison handling changes from surfacing a backend error to poison-safe recovery, so a panicking operation cannot wedge the store. - `Store` is now a concrete facade borrowing `&Access`: it validates logical paths, joins them onto the store mount prefix, delegates to the capability, and maps VFS failures back onto the store vocabulary. - `StoreExt` supplies the `vfs.store(&access)` call shape as a prelude-exported extension trait, because the facade type lives above the VFS crates and cannot be an inherent method on `VfsRef`. - `map_vfs` is the total error mapping: `VfsError::NotFound` becomes `StoreError::NotFound`, `VfsError::Conflict` becomes `StoreError::WriteRace`, and everything else becomes an opaque backend failure. - `Store::str_replace` implements the anchor rules in the facade as read, count, write: an empty anchor is refused, zero or many matches error, exactly one occurrence is replaced. - `Store::delete` stays idempotent by mapping `VfsError::NotFound` to `Ok`. - `Store::glob` delegates matching to the backend under a mount-scoped pattern, refuses backslash patterns itself because the router canonicalizes separators first, and stat-filters matches to files only. - `StoreRef`, `WriteScope`, `write_scoped`, and `next_write_token` are gone; a second live identity's conflicting write or append now surfaces as `StoreError::WriteRace` through the claims model. - `MemStore`, `FileStore`, and the crate-private glob matcher are deleted with their modules; backends and matching move to the VFS crates this crate now imports. - `StorePoisoned` is gone; the poisoned-lock-as-backend-error contract is replaced by recovery, covered by a panic-injection backend test. Design: strategy -> facade @ crates/promptforge-store/src/lib.rs::Store boundary: pub Design: removes shared-mutable-state @ crates/promptforge-store/src/lib.rs::StoreRef Design: new surface-growth @ crates/promptforge-store/src/lib.rs::StoreExt boundary: pub Design: new pure-function @ crates/promptforge-store/src/lib.rs::map_vfs deps: VfsError,str Design: new pure-function @ crates/promptforge-store/src/lib.rs::full deps: str Plan: vibe/2026-09-11-3-vfs-foundation.md --- Cargo.lock | 3 +- Cargo.toml | 1 + crates/promptforge-store/Cargo.toml | 7 +- crates/promptforge-store/src/error.rs | 44 +-- crates/promptforge-store/src/file.rs | 348 ------------------ crates/promptforge-store/src/glob.rs | 229 ------------ crates/promptforge-store/src/lib.rs | 508 ++++++++++++-------------- crates/promptforge-store/src/mem.rs | 319 ---------------- crates/promptforge-store/src/tests.rs | 431 +++++++++++++--------- vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 6 + 11 files changed, 529 insertions(+), 1369 deletions(-) delete mode 100644 crates/promptforge-store/src/file.rs delete mode 100644 crates/promptforge-store/src/glob.rs delete mode 100644 crates/promptforge-store/src/mem.rs diff --git a/Cargo.lock b/Cargo.lock index 6dbcd6ed4..ed1235f12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4908,7 +4908,8 @@ dependencies = [ name = "promptforge-store" version = "0.3.0" dependencies = [ - "tempfile", + "promptforge-vfs", + "shared-vfs", "thiserror 2.0.19", ] diff --git a/Cargo.toml b/Cargo.toml index 420e7176a..31c68c4d3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -38,6 +38,7 @@ promptforge-parser = { path = "crates/promptforge-parser", version = "0.3.0" } shared-progress = { path = "crates/shared-progress", version = "0.3.0" } gateway-stt = { path = "crates/gateway-stt", version = "0.3.0" } promptforge-store = { path = "crates/promptforge-store", version = "0.3.0" } +promptforge-vfs = { path = "crates/promptforge-vfs", version = "0.3.0" } promptforge-webfetch = { path = "crates/promptforge-webfetch", version = "0.3.0" } promptforge-tool-picker = { path = "crates/promptforge-tool-picker", version = "0.3.0" } promptforge-tools = { path = "crates/promptforge-tools", version = "0.3.0" } diff --git a/crates/promptforge-store/Cargo.toml b/crates/promptforge-store/Cargo.toml index 7b4112c89..3336df1c2 100644 --- a/crates/promptforge-store/Cargo.toml +++ b/crates/promptforge-store/Cargo.toml @@ -6,17 +6,16 @@ license.workspace = true repository.workspace = true publish = false -description = "PromptForge run-scoped virtual filesystem: the Store backend contract, in-memory and file backends, and the shared StoreRef handle" +description = "PromptForge run-scoped virtual files: the Store facade over the shared VFS, exposed as vfs.store(&access)" readme = "README.md" keywords = ["prompt", "llm", "virtual-filesystem", "storage"] categories = ["data-structures", "rust-patterns"] documentation = "https://cppalliance.github.io/promptforge/" [dependencies] +promptforge-vfs.workspace = true +shared-vfs.workspace = true thiserror.workspace = true -[dev-dependencies] -tempfile.workspace = true - [lints] workspace = true diff --git a/crates/promptforge-store/src/error.rs b/crates/promptforge-store/src/error.rs index e7d1747a8..5db7f37e7 100644 --- a/crates/promptforge-store/src/error.rs +++ b/crates/promptforge-store/src/error.rs @@ -1,14 +1,5 @@ //! Store error types and their stable classifiers. -/// The backend lock was poisoned by a panicked holder. -/// -/// Kept private and surfaced only as an opaque [`StoreError::Backend`] source -/// (STORE-004), so a poisoned lock is a visible backend failure rather than a -/// silent recovery of state this handle cannot vouch for. -#[derive(Debug, thiserror::Error)] -#[error("store backend lock was poisoned by a panicked holder")] -pub(crate) struct StorePoisoned; - /// Why a logical store path was rejected before any backend saw it. /// /// `StoreRef` validates every caller-supplied path into one canonical form @@ -77,7 +68,7 @@ pub enum StoreErrorKind { InvalidPattern, /// A caller-supplied line range failed validation. InvalidRange, - /// Two arms of one fanout wrote the same path. + /// Two live identities touched the same path. WriteRace, /// The backend itself failed. Backend, @@ -166,9 +157,10 @@ pub enum StoreError { reason: &'static str, }, - /// Two arms of one fanout wrote the same path: a write-write race. The - /// losing write never reached the backend. - #[error("write-write race on {path}: another arm of the same fanout already wrote it")] + /// Two live identities touched the same path: a write-write race + /// detected by the claims model. The losing write never reached the + /// backend. + #[error("write-write race on {path}: another live identity holds a claim on it")] #[non_exhaustive] WriteRace { /// The logical path both arms wrote. @@ -190,9 +182,12 @@ impl StoreError { /// /// # Examples /// ``` - /// use promptforge_store::{StoreErrorKind, StoreRef}; + /// use promptforge_store::{StoreErrorKind, StoreExt}; /// - /// let err = StoreRef::memory().read("missing.txt").unwrap_err(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); + /// let err = store.read("missing.txt").unwrap_err(); /// assert_eq!(err.kind(), StoreErrorKind::NotFound); /// ``` #[must_use] @@ -215,9 +210,12 @@ impl StoreError { /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let err = StoreRef::memory().read("missing.txt").unwrap_err(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); + /// let err = store.read("missing.txt").unwrap_err(); /// assert!(err.is_not_found()); /// ``` #[must_use] @@ -229,9 +227,12 @@ impl StoreError { /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let err = StoreRef::memory().read("missing.txt").unwrap_err(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); + /// let err = store.read("missing.txt").unwrap_err(); /// assert_eq!(err.path(), Some("missing.txt")); /// ``` #[must_use] @@ -250,8 +251,9 @@ impl StoreError { /// Wraps a backend's own error as an opaque [`StoreError::Backend`] source. /// - /// A downstream [`Store`](crate::Store) implementation uses this so its concrete error - /// type never leaks through this crate's public API. + /// The facade uses this for VFS failures without a store-vocabulary + /// home, so the concrete error type never leaks through this crate's + /// public API. /// /// # Examples /// ``` diff --git a/crates/promptforge-store/src/file.rs b/crates/promptforge-store/src/file.rs deleted file mode 100644 index 43ca80f4c..000000000 --- a/crates/promptforge-store/src/file.rs +++ /dev/null @@ -1,348 +0,0 @@ -//! A filesystem-backed [`Store`] that persists virtual files as real files -//! under a caller-provided root directory. - -use std::fs::{self, OpenOptions}; -use std::io::Write as _; -use std::path::{Path, PathBuf}; - -use super::StoreError; -use super::glob::{compile_glob, matches_tokens}; -use super::mem::Store; - -/// A [`Store`] backend that persists virtual files as real files on disk. -/// -/// Each logical path maps to a relative filesystem path under the configured -/// root directory. Operations are synchronous; the runtime serializes access -/// behind a [`StoreRef`](super::StoreRef) mutex. -/// -/// # Examples -/// ```no_run -/// use promptforge_store::FileStore; -/// -/// let store = FileStore::new("/tmp/my-run")?; -/// # Ok::<(), std::io::Error>(()) -/// ``` -#[derive(Debug)] -pub struct FileStore { - root: PathBuf, -} - -impl FileStore { - /// Creates a new file-backed store rooted at `root`. - /// - /// Creates the root directory (and parents) if it does not exist. - /// - /// # Errors - /// Returns [`std::io::Error`] if the directory cannot be created. - pub fn new(root: impl Into) -> std::io::Result { - let root = root.into(); - fs::create_dir_all(&root)?; - Ok(FileStore { root }) - } - - /// Maps a logical store path to its filesystem location under `root`. - /// - /// Returns `Err` if the path would escape `root` (defense in depth; - /// `StorePath::parse` already rejects traversal at the `StoreRef` layer). - fn resolve(&self, path: &str) -> Result { - if path.is_empty() { - return Err(StoreError::NotFound { - path: path.to_owned(), - }); - } - if path.contains('\\') { - return Err(StoreError::backend(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "backslash rejected by confinement check", - ))); - } - let mut resolved = self.root.clone(); - for component in path.split('/') { - if component.is_empty() || component == "." || component == ".." { - return Err(StoreError::backend(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - "path component rejected by confinement check", - ))); - } - resolved.push(component); - } - Ok(resolved) - } - - /// Ensures the parent directory of `fs_path` exists. - fn ensure_parent(fs_path: &Path) -> Result<(), StoreError> { - if let Some(parent) = fs_path.parent() { - fs::create_dir_all(parent).map_err(StoreError::backend)?; - } - Ok(()) - } - - /// Recursively collects all file paths under `dir`, expressed as logical - /// store paths relative to `root`. - fn walk_files(root: &Path, dir: &Path, out: &mut Vec) -> Result<(), StoreError> { - let entries = match fs::read_dir(dir) { - Ok(entries) => entries, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(()), - Err(e) => return Err(StoreError::backend(e)), - }; - for entry in entries { - let entry = entry.map_err(StoreError::backend)?; - let ft = entry.file_type().map_err(StoreError::backend)?; - let entry_path = entry.path(); - if ft.is_dir() { - Self::walk_files(root, &entry_path, out)?; - } else if ft.is_file() - && let Some(logical) = Self::to_logical_path(root, &entry_path) - { - out.push(logical); - } - } - Ok(()) - } - - /// Converts a filesystem path back to a logical `/`-separated store path - /// relative to `root`. Returns `None` if the path cannot be represented - /// (non-UTF-8 components). - fn to_logical_path(root: &Path, fs_path: &Path) -> Option { - let relative = fs_path.strip_prefix(root).ok()?; - let mut segments = Vec::new(); - for component in relative.components() { - match component { - std::path::Component::Normal(s) => { - segments.push(s.to_str()?); - } - _ => return None, - } - } - if segments.is_empty() { - return None; - } - Some(segments.join("/")) - } -} - -impl Store for FileStore { - fn write(&mut self, path: &str, contents: &str) -> Result<(), StoreError> { - let fs_path = self.resolve(path)?; - Self::ensure_parent(&fs_path)?; - fs::write(&fs_path, contents).map_err(StoreError::backend) - } - - fn append(&mut self, path: &str, contents: &str) -> Result<(), StoreError> { - let fs_path = self.resolve(path)?; - Self::ensure_parent(&fs_path)?; - let mut file = OpenOptions::new() - .create(true) - .append(true) - .open(&fs_path) - .map_err(StoreError::backend)?; - file.write_all(contents.as_bytes()) - .map_err(StoreError::backend) - } - - fn read(&self, path: &str) -> Result { - let fs_path = self.resolve(path)?; - match fs::read_to_string(&fs_path) { - Ok(contents) => Ok(contents), - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Err(StoreError::NotFound { - path: path.to_owned(), - }), - Err(e) => Err(StoreError::backend(e)), - } - } - - fn str_replace(&mut self, path: &str, old: &str, new: &str) -> Result<(), StoreError> { - let fs_path = self.resolve(path)?; - let contents = match fs::read_to_string(&fs_path) { - Ok(c) => c, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Err(StoreError::NotFound { - path: path.to_owned(), - }); - } - Err(e) => return Err(StoreError::backend(e)), - }; - let count = contents.matches(old).count(); - match count { - 0 => Err(StoreError::AnchorNotFound { - path: path.to_owned(), - anchor: old.to_owned(), - }), - 1 => { - let replaced = contents.replacen(old, new, 1); - fs::write(&fs_path, replaced).map_err(StoreError::backend) - } - _ => Err(StoreError::AnchorAmbiguous { - path: path.to_owned(), - anchor: old.to_owned(), - count, - }), - } - } - - fn delete(&mut self, path: &str) -> Result<(), StoreError> { - let fs_path = self.resolve(path)?; - match fs::remove_file(&fs_path) { - Ok(()) => Ok(()), - // Idempotent delete: an absent file is already in the end state. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), - Err(e) => Err(StoreError::backend(e)), - } - } - - fn glob(&self, pattern: &str) -> Result, StoreError> { - let mut paths = Vec::new(); - Self::walk_files(&self.root, &self.root, &mut paths)?; - let tokens = compile_glob(pattern.as_bytes()); - paths.retain(|p| matches_tokens(&tokens, p.as_bytes())); - paths.sort(); - Ok(paths) - } - - fn exists(&self, path: &str) -> Result { - let fs_path = self.resolve(path)?; - Ok(fs_path.is_file()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn temp_store() -> (TempDir, FileStore) { - let dir = TempDir::new().expect("create temp dir"); - let store = FileStore::new(dir.path()).expect("create FileStore"); - (dir, store) - } - - #[test] - fn write_and_read() { - let (_dir, mut store) = temp_store(); - store.write("hello.txt", "world").expect("write"); - assert_eq!(store.read("hello.txt").expect("read"), "world"); - } - - #[test] - fn write_creates_parent_dirs() { - let (_dir, mut store) = temp_store(); - store.write("a/b/c.txt", "deep").expect("write nested"); - assert_eq!(store.read("a/b/c.txt").expect("read"), "deep"); - } - - #[test] - fn append_creates_and_accumulates() { - let (_dir, mut store) = temp_store(); - store.append("log.txt", "first\n").expect("append 1"); - store.append("log.txt", "second").expect("append 2"); - assert_eq!(store.read("log.txt").expect("read"), "first\nsecond"); - } - - #[test] - fn read_missing_file_returns_not_found() { - let (_dir, store) = temp_store(); - let err = store.read("ghost.txt").expect_err("should error"); - assert!(err.is_not_found()); - } - - #[test] - fn str_replace_single_occurrence() { - let (_dir, mut store) = temp_store(); - store.write("a.txt", "the quick brown fox").expect("write"); - store - .str_replace("a.txt", "quick", "slow") - .expect("replace"); - assert_eq!(store.read("a.txt").expect("read"), "the slow brown fox"); - } - - #[test] - fn str_replace_missing_anchor() { - let (_dir, mut store) = temp_store(); - store.write("a.txt", "hello").expect("write"); - let err = store - .str_replace("a.txt", "missing", "x") - .expect_err("should error"); - assert!(matches!(err, StoreError::AnchorNotFound { .. })); - } - - #[test] - fn str_replace_ambiguous_anchor() { - let (_dir, mut store) = temp_store(); - store.write("a.txt", "aa").expect("write"); - let err = store - .str_replace("a.txt", "a", "b") - .expect_err("should error"); - assert!(matches!(err, StoreError::AnchorAmbiguous { .. })); - } - - #[test] - fn delete_removes_file() { - let (_dir, mut store) = temp_store(); - store.write("temp.txt", "data").expect("write"); - store.delete("temp.txt").expect("delete"); - assert!(!store.exists("temp.txt").expect("exists")); - } - - #[test] - fn delete_missing_is_silent() { - let (_dir, mut store) = temp_store(); - store.delete("ghost.txt").expect("delete is idempotent"); - } - - #[test] - fn glob_matches_patterns() { - let (_dir, mut store) = temp_store(); - store.write("src/a.rs", "").expect("write"); - store.write("src/b.rs", "").expect("write"); - store.write("src/deep/c.rs", "").expect("write"); - store.write("notes.md", "").expect("write"); - - let matched = store.glob("src/*.rs").expect("glob"); - assert_eq!(matched, vec!["src/a.rs", "src/b.rs"]); - - let all_rs = store.glob("**/*.rs").expect("glob"); - assert_eq!(all_rs, vec!["src/a.rs", "src/b.rs", "src/deep/c.rs"]); - - let everything = store.glob("**").expect("glob"); - assert_eq!(everything.len(), 4); - } - - #[test] - fn exists_reports_correctly() { - let (_dir, mut store) = temp_store(); - assert!(!store.exists("a.txt").expect("exists before")); - store.write("a.txt", "hi").expect("write"); - assert!(store.exists("a.txt").expect("exists after")); - } - - #[test] - fn confinement_rejects_traversal() { - let (_dir, store) = temp_store(); - let err = store.read("../escape.txt").expect_err("should reject"); - assert!(matches!(err, StoreError::Backend { .. })); - } - - #[test] - fn confinement_rejects_dot_segment() { - let (_dir, store) = temp_store(); - let err = store.read("a/./b.txt").expect_err("should reject"); - assert!(matches!(err, StoreError::Backend { .. })); - } - - #[test] - fn confinement_rejects_backslash_segments() { - let (_dir, store) = temp_store(); - for path in ["..\\escape.txt", "a\\b.txt"] { - let error = store.read(path).expect_err("backslash path should reject"); - assert!(matches!(error, StoreError::Backend { .. })); - } - } - - #[test] - fn constructor_creates_missing_root() { - let dir = TempDir::new().expect("create temp dir"); - let nested = dir.path().join("deep").join("nested"); - let _store = FileStore::new(&nested).expect("should create dirs"); - assert!(nested.is_dir()); - } -} diff --git a/crates/promptforge-store/src/glob.rs b/crates/promptforge-store/src/glob.rs deleted file mode 100644 index 14e529927..000000000 --- a/crates/promptforge-store/src/glob.rs +++ /dev/null @@ -1,229 +0,0 @@ -//! Glob-pattern grammar and a bounded, recursion-free matcher. -//! -//! The store validates a caller-supplied glob against one grammar -//! (STORE-006) and then matches stored paths with a bounded iterative -//! dynamic program (STORE-005), so a hostile pattern cannot drive exponential -//! time or blow the stack. - -/// The largest glob pattern, in bytes, the store will attempt to match. -/// -/// The recursion-free matcher is linear, but an unbounded pattern is still a -/// cheap denial-of-service lever, so an over-long pattern is refused outright. -pub(crate) const MAX_GLOB_PATTERN_BYTES: usize = 1024; - -/// One unit of a validated glob pattern. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum GlobToken { - /// A literal byte that must match exactly. - Literal(u8), - /// `*`: zero or more bytes, none of them `/` (stays within one segment). - Star, - /// `**` not bounded by a `/`: zero or more bytes of any kind. - DoubleStar, - /// `**/`: zero or more whole path segments (empty, or any run ending `/`). - DoubleStarSlash, -} - -/// Validates the glob grammar (STORE-006), rejecting unsupported forms. -/// -/// The grammar is: literal bytes, `*` (within a segment), and `**` occupying a -/// whole segment (`**`, `**/...`, `.../**`, `.../**/...`). There is no escape -/// syntax, so a backslash is unsupported and runs of three or more `*` are -/// rejected rather than silently reinterpreted. -pub(crate) fn validate_glob_grammar(pattern: &str) -> Result<(), &'static str> { - if pattern.contains('\\') { - return Err("pattern does not support backslash escapes"); - } - let bytes = pattern.as_bytes(); - let mut index = 0; - while index < bytes.len() { - if bytes[index] != b'*' { - index += 1; - continue; - } - let run_start = index; - while index < bytes.len() && bytes[index] == b'*' { - index += 1; - } - let run_len = index - run_start; - if run_len > 2 { - return Err("more than two consecutive '*' are not supported"); - } - if run_len == 2 { - let before_ok = run_start == 0 || bytes[run_start - 1] == b'/'; - let after_ok = index == bytes.len() || bytes[index] == b'/'; - if !before_ok || !after_ok { - return Err("'**' must occupy a whole path segment"); - } - } - } - Ok(()) -} - -/// Compiles an already-grammar-validated pattern into reusable [`GlobToken`]s. -/// -/// STORE-020: callers that match one pattern against many paths compile once -/// and reuse the tokens via [`matches_tokens`], so the per-path tokenization -/// cost is not repeated for every key while a store lock is held. -pub(crate) fn compile_glob(pattern: &[u8]) -> Vec { - debug_assert!( - std::str::from_utf8(pattern).is_ok_and(|text| validate_glob_grammar(text).is_ok()), - "compile_glob requires validated grammar" - ); - tokenize_glob(pattern) -} - -/// Tokenizes an already-grammar-validated pattern into [`GlobToken`]s. -fn tokenize_glob(pattern: &[u8]) -> Vec { - let mut tokens = Vec::with_capacity(pattern.len()); - let mut index = 0; - while index < pattern.len() { - match pattern[index] { - b'*' => { - if pattern.get(index + 1) == Some(&b'*') { - if pattern.get(index + 2) == Some(&b'/') { - tokens.push(GlobToken::DoubleStarSlash); - index += 3; - } else { - tokens.push(GlobToken::DoubleStar); - index += 2; - } - } else { - tokens.push(GlobToken::Star); - index += 1; - } - } - byte => { - tokens.push(GlobToken::Literal(byte)); - index += 1; - } - } - } - tokens -} - -/// Matches `text` against a glob `pattern` where `*` stays within a segment and -/// `**` spans `/`. -/// -/// STORE-005: bounded iterative dynamic programming over reachable text -/// positions, with no recursion and no suffix backtracking, so a hostile -/// pattern cannot drive exponential time or blow the stack. Runs in -/// `O(tokens * text_len)` time and `O(text_len)` space. -/// Compiles and matches in one shot. Retained as the parity reference for -/// [`matches_tokens`]; the store path uses [`compile_glob`] + [`matches_tokens`] -/// so it is only needed in tests. -#[cfg(test)] -pub(crate) fn glob_match(pattern: &[u8], text: &[u8]) -> bool { - matches_tokens(&tokenize_glob(pattern), text) -} - -/// Matches `text` against already-compiled glob `tokens` (see [`compile_glob`]). -/// -/// Split from `glob_match` so one compiled pattern can be reused across many -/// paths without re-tokenizing per path (STORE-020). Same bounded iterative -/// dynamic program: `O(tokens * text_len)` time, `O(text_len)` space. -pub(crate) fn matches_tokens(tokens: &[GlobToken], text: &[u8]) -> bool { - let len = text.len(); - // `reachable[j]` is true when some prefix of the pattern consumed so far - // matches exactly `text[..j]`. - let mut reachable = vec![false; len + 1]; - reachable[0] = true; - let mut next = vec![false; len + 1]; - - for &token in tokens { - next.fill(false); - match token { - GlobToken::Literal(byte) => { - for j in 0..len { - if reachable[j] && text[j] == byte { - next[j + 1] = true; - } - } - } - GlobToken::Star => { - // Zero or more non-`/` bytes: sweep left to right, carrying - // reachability forward across each non-slash byte. - let mut carry = false; - for j in 0..=len { - let here = reachable[j] || carry; - next[j] = here; - carry = here && j < len && text[j] != b'/'; - } - } - GlobToken::DoubleStar => { - // Zero or more bytes of any kind: once any position is - // reachable, every later position is too. - let mut seen = false; - for j in 0..=len { - seen |= reachable[j]; - next[j] = seen; - } - } - GlobToken::DoubleStarSlash => { - // Empty, or any run ending in `/` (whole path segments). - let mut seen = false; - for j in 0..=len { - let mut here = reachable[j]; - if seen && j > 0 && text[j - 1] == b'/' { - here = true; - } - next[j] = here; - if reachable[j] { - seen = true; - } - } - } - } - std::mem::swap(&mut reachable, &mut next); - } - reachable[len] -} - -#[cfg(test)] -mod tests { - use super::{compile_glob, glob_match, matches_tokens}; - - #[test] - fn compiled_and_one_shot_match_expected_results_across_many_paths() { - // STORE-020: a pattern compiled once and reused across many keys must - // produce the pinned result for every key. The one-shot matcher is - // checked against the same independent expectations. - let paths = [ - "a.txt", - "src/a.rs", - "src/b.rs", - "src/deep/c.rs", - "src/deep/deeper/d.rs", - "notes/today.md", - ]; - for (pattern, expected) in [ - ("*.txt", [true, false, false, false, false, false]), - ("src/*.rs", [false, true, true, false, false, false]), - ("src/**/*.rs", [false, true, true, true, true, false]), - ("**/*.md", [false, false, false, false, false, true]), - ("src/**", [false, true, true, true, true, false]), - ("no*match", [false, false, false, false, false, false]), - ] { - let tokens = compile_glob(pattern.as_bytes()); - for (path, expected) in paths.iter().zip(expected) { - assert_eq!( - matches_tokens(&tokens, path.as_bytes()), - expected, - "compiled pattern {pattern:?} produced the wrong result for {path:?}", - ); - assert_eq!( - glob_match(pattern.as_bytes(), path.as_bytes()), - expected, - "one-shot pattern {pattern:?} produced the wrong result for {path:?}", - ); - } - } - } - - #[cfg(debug_assertions)] - #[test] - #[should_panic(expected = "compile_glob requires validated grammar")] - fn compile_glob_rejects_unvalidated_direct_input_in_debug_builds() { - let _ = compile_glob(b"bad/***/pattern"); - } -} diff --git a/crates/promptforge-store/src/lib.rs b/crates/promptforge-store/src/lib.rs index e7cad09be..bdbb58d3a 100644 --- a/crates/promptforge-store/src/lib.rs +++ b/crates/promptforge-store/src/lib.rs @@ -1,290 +1,140 @@ //! Run-scoped virtual files, shared by Lua and the model. //! //! A prompt run keeps its bulk state in virtual files addressed by logical -//! string paths. [`Store`] is the backend contract, [`MemStore`] is an -//! in-memory backend, and [`StoreRef`] is the cheaply cloneable, thread-safe -//! handle the runtime hands to both the Lua VM and (later) the model's file -//! tools. [`StoreRef::read`] returns verbatim contents for trusted handoff, -//! [`StoreRef::read_range`] slices a 1-based inclusive line range out of the -//! same verbatim contents, and [`StoreRef::read_range_numbered`] numbers such -//! a slice absolutely (with no bounds it numbers the whole file from 1). For -//! model-facing re-injection the caller wraps a verbatim read in an -//! untrusted guard envelope (the `untrusted` Lua global). -//! Edits are anchor-based ([`Store::str_replace`]) rather than offset-based, -//! the shape that works for a model. +//! string paths. [`Store`] is a concrete facade over a prefix-scoped +//! [`Access`] capability from the shared VFS: the [`StoreExt`] extension +//! trait (re-exported in [`prelude`]) gives every `VfsRef` the +//! `vfs.store(&access)` call shape, binding the facade to the caller's +//! identity so its operations participate in the claims model - a second +//! live identity's conflicting write surfaces as [`StoreError::WriteRace`]. //! -//! This crate wires no execution; it defines the store and its backends only. - -use std::collections::HashMap; -use std::fmt; -use std::fmt::Write as _; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, MutexGuard}; +//! The facade keeps the store's caller-facing contract: logical paths +//! validated before dispatch, verbatim reads, anchor-based edits +//! ([`Store::str_replace`]), 1-based inclusive line ranges with optional +//! absolute numbering, idempotent deletes, and the `*`/`**` glob grammar. +//! The `Store` trait, `MemStore`, `FileStore`, and the `WriteScope` +//! registry are gone: backends live in `shared-vfs`, the mount layout in +//! `promptforge-vfs`, and race detection in the claims model. +//! +//! This crate wires no execution; it defines the facade and its error +//! vocabulary only. mod error; -mod file; -mod glob; -mod mem; mod path; -use error::StorePoisoned; +use std::fmt::Write as _; + +use promptforge_vfs::STORE_MOUNT; +use shared_vfs::{Access, FileType, VfsError, VfsRef}; + pub use error::{PathReason, StoreError, StoreErrorKind}; -pub use file::FileStore; -use glob::{MAX_GLOB_PATTERN_BYTES, compile_glob, matches_tokens, validate_glob_grammar}; -pub use mem::{MemStore, Store}; use path::StorePath; -/// The provenance of one fanout arm's scoped write: which fanout, and which -/// arm within it. +/// The largest glob pattern, in bytes, the facade will attempt to match. /// -/// Vended per fanout by [`StoreRef::next_write_token`] and paired with the -/// arm's 1-based index, so the write registry can tell "another arm of the -/// same fanout" (a write-write race) from "the same arm again" or "a later -/// fanout" (both legal). -/// -/// `#[doc(hidden)]`: a cross-crate seam for the executor's fanout machinery -/// in `promptforge-core`, not host API. -#[doc(hidden)] -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct WriteScope { - token: u64, - arm: usize, -} - -impl WriteScope { - /// Pairs one fanout's token with the arm's 1-based index within it. - #[must_use] - pub fn new(token: u64, arm: usize) -> WriteScope { - WriteScope { token, arm } - } -} +/// The matcher is linear, but an unbounded pattern is still a cheap +/// denial-of-service lever, so an over-long pattern is refused outright. +pub(crate) const MAX_GLOB_PATTERN_BYTES: usize = 1024; -/// A cheaply cloneable, thread-safe handle to a run's virtual files. +/// A concrete facade over one identity's prefix-scoped VFS capability. /// -/// The handle wraps `Arc>>`: the `Mutex` supplies -/// the synchronization around a `Send` (not necessarily `Sync`) backend -/// (STORE-008), so cloning shares one backend and the store can be held by both -/// the synchronous Lua VM and an asynchronous tool whose `call` crosses an -/// `.await`. The inherent -/// methods mirror [`Store`], each taking the lock, delegating, and -/// releasing it before returning; no lock is ever held across an await, and the -/// operations are synchronous in any case. -/// -/// Beside the backend lock the handle keeps a write registry mapping each -/// path to the `WriteScope` that last wrote it: a fanout arm's scoped -/// write (`StoreRef::write_scoped`) to a path already written by a -/// different arm of the same fanout fails with [`StoreError::WriteRace`]. -/// Plain [`StoreRef::write`] (walk sections), `append`, and reads never -/// touch the registry. +/// A `Store` borrows the caller's [`Access`]: every operation is +/// attributed to the caller's identity and participates in its claims, so +/// a conflicting operation by a second live identity surfaces as +/// [`StoreError::WriteRace`]. Paths are logical (relative to the store +/// mount); the facade validates them, joins them onto the mount prefix, +/// and maps the VFS error vocabulary back onto [`StoreError`]. /// /// # Examples /// ``` -/// use promptforge_store::StoreRef; +/// use promptforge_store::StoreExt; /// -/// let store = StoreRef::memory(); -/// let clone = store.clone(); +/// let vfs = promptforge_vfs::empty(); +/// let access = vfs.acquire(); +/// let store = vfs.store(&access); /// store.write("shared.txt", "state")?; -/// assert_eq!(clone.read("shared.txt")?, "state"); +/// assert_eq!(store.read("shared.txt")?, "state"); /// # Ok::<(), promptforge_store::StoreError>(()) /// ``` -#[derive(Clone)] -#[non_exhaustive] -pub struct StoreRef { - inner: Arc>>, - writers: Arc>>, - write_tokens: Arc, -} - -impl fmt::Debug for StoreRef { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("StoreRef").finish_non_exhaustive() - } +#[derive(Debug, Clone)] +pub struct Store<'a> { + access: &'a Access, } -impl StoreRef { - /// Wraps `backend` in a shareable handle. - /// - /// # Examples - /// ``` - /// use promptforge_store::{MemStore, StoreRef}; - /// - /// let store = StoreRef::new(Box::new(MemStore::new())); - /// # let _ = store; - /// ``` - #[must_use] - pub fn new(backend: Box) -> StoreRef { - StoreRef { - inner: Arc::new(Mutex::new(backend)), - writers: Arc::new(Mutex::new(HashMap::new())), - write_tokens: Arc::new(AtomicU64::new(0)), - } - } - - /// Builds a handle over a [`MemStore`] pre-populated with the given files. - /// - /// Each path is validated at construction time. See - /// [`MemStore::with_files`] for details. - /// - /// # Errors - /// Returns [`StoreError::InvalidPath`] if any path fails validation. - /// - /// # Examples - /// ``` - /// use promptforge_store::StoreRef; - /// - /// let store = StoreRef::with_files([ - /// ("data.txt".to_owned(), "contents".to_owned()), - /// ])?; - /// assert_eq!(store.read("data.txt")?, "contents"); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - pub fn with_files( - files: impl IntoIterator, - ) -> Result { - Ok(StoreRef::new(Box::new(MemStore::with_files(files)?))) - } - - /// Builds a handle over a fresh in-memory [`MemStore`] backend. - /// - /// # Examples - /// ``` - /// use promptforge_store::StoreRef; - /// - /// let store = StoreRef::memory(); - /// # let _ = store; - /// ``` - #[must_use] - pub fn memory() -> StoreRef { - StoreRef::new(Box::new(MemStore::new())) - } - - /// Locks the shared backend, or reports it unavailable if a prior holder - /// panicked while mutating it. - /// - /// STORE-004: the backend behind this handle is an arbitrary [`Store`] trait - /// object, not a known-consistent [`MemStore`]. A panic mid-mutation can - /// leave a filesystem/network backend in a half-applied state, so we do NOT - /// blindly `PoisonError::into_inner` and hand back state we cannot vouch - /// for. Absent an explicit backend recovery contract, a poisoned lock is a - /// backend failure the caller must see. - fn lock(&self) -> Result>, StoreError> { - self.inner - .lock() - .map_err(|_| StoreError::backend(StorePoisoned)) - } - - /// Creates or overwrites the file at `path`. See [`Store::write`]. +impl Store<'_> { + /// Creates or overwrites the file at `path`. /// /// # Errors - /// Propagates any [`StoreError`] from the backend. + /// Returns [`StoreError::InvalidPath`] if `path` fails validation, + /// [`StoreError::WriteRace`] if another live identity holds a claim on + /// `path`, or [`StoreError::Backend`] if the backend fails. /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.write("a.txt", "hi")?; /// # Ok::<(), promptforge_store::StoreError>(()) /// ``` pub fn write(&self, path: &str, contents: &str) -> Result<(), StoreError> { let path = StorePath::parse(path)?; - self.lock()?.write(path.as_str(), contents) - } - - /// Vends a fresh token identifying one fanout's write scope. - /// - /// Each fanout takes one token and every arm pairs it with its own index - /// via [`WriteScope::new`]; tokens are unique per [`StoreRef`], so two - /// fanouts (sequential or nested) never share a scope. - /// - /// `#[doc(hidden)]`: a cross-crate seam for the executor's fanout - /// machinery in `promptforge-core`, not host API. - #[doc(hidden)] - #[must_use] - pub fn next_write_token(&self) -> u64 { - self.write_tokens.fetch_add(1, Ordering::Relaxed) - } - - /// Creates or overwrites the file at `path` on behalf of one fanout arm, - /// recording the arm's [`WriteScope`] as the path's writer. - /// - /// The registry is checked and updated atomically before the backend is - /// touched: a path already written by a different arm of the SAME fanout - /// is a write-write race and fails without reaching the backend; the same - /// arm rewriting its own path succeeds, and a write carrying a different - /// fanout's token overwrites the record, so sequential fanouts stay - /// legal. - /// - /// `#[doc(hidden)]`: a cross-crate seam for the executor's fanout - /// machinery in `promptforge-core`, not host API. - /// - /// # Errors - /// Returns [`StoreError::WriteRace`] on a same-fanout write-write race, - /// [`StoreError::InvalidPath`] if `path` fails validation, or any - /// [`StoreError`] the backend reports. - #[doc(hidden)] - pub fn write_scoped( - &self, - path: &str, - contents: &str, - scope: WriteScope, - ) -> Result<(), StoreError> { - let path = StorePath::parse(path)?; - { - let mut writers = self - .writers - .lock() - .map_err(|_| StoreError::backend(StorePoisoned))?; - if let Some(&prior) = writers.get(path.as_str()) - && prior.token == scope.token - && prior.arm != scope.arm - { - return Err(StoreError::WriteRace { - path: path.as_str().to_owned(), - }); - } - writers.insert(path.as_str().to_owned(), scope); - } - self.lock()?.write(path.as_str(), contents) + self.access + .write(&full(path.as_str()), contents.as_bytes()) + .map_err(|err| map_vfs(err, path.as_str())) } - /// Appends to the file at `path`, creating it if absent. See - /// [`Store::append`]. + /// Appends to the file at `path`, creating it if absent. /// /// # Errors - /// Propagates any [`StoreError`] from the backend. + /// Returns [`StoreError::InvalidPath`] if `path` fails validation, + /// [`StoreError::WriteRace`] if another live identity holds a claim on + /// `path`, or [`StoreError::Backend`] if the backend fails. /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.append("a.txt", "hi")?; /// # Ok::<(), promptforge_store::StoreError>(()) /// ``` pub fn append(&self, path: &str, contents: &str) -> Result<(), StoreError> { let path = StorePath::parse(path)?; - self.lock()?.append(path.as_str(), contents) + self.access + .append(&full(path.as_str()), contents.as_bytes()) + .map_err(|err| map_vfs(err, path.as_str())) } /// Reads the file at `path` exactly as stored, with no line numbering. - /// See [`Store::read`]. + /// + /// This is the accessor for verbatim handoff, clean dumps, and trusted + /// re-injection. Numbered output for navigation is derived from a read + /// at this layer. /// /// # Errors /// Returns [`StoreError::NotFound`] if no file exists at `path`. /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.write("a.txt", "hi\n")?; /// assert_eq!(store.read("a.txt")?, "hi\n"); /// # Ok::<(), promptforge_store::StoreError>(()) /// ``` pub fn read(&self, path: &str) -> Result { let path = StorePath::parse(path)?; - self.lock()?.read(path.as_str()) + self.access + .read_string(&full(path.as_str())) + .map_err(|err| map_vfs(err, path.as_str())) } /// Reads lines `start..=end` of the file at `path`, 1-based and @@ -302,9 +152,11 @@ impl StoreRef { /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.write("a.txt", "one\ntwo\nthree\n")?; /// assert_eq!(store.read_range("a.txt", 2, None)?, "two\nthree"); /// assert_eq!(store.read_range("a.txt", 2, Some(99))?, "two\nthree"); @@ -327,7 +179,7 @@ impl StoreRef { /// the largest emitted number, followed by `"| "`; lines are joined with /// `"\n"` and there is no trailing newline. With `start` of 1 and no /// `end` the whole file is numbered from 1. Bounds are evaluated exactly - /// as in [`StoreRef::read_range`]: a `start` below 1 is an error; a + /// as in [`Store::read_range`]: a `start` below 1 is an error; a /// `start` past the last line reads as the empty string; an omitted /// `end` means the last line, and a given `end` clamps down to it; an /// `end` before `start` at that point is an error. @@ -339,9 +191,11 @@ impl StoreRef { /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.write("a.txt", "one\ntwo\nthree\n")?; /// assert_eq!( /// store.read_range_numbered("a.txt", 1, None)?, @@ -369,7 +223,7 @@ impl StoreRef { render: impl FnOnce(&[&str], usize) -> String, ) -> Result { let path = StorePath::parse(path)?; - let contents = self.lock()?.read(path.as_str())?; + let contents = self.read(path.as_str())?; let lines: Vec<&str> = contents.lines().collect(); let Some((start, end)) = resolve_line_range(path.as_str(), lines.len(), start, end)? else { return Ok(String::new()); @@ -377,19 +231,25 @@ impl StoreRef { Ok(render(&lines[start - 1..end], start)) } - /// Replaces the unique occurrence of `old` with `new`. See - /// [`Store::str_replace`]. + /// Replaces the unique occurrence of `old` with `new`. + /// + /// The edit is anchor-based: `old` must occur exactly once. Zero matches + /// and more-than-one match are both refused, so an edit never lands on + /// an arbitrary match. /// /// # Errors - /// Returns [`StoreError::InvalidAnchor`] when `old` is empty. Otherwise, - /// returns [`StoreError::NotFound`], [`StoreError::AnchorNotFound`], or - /// [`StoreError::AnchorAmbiguous`] per [`Store::str_replace`]. + /// Returns [`StoreError::InvalidAnchor`] when `old` is empty, + /// [`StoreError::NotFound`] if no file exists at `path`, + /// [`StoreError::AnchorNotFound`] if `old` does not occur, or + /// [`StoreError::AnchorAmbiguous`] if `old` occurs more than once. /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.write("a.txt", "one two")?; /// store.str_replace("a.txt", "two", "three")?; /// assert_eq!(store.read("a.txt")?, "one three"); @@ -400,28 +260,46 @@ impl StoreRef { if old.is_empty() { // STORE-007: an empty anchor is a malformed edit request, not an // anchor that merely failed to match; refuse it with a dedicated - // invalid-anchor condition before any backend search. + // invalid-anchor condition before any search. return Err(StoreError::InvalidAnchor { path: path.as_str().to_owned(), reason: "anchor must not be empty", }); } - self.lock()?.str_replace(path.as_str(), old, new) + let contents = self.read(path.as_str())?; + let count = contents.matches(old).count(); + match count { + 0 => Err(StoreError::AnchorNotFound { + path: path.as_str().to_owned(), + anchor: old.to_owned(), + }), + 1 => { + let replaced = contents.replacen(old, new, 1); + self.write(path.as_str(), &replaced) + } + count => Err(StoreError::AnchorAmbiguous { + path: path.as_str().to_owned(), + anchor: old.to_owned(), + count, + }), + } } - /// Removes the file at `path`. See [`Store::delete`]. + /// Removes the file at `path`. /// /// Delete is idempotent: a missing file is not an error. /// /// # Errors - /// Returns [`StoreError::InvalidPath`] if `path` fails validation, or any - /// [`StoreError`] the backend reports. + /// Returns [`StoreError::InvalidPath`] if `path` fails validation, or + /// [`StoreError::Backend`] if the backend fails. /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.write("a.txt", "hi")?; /// store.delete("a.txt")?; /// store.delete("a.txt")?; // already gone; still Ok @@ -429,19 +307,35 @@ impl StoreRef { /// ``` pub fn delete(&self, path: &str) -> Result<(), StoreError> { let path = StorePath::parse(path)?; - self.lock()?.delete(path.as_str()) + match self.access.remove(&full(path.as_str()), false) { + // Idempotent: an absent path is already in the post-delete state. + Ok(()) | Err(VfsError::NotFound(_)) => Ok(()), + Err(other) => Err(map_vfs(other, path.as_str())), + } } - /// Returns stored paths matching `pattern`, sorted. See [`Store::glob`]. + /// Returns stored paths matching `pattern`, sorted. + /// + /// Two wildcards are supported: `*` matches any run of characters within + /// a single path segment (it never crosses `/`), and `**` matches any + /// run of characters including `/`. All other characters match + /// literally. Only files are listed: the store vocabulary has no + /// directories. /// /// # Errors - /// Propagates any [`StoreError`] from the backend. + /// Returns [`StoreError::InvalidPattern`] if `pattern` is empty, + /// over-long, control-bearing, or grammar-invalid, + /// [`StoreError::WriteRace`] if another live identity holds a writer + /// claim on a matched path, or [`StoreError::Backend`] if the backend + /// fails. /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// store.write("a.txt", "")?; /// store.write("b.md", "")?; /// assert_eq!(store.glob("*.txt")?, vec!["a.txt"]); @@ -466,38 +360,59 @@ impl StoreRef { reason: "pattern contains a control character".to_owned(), }); } - if let Err(reason) = validate_glob_grammar(pattern) { + // The grammar has no escape syntax, and the router canonicalizes + // patterns (separators included) before the backend can reject + // them, so the backslash refusal must happen here. + if pattern.contains('\\') { return Err(StoreError::InvalidPattern { pattern: pattern.to_owned(), - reason: reason.to_owned(), + reason: "pattern does not support backslash escapes".to_owned(), }); } - // AUDIT-MUTEX-EXPENSIVE: snapshot every stored path under a brief lock - // (a trivial `**` full enumeration), then release the lock and run the - // arbitrary-pattern matcher on the owned snapshot. The O(tokens * path) - // matching never executes while the shared backend mutex is held; only - // the backend's own enumeration does. - let snapshot = self.lock()?.glob("**")?; - let tokens = compile_glob(pattern.as_bytes()); - Ok(snapshot - .into_iter() - .filter(|path| matches_tokens(&tokens, path.as_bytes())) - .collect()) + // One glob implementation lives in shared-vfs; the facade scopes + // the pattern to the mount and maps a grammar rejection back onto + // the store vocabulary. + let scoped = format!("{STORE_MOUNT}/{pattern}"); + let matches = self.access.glob(&scoped).map_err(|err| match err { + VfsError::InvalidPath(reason) => StoreError::InvalidPattern { + pattern: pattern.to_owned(), + reason, + }, + other => map_vfs(other, pattern), + })?; + let prefix = format!("{STORE_MOUNT}/"); + let mut paths = Vec::new(); + for matched in matches { + // The VFS glob lists directories as well as files; the store + // vocabulary lists files only. + let logical = matched.strip_prefix(&prefix).unwrap_or(&matched); + let stat = self + .access + .stat(&matched) + .map_err(|err| map_vfs(err, logical))?; + if stat.file_type != FileType::File { + continue; + } + paths.push(logical.to_owned()); + } + Ok(paths) } - /// Returns whether a file exists at `path`. See [`Store::exists`]. + /// Returns whether a file exists at `path`. /// /// A confirmed absence is `Ok(false)`; a backend failure is `Err`. /// /// # Errors - /// Returns [`StoreError::InvalidPath`] if `path` fails validation, or any - /// [`StoreError`] the backend reports. + /// Returns [`StoreError::InvalidPath`] if `path` fails validation, or + /// [`StoreError::Backend`] if the backend fails. /// /// # Examples /// ``` - /// use promptforge_store::StoreRef; + /// use promptforge_store::StoreExt; /// - /// let store = StoreRef::memory(); + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); /// assert!(!store.exists("a.txt")?); /// store.write("a.txt", "hi")?; /// assert!(store.exists("a.txt")?); @@ -505,7 +420,64 @@ impl StoreRef { /// ``` pub fn exists(&self, path: &str) -> Result { let path = StorePath::parse(path)?; - self.lock()?.exists(path.as_str()) + self.access + .exists(&full(path.as_str())) + .map_err(|err| map_vfs(err, path.as_str())) + } +} + +/// The extension trait behind the `vfs.store(&access)` call shape. +/// +/// The [`Store`] facade type lives in this crate, above `promptforge-vfs` +/// and `shared-vfs` in the dependency stack, so the method cannot be +/// inherent on `VfsRef`; a prelude-exported extension trait preserves the +/// declared call shape without inverting the stack. +pub trait StoreExt { + /// Returns the store facade scoped to the stock store mount, bound to + /// `access`'s identity. + /// + /// # Examples + /// ``` + /// use promptforge_store::StoreExt; + /// + /// let vfs = promptforge_vfs::empty(); + /// let access = vfs.acquire(); + /// let store = vfs.store(&access); + /// store.write("seeded.txt", "input")?; + /// # Ok::<(), promptforge_store::StoreError>(()) + /// ``` + fn store<'a>(&self, access: &'a Access) -> Store<'a>; +} + +impl StoreExt for VfsRef { + fn store<'a>(&self, access: &'a Access) -> Store<'a> { + Store { access } + } +} + +/// The integrator prelude: the facade and its extension trait. +pub mod prelude { + pub use crate::{Store, StoreExt}; +} + +/// Joins a validated logical path onto the store mount prefix. +fn full(path: &str) -> String { + format!("{STORE_MOUNT}/{path}") +} + +/// Maps the VFS error vocabulary onto the store's, keeping the logical +/// path the caller supplied. A claim conflict is the write-write race the +/// claims model detects; everything without a store-vocabulary home is an +/// opaque backend failure. +fn map_vfs(err: VfsError, path: &str) -> StoreError { + match err { + VfsError::NotFound(_) => StoreError::NotFound { + path: path.to_owned(), + }, + VfsError::Conflict(_) => StoreError::WriteRace { + path: path.to_owned(), + }, + other => StoreError::backend(other), } } diff --git a/crates/promptforge-store/src/mem.rs b/crates/promptforge-store/src/mem.rs deleted file mode 100644 index 98753e658..000000000 --- a/crates/promptforge-store/src/mem.rs +++ /dev/null @@ -1,319 +0,0 @@ -//! The [`Store`] backend contract and its in-memory implementation. - -use std::collections::BTreeMap; - -use super::StoreError; -use super::glob::{compile_glob, matches_tokens}; -use super::path::StorePath; - -/// A backend for run-scoped virtual files addressed by logical string paths. -/// -/// All operations are synchronous. Implementors store text keyed by path; the -/// runtime shares one behind a [`StoreRef`](super::StoreRef) handle. Reads are -/// verbatim (see [`Store::read`]); numbered reads are derived at the -/// [`StoreRef`](super::StoreRef) layer. Edits are anchored (see -/// [`Store::str_replace`]). -/// -/// # Examples -/// ``` -/// use promptforge_store::{Store, MemStore}; -/// -/// let mut fs = MemStore::new(); -/// fs.write("greeting.txt", "hello")?; -/// assert_eq!(fs.read("greeting.txt")?, "hello"); -/// # Ok::<(), promptforge_store::StoreError>(()) -/// ``` -/// -/// The `Send` bound lets a backend cross a `spawn_blocking` boundary; `Sync` is -/// deliberately not required, since the runtime serializes access behind a -/// [`StoreRef`](super::StoreRef) mutex. -pub trait Store: Send { - /// Creates the file at `path`, or overwrites it if it already exists. - /// - /// # Errors - /// This operation does not fail for the in-memory backend, but the return - /// type is fallible so a filesystem-backed backend can report I/O errors. - /// - /// # Examples - /// ``` - /// use promptforge_store::{Store, MemStore}; - /// - /// let mut fs = MemStore::new(); - /// fs.write("a.txt", "one")?; - /// fs.write("a.txt", "two")?; - /// assert_eq!(fs.read("a.txt")?, "two"); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - fn write(&mut self, path: &str, contents: &str) -> Result<(), StoreError>; - - /// Appends `contents` to the file at `path`, creating it if it is absent. - /// - /// # Errors - /// This operation does not fail for the in-memory backend; the return type - /// stays fallible for filesystem-backed backends. - /// - /// # Examples - /// ``` - /// use promptforge_store::{Store, MemStore}; - /// - /// let mut fs = MemStore::new(); - /// fs.append("log.txt", "first\n")?; - /// fs.append("log.txt", "second")?; - /// assert_eq!(fs.read("log.txt")?, "first\nsecond"); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - fn append(&mut self, path: &str, contents: &str) -> Result<(), StoreError>; - - /// Returns the file's contents exactly as stored, with no line numbering. - /// - /// This is the accessor for verbatim handoff, clean dumps, and trusted - /// re-injection. Numbered output for navigation is derived from a read at - /// the [`StoreRef`](super::StoreRef) layer. - /// - /// # Errors - /// Returns [`StoreError::NotFound`] if no file exists at `path`. - /// - /// # Examples - /// ``` - /// use promptforge_store::{Store, MemStore}; - /// - /// let mut fs = MemStore::new(); - /// fs.write("poem.txt", "roses\nviolets\n")?; - /// assert_eq!(fs.read("poem.txt")?, "roses\nviolets\n"); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - fn read(&self, path: &str) -> Result; - - /// Replaces the single occurrence of `old` with `new` in the file at - /// `path`. - /// - /// The edit is anchor-based: `old` must occur exactly once. Zero matches - /// and more-than-one match are both refused, so an edit never lands on an - /// arbitrary match. - /// - /// # Errors - /// Returns [`StoreError::NotFound`] if no file exists at `path`, - /// [`StoreError::AnchorNotFound`] if `old` does not occur, or - /// [`StoreError::AnchorAmbiguous`] if `old` occurs more than once. - /// - /// # Examples - /// ``` - /// use promptforge_store::{Store, MemStore}; - /// - /// let mut fs = MemStore::new(); - /// fs.write("a.txt", "the quick brown fox")?; - /// fs.str_replace("a.txt", "quick", "slow")?; - /// assert_eq!(fs.read("a.txt")?, "the slow brown fox"); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - fn str_replace(&mut self, path: &str, old: &str, new: &str) -> Result<(), StoreError>; - - /// Removes the file at `path`. - /// - /// Delete is idempotent: removing a file that does not exist succeeds, so - /// a caller never has to check [`Store::exists`] before deleting. - /// - /// # Errors - /// This operation does not fail for the in-memory backend; the return type - /// stays fallible for filesystem-backed backends. - /// - /// # Examples - /// ``` - /// use promptforge_store::{Store, MemStore}; - /// - /// let mut fs = MemStore::new(); - /// fs.write("temp.txt", "scratch")?; - /// fs.delete("temp.txt")?; - /// assert!(fs.read("temp.txt").is_err()); - /// fs.delete("temp.txt")?; // already gone; still Ok - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - fn delete(&mut self, path: &str) -> Result<(), StoreError>; - - /// Returns every stored path matching `pattern`, in sorted order. - /// - /// Two wildcards are supported: `*` matches any run of characters within a - /// single path segment (it never crosses `/`), and `**` matches any run of - /// characters including `/`. All other characters match literally. - /// - /// # Errors - /// This operation does not fail for the in-memory backend; the return type - /// stays fallible for filesystem-backed backends. - /// - /// # Examples - /// ``` - /// use promptforge_store::{Store, MemStore}; - /// - /// let mut fs = MemStore::new(); - /// fs.write("src/a.rs", "")?; - /// fs.write("src/b.rs", "")?; - /// fs.write("src/deep/c.rs", "")?; - /// assert_eq!(fs.glob("src/*.rs")?, vec!["src/a.rs", "src/b.rs"]); - /// assert_eq!( - /// fs.glob("src/**/*.rs")?, - /// vec!["src/a.rs", "src/b.rs", "src/deep/c.rs"], - /// ); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - fn glob(&self, pattern: &str) -> Result, StoreError>; - - /// Returns whether a file exists at `path`. - /// - /// This is fallible so a backend distinguishes a confirmed absence - /// (`Ok(false)`) from an inability to answer (`Err`), rather than - /// collapsing a backend failure into "does not exist". - /// - /// # Errors - /// Returns a [`StoreError`] if the backend cannot determine existence. - /// - /// # Examples - /// ``` - /// use promptforge_store::{Store, MemStore}; - /// - /// let mut fs = MemStore::new(); - /// assert!(!fs.exists("a.txt")?); - /// fs.write("a.txt", "hi")?; - /// assert!(fs.exists("a.txt")?); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - fn exists(&self, path: &str) -> Result; -} - -/// An in-memory [`Store`] backend. -/// -/// Files live in a [`BTreeMap`] keyed by path, so listing and [`glob`] results -/// are ordered without a sort step. It holds no resources and drops with the -/// run. -/// -/// [`glob`]: Store::glob -/// -/// # Examples -/// ``` -/// use promptforge_store::{Store, MemStore}; -/// -/// let mut fs = MemStore::new(); -/// fs.write("notes.md", "todo")?; -/// assert_eq!(fs.glob("*.md")?, vec!["notes.md"]); -/// # Ok::<(), promptforge_store::StoreError>(()) -/// ``` -#[derive(Debug, Default, Clone)] -#[non_exhaustive] -pub struct MemStore { - files: BTreeMap, -} - -impl MemStore { - /// Creates an empty in-memory store. - /// - /// # Examples - /// ``` - /// use promptforge_store::MemStore; - /// - /// let fs = MemStore::new(); - /// # let _ = fs; - /// ``` - #[must_use] - pub fn new() -> MemStore { - MemStore::default() - } - - /// Creates a store pre-populated with the given files. - /// - /// Each path is validated through `StorePath::parse` at - /// construction time, so the store never holds a path unreachable through - /// the normal read/write API. - /// - /// # Errors - /// Returns [`StoreError::InvalidPath`] if any path fails validation. - /// - /// # Examples - /// ``` - /// use promptforge_store::{MemStore, Store}; - /// - /// let fs = MemStore::with_files([ - /// ("input.md".to_owned(), "# Hello".to_owned()), - /// ])?; - /// assert_eq!(fs.read("input.md")?, "# Hello"); - /// # Ok::<(), promptforge_store::StoreError>(()) - /// ``` - pub fn with_files( - files: impl IntoIterator, - ) -> Result { - let mut map = BTreeMap::new(); - for (path, contents) in files { - let validated = StorePath::parse(&path)?; - map.insert(validated.as_str().to_owned(), contents); - } - Ok(MemStore { files: map }) - } -} - -impl Store for MemStore { - fn write(&mut self, path: &str, contents: &str) -> Result<(), StoreError> { - self.files.insert(path.to_string(), contents.to_string()); - Ok(()) - } - - fn append(&mut self, path: &str, contents: &str) -> Result<(), StoreError> { - self.files - .entry(path.to_string()) - .or_default() - .push_str(contents); - Ok(()) - } - - fn read(&self, path: &str) -> Result { - self.files - .get(path) - .cloned() - .ok_or_else(|| StoreError::NotFound { - path: path.to_string(), - }) - } - - fn str_replace(&mut self, path: &str, old: &str, new: &str) -> Result<(), StoreError> { - let contents = self.files.get(path).ok_or_else(|| StoreError::NotFound { - path: path.to_string(), - })?; - let count = contents.matches(old).count(); - match count { - 0 => Err(StoreError::AnchorNotFound { - path: path.to_string(), - anchor: old.to_string(), - }), - 1 => { - let replaced = contents.replacen(old, new, 1); - self.files.insert(path.to_string(), replaced); - Ok(()) - } - count => Err(StoreError::AnchorAmbiguous { - path: path.to_string(), - anchor: old.to_string(), - count, - }), - } - } - - fn delete(&mut self, path: &str) -> Result<(), StoreError> { - // Idempotent: an absent path is already in the post-delete state. - self.files.remove(path); - Ok(()) - } - - fn glob(&self, pattern: &str) -> Result, StoreError> { - // STORE-020: compile the pattern once, then reuse the tokens across every - // key, so the per-key tokenization no longer repeats while the shared - // store mutex is held. Matching itself is bounded and non-backtracking. - let tokens = compile_glob(pattern.as_bytes()); - Ok(self - .files - .keys() - .filter(|key| matches_tokens(&tokens, key.as_bytes())) - .cloned() - .collect()) - } - - fn exists(&self, path: &str) -> Result { - Ok(self.files.contains_key(path)) - } -} diff --git a/crates/promptforge-store/src/tests.rs b/crates/promptforge-store/src/tests.rs index 19c5f1ffd..e848cc35b 100644 --- a/crates/promptforge-store/src/tests.rs +++ b/crates/promptforge-store/src/tests.rs @@ -1,61 +1,30 @@ -use super::*; +//! The parity suite: the store contract ported onto the VFS facade, +//! unmodified in intent. Two tests change shape with the mechanism under +//! test: write-race detection is now the claims model's (a second live +//! identity conflicts; the WriteScope registry is gone), and poison +//! handling is now poison-safe recovery (a panic mid-operation cannot +//! wedge the store) rather than a surfaced backend error. -/// A backend whose `glob` ignores the pattern and always returns every path. -/// -/// If `StoreRef::glob` delegated matching to the backend it would return all -/// paths unfiltered; the filtered results below prove `StoreRef` applies the -/// caller's pattern to the backend snapshot itself. -struct GlobSpyStore { - paths: Vec, -} +use promptforge_vfs::STORE_MOUNT; +use shared_vfs::{ + Access, Entry, ExecId, MemoryBackend, Stat, Vfs, VfsAccess, VfsError, VfsPath, VfsRef, +}; -impl Store for GlobSpyStore { - fn write(&mut self, _path: &str, _contents: &str) -> Result<(), StoreError> { - Ok(()) - } - fn append(&mut self, _path: &str, _contents: &str) -> Result<(), StoreError> { - Ok(()) - } - fn read(&self, _path: &str) -> Result { - Ok(String::new()) - } - fn str_replace(&mut self, _path: &str, _old: &str, _new: &str) -> Result<(), StoreError> { - Ok(()) - } - fn delete(&mut self, _path: &str) -> Result<(), StoreError> { - Ok(()) - } - fn glob(&self, _pattern: &str) -> Result, StoreError> { - Ok(self.paths.clone()) - } - fn exists(&self, _path: &str) -> Result { - Ok(false) - } -} +use super::path::MAX_STORE_PATH_BYTES; +use super::{MAX_GLOB_PATTERN_BYTES, PathReason, Store, StoreError, StoreErrorKind, StoreExt}; -#[test] -fn glob_filters_backend_snapshot_with_caller_pattern() { - let spy = GlobSpyStore { - paths: vec![ - "src/a.rs".to_owned(), - "src/b.md".to_owned(), - "src/deep/c.rs".to_owned(), - ], - }; - let store = StoreRef::new(Box::new(spy)); - // The caller's real pattern is applied by `StoreRef` to the backend result. - let matched = store.glob("src/*.rs").expect("glob"); - assert_eq!(matched, vec!["src/a.rs".to_owned()]); - let matched = store.glob("src/**/*.rs").expect("glob"); - assert_eq!( - matched, - vec!["src/a.rs".to_owned(), "src/deep/c.rs".to_owned()] - ); +/// A stock handle with one acquired identity: the fixture every +/// single-identity test starts from. +fn stock() -> (VfsRef, Access) { + let vfs = promptforge_vfs::empty(); + let access = vfs.acquire(); + (vfs, access) } #[test] fn write_then_read_numbered_numbers_lines() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "first\nsecond\nthird").expect("write"); assert_eq!( store @@ -67,7 +36,8 @@ fn write_then_read_numbered_numbers_lines() { #[test] fn read_numbered_pads_numbers_to_width() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); numbered_fixture(&store, "a.txt", 10); let numbered = store .read_range_numbered("a.txt", 1, None) @@ -78,7 +48,8 @@ fn read_numbered_pads_numbers_to_width() { #[test] fn read_returns_contents_verbatim() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "first\nsecond\n").expect("write"); assert_eq!(store.read("a.txt").expect("read"), "first\nsecond\n"); assert_eq!( @@ -91,55 +62,103 @@ fn read_returns_contents_verbatim() { #[test] fn read_missing_file_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); let err = store.read("absent.txt").expect_err("should fail"); assert!(matches!(err, StoreError::NotFound { .. })); } #[test] -fn scoped_writes_reject_a_second_arm_of_one_fanout() { - // Note 74: the write registry records each scoped write's (fanout token, - // arm index). A second arm of the same fanout writing the same path is a - // hard write-write race; the same arm rewriting, a later fanout's write, - // and untracked writes all stay legal. - let store = StoreRef::memory(); - let token = store.next_write_token(); - let arm_one = WriteScope::new(token, 1); - let arm_two = WriteScope::new(token, 2); +fn a_second_identitys_write_to_a_claimed_path_races() { + // Write-race detection is now the claims model's: a write booms when + // another live identity holds a claim on the path. One identity + // rewriting its own path stays legal, and releasing the first + // identity frees the path. + let (vfs, first) = stock(); + let store = vfs.store(&first); + store.write("a.txt", "one").expect("first write"); store - .write_scoped("a.txt", "one", arm_one) - .expect("first write"); - store - .write_scoped("a.txt", "uno", arm_one) - .expect("the same arm may rewrite its own path"); - let err = store - .write_scoped("a.txt", "two", arm_two) - .expect_err("a second arm of the same fanout must race"); + .write("a.txt", "uno") + .expect("one identity may rewrite its own path"); + let second = vfs.acquire(); + let contender = vfs.store(&second); + let err = contender + .write("a.txt", "two") + .expect_err("a second live identity must race"); assert_eq!(err.kind(), StoreErrorKind::WriteRace); assert_eq!(err.path(), Some("a.txt")); assert!( - err.to_string().contains("another arm of the same fanout"), + err.to_string().contains("write-write race"), "error was: {err}" ); - // The raced write never reached the backend. + // The claims model covers appends, which WriteScope never did. + let err = contender + .append("a.txt", "+") + .expect_err("a second live identity's append must race"); + assert_eq!(err.kind(), StoreErrorKind::WriteRace); + // The raced writes never reached the backend. assert_eq!(store.read("a.txt").expect("read"), "uno"); - // A later fanout overwrites the record, so sequential fanouts stay legal. - let later = WriteScope::new(store.next_write_token(), 1); - store - .write_scoped("a.txt", "new", later) - .expect("a later fanout may write the path"); - assert_eq!(store.read("a.txt").expect("read"), "new"); - // Untracked writes neither record nor race. + // Releasing the first identity retires its claims; the path is free. + drop(first); + contender + .write("a.txt", "new") + .expect("a released claim no longer conflicts"); + assert_eq!(contender.read("a.txt").expect("read"), "new"); +} + +#[test] +fn a_glob_over_a_claimed_path_races_like_a_read() { + // Error-mapping parity: a read booms when another live identity holds + // a writer claim, and a glob (or its per-match stat) that touches the + // same claimed path must surface the same WriteRace vocabulary, not an + // opaque Backend. + let (vfs, first) = stock(); + let store = vfs.store(&first); + store.write("a.txt", "one").expect("first write"); + let second = vfs.acquire(); + let contender = vfs.store(&second); + let err = contender + .glob("*.txt") + .expect_err("a glob touching a claimed path must race"); + assert_eq!(err.kind(), StoreErrorKind::WriteRace); + // Releasing the first identity retires its claims; the glob succeeds. + drop(first); + assert_eq!(contender.glob("*.txt").expect("glob"), vec!["a.txt"]); +} + +#[test] +fn two_facades_bound_to_one_identity_never_conflict() { + // Borrow semantics: blocking call chains share the parent's access, + // so two facades over one identity touch one path without a false + // conflict. + let (vfs, access) = stock(); + let store = vfs.store(&access); + let also = store.clone(); + store.write("a.txt", "one").expect("write"); + also.append("a.txt", "two").expect("append"); + assert_eq!(store.read("a.txt").expect("read"), "onetwo"); +} + +#[test] +fn identities_share_backing_state_once_claims_are_released() { + let (vfs, first) = stock(); + let store = vfs.store(&first); store - .write("a.txt", "walk") - .expect("walk-section writes are untracked"); - store.append("a.txt", "+").expect("append is untracked"); - assert_eq!(store.read("a.txt").expect("read"), "walk+"); + .write("shared.txt", "written by the first") + .expect("write"); + drop(first); + let second = vfs.acquire(); + let reader = vfs.store(&second); + assert_eq!( + reader.read("shared.txt").expect("read"), + "written by the first" + ); } #[test] fn read_range_with_start_only_reads_to_end() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "one\ntwo\nthree\n").expect("write"); assert_eq!( store.read_range("a.txt", 2, None).expect("read_range"), @@ -149,7 +168,8 @@ fn read_range_with_start_only_reads_to_end() { #[test] fn read_range_with_start_and_end_slices_inclusively() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "one\ntwo\nthree\n").expect("write"); assert_eq!( store.read_range("a.txt", 2, Some(2)).expect("read_range"), @@ -163,7 +183,8 @@ fn read_range_with_start_and_end_slices_inclusively() { #[test] fn read_range_clamps_end_to_the_last_line() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "one\ntwo\nthree\n").expect("write"); assert_eq!( store.read_range("a.txt", 2, Some(99)).expect("read_range"), @@ -173,7 +194,8 @@ fn read_range_clamps_end_to_the_last_line() { #[test] fn read_range_beyond_eof_is_empty() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "one\ntwo\nthree\n").expect("write"); assert_eq!(store.read_range("a.txt", 4, None).expect("read_range"), ""); // The end bound is never evaluated when the range starts beyond EOF. @@ -185,14 +207,16 @@ fn read_range_beyond_eof_is_empty() { #[test] fn read_range_empty_file_is_empty_string() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("e.txt", "").expect("write"); assert_eq!(store.read_range("e.txt", 1, None).expect("read_range"), ""); } #[test] fn read_range_start_below_one_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "one\ntwo\n").expect("write"); for style in [RangeStyle::Plain, RangeStyle::Numbered] { let err = style @@ -206,7 +230,8 @@ fn read_range_start_below_one_errors() { #[test] fn read_range_end_before_start_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "one\ntwo\nthree\n").expect("write"); for style in [RangeStyle::Plain, RangeStyle::Numbered] { let err = style @@ -219,7 +244,8 @@ fn read_range_end_before_start_errors() { #[test] fn read_range_missing_file_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); for style in [RangeStyle::Plain, RangeStyle::Numbered] { let err = style .read(&store, "absent.txt", 1, None) @@ -237,7 +263,7 @@ enum RangeStyle { impl RangeStyle { fn read( self, - store: &StoreRef, + store: &Store, path: &str, start: usize, end: Option, @@ -250,7 +276,7 @@ impl RangeStyle { } /// Writes `line1` through `line` into `path`. -fn numbered_fixture(store: &StoreRef, path: &str, line_count: usize) { +fn numbered_fixture(store: &Store, path: &str, line_count: usize) { let mut body = String::new(); for n in 1..=line_count { use std::fmt::Write as _; @@ -261,7 +287,8 @@ fn numbered_fixture(store: &StoreRef, path: &str, line_count: usize) { #[test] fn read_range_numbered_without_bounds_numbers_from_one() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); numbered_fixture(&store, "a.txt", 12); assert_eq!( store @@ -273,7 +300,8 @@ fn read_range_numbered_without_bounds_numbers_from_one() { #[test] fn read_range_numbered_empty_file_is_empty_string() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("e.txt", "").expect("write"); assert_eq!( store @@ -285,7 +313,8 @@ fn read_range_numbered_empty_file_is_empty_string() { #[test] fn read_range_numbered_numbers_a_slice_absolutely() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); numbered_fixture(&store, "a.txt", 85); assert_eq!( store @@ -297,7 +326,8 @@ fn read_range_numbered_numbers_a_slice_absolutely() { #[test] fn read_range_numbered_pads_across_the_hundred_boundary() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); numbered_fixture(&store, "a.txt", 100); assert_eq!( store @@ -309,7 +339,8 @@ fn read_range_numbered_pads_across_the_hundred_boundary() { #[test] fn read_range_numbered_clamps_end_to_the_last_line() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); numbered_fixture(&store, "a.txt", 100); assert_eq!( store @@ -330,7 +361,8 @@ fn read_range_numbered_clamps_end_to_the_last_line() { #[test] fn read_range_numbered_beyond_eof_is_empty() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "one\ntwo\nthree\n").expect("write"); assert_eq!( store @@ -342,7 +374,8 @@ fn read_range_numbered_beyond_eof_is_empty() { #[test] fn write_overwrites() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "old").expect("write"); store.write("a.txt", "new").expect("overwrite"); assert_eq!(store.read("a.txt").expect("read"), "new"); @@ -350,7 +383,8 @@ fn write_overwrites() { #[test] fn append_creates_then_extends() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.append("log.txt", "one\n").expect("create via append"); store.append("log.txt", "two").expect("extend"); assert_eq!(store.read("log.txt").expect("read"), "one\ntwo"); @@ -358,7 +392,8 @@ fn append_creates_then_extends() { #[test] fn str_replace_replaces_unique() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "the quick brown fox").expect("write"); store .str_replace("a.txt", "quick", "slow") @@ -368,7 +403,8 @@ fn str_replace_replaces_unique() { #[test] fn str_replace_missing_anchor_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "hello world").expect("write"); let err = store .str_replace("a.txt", "absent", "x") @@ -378,7 +414,8 @@ fn str_replace_missing_anchor_errors() { #[test] fn str_replace_ambiguous_anchor_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "na na na").expect("write"); let err = store .str_replace("a.txt", "na", "la") @@ -391,7 +428,8 @@ fn str_replace_ambiguous_anchor_errors() { #[test] fn str_replace_on_missing_file_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); let err = store .str_replace("nope.txt", "a", "b") .expect_err("should fail"); @@ -400,7 +438,8 @@ fn str_replace_on_missing_file_errors() { #[test] fn delete_then_read_errors() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "gone soon").expect("write"); store.delete("a.txt").expect("delete"); let err = store.read("a.txt").expect_err("should fail"); @@ -409,14 +448,16 @@ fn delete_then_read_errors() { #[test] fn delete_missing_is_silent() { - // Note 55: delete is idempotent, so deleting an absent path succeeds. - let store = StoreRef::memory(); + // Delete is idempotent: deleting an absent path succeeds. + let (vfs, access) = stock(); + let store = vfs.store(&access); store.delete("absent.txt").expect("delete is idempotent"); } #[test] fn glob_matches_sorted() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); for path in ["src/b.rs", "src/a.rs", "src/deep/c.rs", "notes.md"] { store.write(path, "").expect("write"); } @@ -429,18 +470,21 @@ fn glob_matches_sorted() { vec!["src/a.rs", "src/b.rs", "src/deep/c.rs"], ); assert_eq!(store.glob("*.md").expect("glob"), vec!["notes.md"]); + // The store vocabulary lists files only: the materialized ancestor + // directories the VFS glob also matches are filtered out. assert_eq!(store.glob("**").expect("glob").len(), 4); } #[test] fn glob_star_stops_at_slash() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a/b.txt", "").expect("write"); assert!(store.glob("*.txt").expect("glob").is_empty()); assert_eq!(store.glob("a/*.txt").expect("glob"), vec!["a/b.txt"]); } -fn assert_invalid_paths(store: &StoreRef, cases: &[(&str, PathReason)]) { +fn assert_invalid_paths(store: &Store, cases: &[(&str, PathReason)]) { for (path, reason) in cases { let err = store.read(path).expect_err("path must be rejected"); assert_eq!(err.kind(), StoreErrorKind::InvalidPath, "{path}"); @@ -453,7 +497,8 @@ fn assert_invalid_paths(store: &StoreRef, cases: &[(&str, PathReason)]) { #[test] fn invalid_paths_are_rejected_before_dispatch() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); assert_invalid_paths( &store, &[ @@ -469,7 +514,8 @@ fn invalid_paths_are_rejected_before_dispatch() { #[test] fn exists_reports_confirmed_absence_and_presence() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); assert!(!store.exists("a.txt").expect("absence is not an error")); store.write("a.txt", "hi").expect("write"); assert!(store.exists("a.txt").expect("presence is not an error")); @@ -477,7 +523,8 @@ fn exists_reports_confirmed_absence_and_presence() { #[test] fn glob_rejects_empty_and_oversized_and_control_patterns() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); assert_eq!( store.glob("").expect_err("empty").kind(), StoreErrorKind::InvalidPattern @@ -495,7 +542,8 @@ fn glob_rejects_empty_and_oversized_and_control_patterns() { #[test] fn empty_anchor_is_refused() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a.txt", "body").expect("write"); let err = store .str_replace("a.txt", "", "x") @@ -509,7 +557,8 @@ fn empty_anchor_is_refused() { #[test] fn str_replace_reports_empty_ascii_and_multibyte_contents() { // STORE-007 coverage: empty, ASCII, and multibyte file contents. - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("empty.txt", "").expect("write empty"); let empty_err = store @@ -541,7 +590,8 @@ fn str_replace_reports_empty_ascii_and_multibyte_contents() { #[test] fn platform_unsafe_paths_are_rejected_before_dispatch() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); assert_invalid_paths( &store, &[ @@ -555,7 +605,7 @@ fn platform_unsafe_paths_are_rejected_before_dispatch() { ], ); // A path at the exact byte limit is accepted; one byte over is rejected. - let maximum = "a".repeat(path::MAX_STORE_PATH_BYTES); + let maximum = "a".repeat(MAX_STORE_PATH_BYTES); store .write(&maximum, "at-limit") .expect("a 1024-byte path must be accepted"); @@ -563,7 +613,7 @@ fn platform_unsafe_paths_are_rejected_before_dispatch() { store.read(&maximum).expect("read at-limit path"), "at-limit" ); - let too_long = "a".repeat(path::MAX_STORE_PATH_BYTES + 1); + let too_long = "a".repeat(MAX_STORE_PATH_BYTES + 1); let error = store .read(&too_long) .expect_err("a 1025-byte path must be rejected"); @@ -584,7 +634,8 @@ fn platform_unsafe_paths_are_rejected_before_dispatch() { #[test] fn glob_grammar_rejects_unsupported_forms() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); for bad in ["a**b", "***", "a/***/b", "a\\*.txt"] { assert_eq!( store.glob(bad).expect_err(bad).kind(), @@ -601,9 +652,10 @@ fn glob_grammar_rejects_unsupported_forms() { #[test] fn glob_matcher_is_bounded_against_adversarial_patterns() { // STORE-005: a pattern packed with single-segment stars against a long - // non-matching name completes promptly (the old recursive/backtracking + // non-matching name completes promptly (a recursive/backtracking // matcher would blow up here). The iterative matcher is O(tokens*len). - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); let name = "a".repeat(200); store.write(&name, "").expect("write"); // A grammar-valid pattern of many single `*` separated by literals: the @@ -627,7 +679,8 @@ fn glob_matcher_is_bounded_against_adversarial_patterns() { #[test] fn glob_double_star_slash_matches_zero_segments() { - let store = StoreRef::memory(); + let (vfs, access) = stock(); + let store = vfs.store(&access); store.write("a/b.rs", "").expect("write"); // `a/**/b.rs` matches `a/b.rs` (zero intermediate segments). assert_eq!(store.glob("a/**/b.rs").expect("glob"), vec!["a/b.rs"]); @@ -646,67 +699,89 @@ fn backend_ctor_classifies_and_hides_source() { } #[test] -fn store_ref_is_send_sync_and_static() { - // STORE-009: the handle must also be `'static` (it is shared across - // spawned tasks that outlive the caller), so the assertion carries the - // promised `'static` bound, not just `Send + Sync`. - fn assert_send_sync_static() {} - assert_send_sync_static::(); +fn the_facade_is_send_and_sync() { + // The facade is shared across spawned tasks that outlive the caller, + // so the assertion carries the promised bounds. + fn assert_send_sync() {} + assert_send_sync::>(); } -#[test] -fn a_poisoned_backend_lock_surfaces_as_a_backend_error() { - let store = StoreRef::memory(); - let clone = store.clone(); - let _ = std::thread::spawn(move || { - let _guard = clone.inner.lock().expect("lock"); - panic!("poison the store lock"); - }) - .join(); - // STORE-004: after a holder panicked mid-hold, operations report a - // backend failure rather than trusting the possibly half-mutated state. - let err = store.read("a.txt").expect_err("a poisoned lock must error"); - assert_eq!(err.kind(), StoreErrorKind::Backend); - assert!(std::error::Error::source(&err).is_some()); -} +/// A backend whose writes panic mid-call, poisoning every lock on the +/// operation path. All other operations delegate to a memory backend. +struct PanicBackend(MemoryBackend); -#[test] -fn with_files_populates_store() { - let store = StoreRef::with_files([ - ("a.txt".to_owned(), "alpha".to_owned()), - ("b.txt".to_owned(), "beta".to_owned()), - ]) - .expect("valid paths"); - assert_eq!(store.read("a.txt").unwrap(), "alpha"); - assert_eq!(store.read("b.txt").unwrap(), "beta"); -} +impl Vfs for PanicBackend { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + Ok(Box::new(PanicAccess(self.0.acquire(id)?))) + } -#[test] -fn with_files_rejects_invalid_path() { - let result = StoreRef::with_files([("../escape.txt".to_owned(), "bad".to_owned())]); - assert!(result.is_err()); + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + self.0.release(id) + } } -#[test] -fn with_files_empty_is_empty_store() { - let store = - StoreRef::with_files(std::iter::empty::<(String, String)>()).expect("empty is valid"); - assert!(!store.exists("anything.txt").unwrap()); +struct PanicAccess(Box); + +impl VfsAccess for PanicAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + self.0.read(path) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + let _ = (path, contents); + panic!("poison the operation path"); + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.0.append(path, contents) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.0.remove(path, recursive) + } + + fn exists(&self, path: &VfsPath) -> Result { + self.0.exists(path) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + self.0.glob(pattern) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + self.0.list(path) + } + + fn stat(&self, path: &VfsPath) -> Result { + self.0.stat(path) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.0.mkdir(path, recursive) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.0.rename(from, to) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.0.copy(from, to) + } } #[test] -fn clones_share_backing_state() { - let store = StoreRef::memory(); - let clone = store.clone(); - store - .write("shared.txt", "written by original") - .expect("write"); - assert_eq!( - clone.read("shared.txt").expect("read"), - "written by original" - ); - clone - .write("second.txt", "written by clone") - .expect("write"); - assert_eq!(store.read("second.txt").expect("read"), "written by clone"); +fn a_panicking_operation_does_not_wedge_the_store() { + // Poison handling is now poison-safe recovery: a panic mid-operation + // poisons the locks on the operation path, the poison-safe locking + // recovers, and the very next operation works. + let vfs = VfsRef::builder() + .mount(STORE_MOUNT, PanicBackend(MemoryBackend::new())) + .build(); + let access = vfs.acquire(); + let store = vfs.store(&access); + let outcome = + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| store.write("a.txt", "x"))); + assert!(outcome.is_err(), "the panic must propagate"); + store.append("b.txt", "y").expect("append after the panic"); + assert_eq!(store.read("b.txt").expect("read"), "y"); } diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index b224ebf8b..a7f5299e6 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -634,7 +634,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 8: Store facade rewrite and parity suite +### Step 8: Store facade rewrite and parity suite [completed] - Component: promptforge-store - Rewrite `promptforge-store`: the Store trait, MemStore, and FileStore disappear from the public API; a public concrete `Store` facade wraps a prefix-scoped Access and is exposed as `vfs.store(&access)` via a prelude-exported extension trait (see decision record). Preserve the StoreError vocabulary exactly with a total VfsError-to-StoreError mapping, anchor-edit rules, numbered reads, idempotent delete (NotFound maps to Ok), and the glob grammar (port one glob implementation, delete the other). Delete the WriteScope registry. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index e4ef180da..cc9725cee 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -109,3 +109,9 @@ - Decision: Ask mode returns `Verdict::Ask` (not `Deny`) for mutations; the capability layer maps both to `PermissionDenied`, and the Ask string is the approval-dialog text per the contract | Falsifier: a later integration step asserts `Verdict::Deny` from `ModePolicy::check` in Ask mode. - Decision: Plan's markdown rule is a case-sensitive `.md` suffix (clippy's case-insensitive suggestion explicitly `#[expect]`-overridden), matching the POSIX-strict virtual namespace | Falsifier: a host needs `.MD`/`.markdown` admitted in Plan mode. - Decision: `ModeHandle` is a separate cloneable UI half (`policy.handle()`), per "one-way vs reversible is just who still holds the mode handle" | Falsifier: a caller needs to flip modes holding only the `ModePolicy`. +- Step 8: Store facade rewrite and parity suite - `cargo nextest run -p promptforge-store` - 44 passed, 0 failed, plus 15 doctests; `cargo build` and `cargo fmt --all --check` pass. Review: clean after a needs-context re-review supplied the decision-record extension-trait rationale; 1 Important (glob/stat claim conflicts surfaced as opaque Backend instead of WriteRace), closed with a regression test. DEVIATION: the COMPONENT clippy leg is deferred to run jointly after step 9 - the plan's step ordering leaves promptforge-lua referencing the removed StoreRef/WriteScope API, and the verification-fix round correctly returned blocked rather than perform step-9 migration inside step 8 | Falsifier: the joint verify after step 9 fails the promptforge-store component. + - Decision: `WriteRace` display text now names the claims model, not fanout arms | Falsifier: a caller matching on the old message text. + - Decision: facade implements anchor edits and line ranges itself (read/count/write) to preserve the exact `StoreError` vocabulary | Falsifier: a need for backend-atomic `str_replace`. + - Decision: glob delegates matching to the backend and stat-filters to files only | Falsifier: glob latency complaints on large trees. + - Decision: backslash glob rejection lives in the facade because the router canonicalizes patterns before the backend sees them | Falsifier: router forwarding verbatim patterns. + - Decision: `GlobSpyStore`, `with_files`, and `FileStore` tests dropped; their mechanisms no longer exist (matching moved to the backend, `with_files` had no external callers, `FileStore` superseded by `HostBackend`) | Falsifier: a host depending on file-backed store behavior through this crate. From 9df19b1a4d814e0ffa383e3e112fea89e24e2a9b Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 20:26:01 -0700 Subject: [PATCH 09/26] Pivot the executor API from StoreRef to VfsRef Every entry point that used to take the run's store handle now takes the virtual filesystem handle, and each chain step of a run installs its own access capability so every store operation is attributed to a live identity. The walk and the live pass acquire their own capabilities, a blocking call chain borrows its parent's, and each fanout arm spawns one from the caller, which retires the caller's claims as the happens-before edge. A capability is released when its chain ends, so a finished arm's claims never linger into the join's merge. A hand-built handle lacking the store mount gets a fresh memory store overlaid as a defensive fallback, while a mounted but failing backend fails the run instead of being shadowed. The fanout-specific write registry is gone: conflict detection now comes entirely from the claims model, which also covers appends. - `Chain` holds the chain step's access capability in an `Option` that `finish` and `abort_subtree` take at chain end, so a finished arm's claims release before the fanout join's merge rather than lingering until scheduler drop. A call chain stores an `Arc` clone of its parent's capability: a blocking child is the same serial thread, so it gets no new identity and never false-conflicts with the caller. - `ArmTemplate` carries the fanout caller's capability in place of the per-fanout write token; each arm's chain spawns its own capability from it at dispatch, retiring the caller's claims as the happens-before edge, so two live arms touching one path meet the claims model's conflict rule. - `run_agent` acquires one capability for the whole run and passes it into the agent VM setup, matching the agent's single serial thread of execution. - `install_store_table` captures an `Arc` clone of the section's capability in each closure and builds the borrowing `Store` facade per call, so every store op is attributed to the identity the executor installed for that chain step. - `store_mount_present` stats the store mount root through a throwaway capability and treats only `NotFound` as absent; any other error fails the run through the new `Error::Store` variant, so a mounted-but-failing backend is never shadowed by the defensive memory overlay. - `WriteScope` is removed from the scheduler's arm state, the VM setup struct, and the store crate's re-exports; the claims model is now the only write-conflict guard, and it covers the appends the registry never tracked. - `fanout-store-writes.md` drops its ready-marker rendezvous and yield section: polling a live sibling's writes is the cross-arm pattern the claims model rejects, so the fixture now writes only arm-scoped paths. Design: new surface-growth @ crates/promptforge-core/src/execute.rs::run boundary: pub Design: new temporal-coupling @ crates/promptforge-core/src/execute/scheduler.rs::Chain Repairs: one store path claimed by two live identities fails the second claimant @ crates/promptforge-core/src/execute/scheduler.rs - two live fanout arms appending one path ran untracked instead of failing with a write-write race Plan: vibe/2026-09-11-3-vfs-foundation.md --- Cargo.lock | 9 +- crates/promptforge-agent/Cargo.toml | 2 + crates/promptforge-agent/src/agent.rs | 66 ++-- crates/promptforge-agent/src/tests.rs | 25 +- crates/promptforge-core/Cargo.toml | 2 + crates/promptforge-core/README.md | 6 +- .../promptforge-core/benches/models_loop.rs | 6 +- crates/promptforge-core/src/error.rs | 7 + crates/promptforge-core/src/execute.rs | 43 ++- .../promptforge-core/src/execute/context.rs | 34 +- crates/promptforge-core/src/execute/error.rs | 1 + .../promptforge-core/src/execute/scheduler.rs | 104 ++++-- .../src/execute/section_context.rs | 30 +- .../src/execute/section_vm.rs | 20 +- .../src/execute/tests/debug_and_counts.rs | 16 +- .../src/execute/tests/exec_flow.rs | 109 ++++--- .../src/execute/tests/exit_rules.rs | 2 +- .../src/execute/tests/input.rs | 2 +- .../src/execute/tests/live_infer.rs | 28 +- .../src/execute/tests/local_tools.rs | 2 +- .../promptforge-core/src/execute/tests/mod.rs | 60 +++- .../src/execute/tests/model_and_reply.rs | 50 +-- .../src/execute/tests/models_loop.rs | 2 +- .../src/execute/tests/observations.rs | 10 +- .../src/execute/tests/scheduler.rs | 166 ++++++---- .../src/execute/tests/tool_scoping.rs | 2 +- .../src/execute/tests/unified_pipeline.rs | 2 +- crates/promptforge-core/src/lib.rs | 5 +- crates/promptforge-core/src/lua/coro_tests.rs | 6 +- .../src/model/tests/always.rs | 10 +- .../src/model/tests/integration.rs | 6 +- .../promptforge-core/src/model/tests/mod.rs | 8 +- crates/promptforge-core/src/store.rs | 35 +- .../prompts/execution/fanout-store-writes.md | 22 +- .../promptforge-core/tests/suite/execution.rs | 5 +- crates/promptforge-core/tests/suite/fanout.rs | 31 +- crates/promptforge-core/tests/suite/main.rs | 1 + .../promptforge-core/tests/suite/support.rs | 31 +- crates/promptforge-core/tests/suite/vfs.rs | 202 ++++++++++++ crates/promptforge-lua/Cargo.toml | 2 + crates/promptforge-lua/benches/surface.rs | 9 +- crates/promptforge-lua/src/host.rs | 76 +++-- crates/promptforge-lua/src/lib.rs | 5 +- crates/promptforge-lua/src/messages/tests.rs | 8 +- crates/promptforge-lua/src/models/tests.rs | 2 +- crates/promptforge-lua/src/tests.rs | 305 +++++++++++------- crates/promptforge-lua/src/tools/tests.rs | 8 +- crates/promptforge-lua/src/vm.rs | 62 ++-- crates/promptforge-store/README.md | 13 +- crates/promptforge-store/src/error.rs | 4 +- crates/promptforge-store/src/lib.rs | 18 +- crates/promptforge-store/src/path.rs | 7 +- crates/workshop-server/Cargo.toml | 3 +- crates/workshop-server/src/session_agents.rs | 2 +- .../src/session_agents/supervisor/effects.rs | 18 +- crates/workshop-server/tests/it/chat_gate.rs | 3 +- vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 7 + 58 files changed, 1118 insertions(+), 604 deletions(-) create mode 100644 crates/promptforge-core/tests/suite/vfs.rs diff --git a/Cargo.lock b/Cargo.lock index ed1235f12..5e6bfa885 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4817,7 +4817,9 @@ dependencies = [ "promptforge-model-client", "promptforge-store", "promptforge-tools", + "promptforge-vfs", "serde_json", + "shared-vfs", "thiserror 2.0.19", "tokio", ] @@ -4837,10 +4839,12 @@ dependencies = [ "promptforge-store", "promptforge-tool-picker", "promptforge-tools", + "promptforge-vfs", "promptforge-web-search", "rand 0.9.5", "serde", "serde_json", + "shared-vfs", "thiserror 2.0.19", "time", "tokio", @@ -4868,7 +4872,9 @@ dependencies = [ "promptforge-model-client", "promptforge-store", "promptforge-tools", + "promptforge-vfs", "serde_json", + "shared-vfs", "thiserror 2.0.19", "tokio", ] @@ -8564,9 +8570,9 @@ dependencies = [ "promptforge-core", "promptforge-core-support", "promptforge-model-client", - "promptforge-store", "promptforge-tool-picker", "promptforge-tools", + "promptforge-vfs", "rand 0.9.5", "reqwest 0.12.28", "rust-embed", @@ -8575,6 +8581,7 @@ dependencies = [ "shared-loopback", "shared-progress", "shared-sidecar", + "shared-vfs", "socket2", "tempfile", "thiserror 2.0.19", diff --git a/crates/promptforge-agent/Cargo.toml b/crates/promptforge-agent/Cargo.toml index 791181a78..d72e96954 100644 --- a/crates/promptforge-agent/Cargo.toml +++ b/crates/promptforge-agent/Cargo.toml @@ -17,12 +17,14 @@ promptforge-model-client.workspace = true promptforge-store.workspace = true promptforge-tools.workspace = true serde_json.workspace = true +shared-vfs.workspace = true thiserror.workspace = true tokio.workspace = true [dev-dependencies] async-trait.workspace = true axum.workspace = true +promptforge-vfs.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread", "net"] } [lints] diff --git a/crates/promptforge-agent/src/agent.rs b/crates/promptforge-agent/src/agent.rs index 17a40d933..8f8c9fa3b 100644 --- a/crates/promptforge-agent/src/agent.rs +++ b/crates/promptforge-agent/src/agent.rs @@ -43,8 +43,9 @@ use promptforge_model_client::client::{ use promptforge_model_client::model::{ ModelBinding, ModelCatalog, ModelInvocation, ModelSet, ModelView, }; -use promptforge_store::StoreRef; +use promptforge_store::Access; use promptforge_tools::ToolCatalog; +use shared_vfs::VfsRef; use crate::config::AgentConfig; @@ -152,10 +153,10 @@ pub async fn run_agent( source: &str, tools: &ToolCatalog, models: &ModelCatalog, - store: &StoreRef, + vfs: &VfsRef, config: AgentConfig, ) -> Result<(), AgentError> { - run_agent_with_client(source, tools, models, store, config, None).await + run_agent_with_client(source, tools, models, vfs, config, None).await } /// [`run_agent`] with an explicit gateway client instead of the lazy @@ -177,12 +178,12 @@ pub async fn run_agent_with_client( source: &str, tools: &ToolCatalog, models: &ModelCatalog, - store: &StoreRef, + vfs: &VfsRef, config: AgentConfig, client: Option, ) -> Result<(), AgentError> { let cancel = config.cancel.clone(); - cancel::scope(cancel, drive(source, tools, models, store, config, client)).await + cancel::scope(cancel, drive(source, tools, models, vfs, config, client)).await } /// One agent run: compile, build the agent VM, drive the program coroutine @@ -191,7 +192,7 @@ async fn drive( source: &str, tools: &ToolCatalog, models: &ModelCatalog, - store: &StoreRef, + vfs: &VfsRef, config: AgentConfig, client: Option, ) -> Result<(), AgentError> { @@ -227,8 +228,11 @@ async fn drive( // A limits failure propagates bare, before any teardown observation // exists - the section drivers' contract. vm.apply_lua_limits(limits.lua_memory_bytes, limits.lua_log_events)?; + // The agent is one serial thread of execution: one capability for the + // whole run, released when it drops at the run's end. + let access = Arc::new(vfs.acquire()); let (counts, events) = - match setup_agent_vm(&mut vm, store, &observer, &name, &tool_set, event_log, ui) { + match setup_agent_vm(&mut vm, &access, &observer, &name, &tool_set, event_log, ui) { Ok(installed) => installed, Err(error) => { vm.teardown(observer.as_ref(), &name); @@ -347,14 +351,14 @@ fn agent_model_set(catalog: &ModelCatalog) -> ModelSet { /// never in a section VM. fn setup_agent_vm( vm: &mut SectionVm, - store: &StoreRef, + access: &Arc, observer: &Arc, name: &str, tool_set: &ToolSet, event_log: Option>, ui: Option serde_json::Value + Send + Sync>>, ) -> Result<(ToolCallCounts, Option), AgentError> { - vm.inject_host_with_var("", &serde_json::json!({}), store, None, None)?; + vm.inject_host_with_var("", &serde_json::json!({}), access, None)?; vm.install_host_apis(observer, name)?; vm.install_coro_shims()?; install_agent_chat_shim(vm.lua())?; @@ -892,6 +896,16 @@ mod tests { const EXECUTION: &str = "agent-test"; + /// Reads one store file through a fresh, immediately dropped access: + /// the run's identity dropped with it, so nothing it wrote conflicts. + fn read_store( + vfs: &VfsRef, + path: &str, + ) -> std::result::Result { + let access = vfs.acquire(); + promptforge_store::StoreExt::store(vfs, &access).read(path) + } + fn config() -> AgentConfig { AgentConfig { name: "test-agent".to_owned(), @@ -911,18 +925,18 @@ mod tests { #[tokio::test] async fn a_trivial_agent_writes_to_the_store_and_returns() { - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); run_agent( "store.write('notes.txt', 'from the agent')\nreturn 'done'", &empty_tools(), &ModelCatalog::empty(), - &store, + &vfs, config(), ) .await .expect("the trivial agent runs to completion"); assert_eq!( - store.read("notes.txt").expect("the agent's write persists"), + read_store(&vfs, "notes.txt").expect("the agent's write persists"), "from the agent", "the agent's store write must be visible through the run-scoped handle" ); @@ -932,12 +946,12 @@ mod tests { async fn the_control_globals_are_nil_in_the_agent_vm() { // Absent, not stubbed: a stub function would tostring as // `function: 0x...`; only true absence renders three nils. - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); let error = run_agent( "return tostring(call) .. ' ' .. tostring(fanout) .. ' ' .. tostring(jump)", &empty_tools(), &ModelCatalog::empty(), - &store, + &vfs, config(), ) .await; @@ -951,13 +965,13 @@ mod tests { "store.write('nils.txt', tostring(call) .. ' ' .. tostring(fanout) .. ' ' .. tostring(jump))", &empty_tools(), &ModelCatalog::empty(), - &store, + &vfs, config(), ) .await .expect("the probe agent runs"); assert_eq!( - store.read("nils.txt").expect("the probe wrote its reading"), + read_store(&vfs, "nils.txt").expect("the probe wrote its reading"), "nil nil nil", "call, fanout, and jump must all be nil in the agent VM" ); @@ -966,13 +980,13 @@ mod tests { #[tokio::test] async fn calling_an_absent_control_global_is_an_undefined_global_failure() { for global in ["call", "fanout", "jump"] { - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); let source = format!("{global}('anything')"); let error = run_agent( &source, &empty_tools(), &ModelCatalog::empty(), - &store, + &vfs, config(), ) .await @@ -991,7 +1005,7 @@ mod tests { #[tokio::test] async fn ui_snapshots_are_fresh_per_call_and_json_nulls_read_nil() { - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); let calls = Arc::new(AtomicU32::new(0)); let counter = Arc::clone(&calls); let mut run_config = config(); @@ -1009,13 +1023,13 @@ mod tests { store.write('ui.txt', first .. '|' .. second .. '|' .. root)", &empty_tools(), &ModelCatalog::empty(), - &store, + &vfs, run_config, ) .await .expect("the ui probe agent runs"); assert_eq!( - store.read("ui.txt").expect("the probe wrote its readings"), + read_store(&vfs, "ui.txt").expect("the probe wrote its readings"), "m1|m2|nil", "every ui() call invokes the provider afresh, and a JSON null field reads nil" ); @@ -1028,18 +1042,18 @@ mod tests { #[tokio::test] async fn ui_is_nil_without_a_provider() { - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); run_agent( "store.write('ui.txt', tostring(ui))", &empty_tools(), &ModelCatalog::empty(), - &store, + &vfs, config(), ) .await .expect("the probe agent runs"); assert_eq!( - store.read("ui.txt").expect("the probe wrote its reading"), + read_store(&vfs, "ui.txt").expect("the probe wrote its reading"), "nil", "no provider means no ui global at all - absent, not stubbed" ); @@ -1070,12 +1084,12 @@ mod tests { )]) .expect("the test catalog has one unique model"); let run = tokio::spawn(async move { - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); run_agent_with_client( "models.use('test-model')\nreturn models.infer('hello')", &empty_tools(), &models, - &store, + &vfs, run_config, Some(client), ) diff --git a/crates/promptforge-agent/src/tests.rs b/crates/promptforge-agent/src/tests.rs index ef30daf5f..344e3b421 100644 --- a/crates/promptforge-agent/src/tests.rs +++ b/crates/promptforge-agent/src/tests.rs @@ -26,8 +26,9 @@ use promptforge_core_support::events::{ use promptforge_core_support::observe::{Observation, Observer}; use promptforge_model_client::client::{GatewayClient, GatewayEndpoint, SecretString, StreamDelta}; use promptforge_model_client::model::{ModelCatalog, ModelDescriptor, ModelId, ThinkingMode}; -use promptforge_store::StoreRef; +use promptforge_store::StoreExt; use promptforge_tools::{Tool, ToolCatalog, ToolError, ToolId, ToolOutput}; +use shared_vfs::VfsRef; use crate::agent::run_agent_with_client; use crate::{AgentConfig, AgentError, AgentLimits, run_agent}; @@ -548,18 +549,22 @@ fn config() -> AgentConfig { } /// One completed fixture run: the gateway (its recorded requests), the -/// run-scoped store the program wrote its assertions into, and the run's +/// run's VFS handle the program wrote its assertions into, and the run's /// outcome. struct FixtureRun { gateway: FixtureGateway, - store: StoreRef, + vfs: VfsRef, result: Result<(), AgentError>, } impl FixtureRun { - /// Reads one store file the agent program wrote. + /// Reads one store file the agent program wrote, through a fresh, + /// immediately dropped access: the run's identity dropped with it, so + /// nothing it wrote conflicts with the extraction. fn read(&self, path: &str) -> String { - self.store + let access = self.vfs.acquire(); + self.vfs + .store(&access) .read(path) .unwrap_or_else(|error| panic!("the program wrote {path}: {error}")) } @@ -578,19 +583,19 @@ async fn run_over_fixture( .expect("the fixture endpoint is a valid URL"); let key = SecretString::new("fixture-key").expect("the fixture key is non-empty"); let client = GatewayClient::new(endpoint, key); - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); let result = run_agent_with_client( source, &tools, &fixture_models(), - &store, + &vfs, config, Some(client), ) .await; FixtureRun { gateway, - store, + vfs, result, } } @@ -1417,12 +1422,12 @@ async fn firing_cancel_interrupts_a_suspended_tool_call() { let mut config = config(); config.cancel = cancel; let run = tokio::spawn(async move { - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); run_agent( "tools.call('blocking', {})", &tools, &fixture_models(), - &store, + &vfs, config, ) .await diff --git a/crates/promptforge-core/Cargo.toml b/crates/promptforge-core/Cargo.toml index c1023e474..48d7512ac 100644 --- a/crates/promptforge-core/Cargo.toml +++ b/crates/promptforge-core/Cargo.toml @@ -21,10 +21,12 @@ promptforge-parser.workspace = true promptforge-store.workspace = true promptforge-tool-picker.workspace = true promptforge-tools.workspace = true +promptforge-vfs.workspace = true promptforge-web-search.workspace = true rand.workspace = true serde.workspace = true serde_json.workspace = true +shared-vfs.workspace = true thiserror.workspace = true mlua.workspace = true time.workspace = true diff --git a/crates/promptforge-core/README.md b/crates/promptforge-core/README.md index 72e1b73dd..a0289abea 100644 --- a/crates/promptforge-core/README.md +++ b/crates/promptforge-core/README.md @@ -12,12 +12,12 @@ A Rust library that turns Markdown files into executable AI prompt pipelines. Yo [dependencies] promptforge-core = "0.1" promptforge-tool-picker = "0.1" +promptforge-vfs = "0.1" ``` ```rust use promptforge_core::model::ModelCatalog; use promptforge_core::observe::NullObserver; -use promptforge_core::store::StoreRef; use promptforge_core::tools::ToolCatalog; use promptforge_core::{Prompt, ResolutionContext, RunConfig, run}; use promptforge_tool_picker::{Catalog, Config, ToolPicker}; @@ -27,13 +27,13 @@ async fn execute(source: &str) -> Result> { let picker = ToolPicker::build(Catalog::new(Vec::new()), Config::default())?; let models = ModelCatalog::empty(); let tools = ToolCatalog::new(&[])?; - let store = StoreRef::memory(); + let vfs = promptforge_vfs::empty(); let result = run( &prompt, "", ResolutionContext::new(&picker, &models, &tools), - &store, + &vfs, RunConfig::new("readme"), ) .await?; diff --git a/crates/promptforge-core/benches/models_loop.rs b/crates/promptforge-core/benches/models_loop.rs index ea5ed9ab1..90c11ab4a 100644 --- a/crates/promptforge-core/benches/models_loop.rs +++ b/crates/promptforge-core/benches/models_loop.rs @@ -28,7 +28,7 @@ use criterion::{Criterion, criterion_group, criterion_main}; use promptforge_core::client::{GatewayClient, GatewayEndpoint, SecretString}; use promptforge_core::model::{ModelCatalog, ModelDescriptor, ModelId, ThinkingMode}; use promptforge_core::observe::NullObserver; -use promptforge_core::store::StoreRef; + use promptforge_core::tools::ToolCatalog; use promptforge_core::{Prompt, ResolutionContext, RunConfig, run}; use promptforge_tool_picker::{Catalog, Config, ToolPicker}; @@ -174,7 +174,7 @@ fn models_loop(c: &mut Criterion) { &prompt, "", resolution(&picker, &models, &tools), - &StoreRef::memory(), + &promptforge_vfs::empty(), RunConfig::new(EXECUTION) .observer(Arc::new(NullObserver::default())) .client(gateway.client()), @@ -210,7 +210,7 @@ fn compactors_fail(c: &mut Criterion) { &prompt, "", resolution(&picker, &models, &tools), - &StoreRef::memory(), + &promptforge_vfs::empty(), RunConfig::new(EXECUTION) .observer(Arc::new(NullObserver::default())) .client(gateway.client()), diff --git a/crates/promptforge-core/src/error.rs b/crates/promptforge-core/src/error.rs index 955ffab67..aaa58e072 100644 --- a/crates/promptforge-core/src/error.rs +++ b/crates/promptforge-core/src/error.rs @@ -529,6 +529,13 @@ pub(crate) enum Error { source: Option, }, + /// A run-scoped store operation failed at the virtual filesystem layer, + /// retaining the concrete [`shared_vfs::VfsError`] as the `#[source]` + /// cause so a backend failure survives the public wrappers instead of + /// being flattened to a string. + #[error("store operation failed: {0}")] + Store(#[source] shared_vfs::VfsError), + /// Rendering the current time as an RFC 3339 string failed. /// /// Retains the [`time::error::Format`] failure as the private `#[source]` diff --git a/crates/promptforge-core/src/execute.rs b/crates/promptforge-core/src/execute.rs index cb87fa89e..d15670e3c 100644 --- a/crates/promptforge-core/src/execute.rs +++ b/crates/promptforge-core/src/execute.rs @@ -16,7 +16,7 @@ //! the same rules, and the parent walk resumes after the jumper when that //! level exhausts. //! -//! One run-scoped [`StoreRef`] is created once by the caller and threaded through +//! One run-scoped [`VfsRef`] is created once by the caller and threaded through //! every section, so //! bulk state persists across the context-clearing transitions even though a //! section's Lua state never does. @@ -129,7 +129,7 @@ use crate::Error; use crate::cancel; use crate::observe::detail; use crate::parser::{ParseErrorKind, Prompt}; -use crate::store::StoreRef; +use crate::store::VfsRef; // Re-exported for the executor test glob. #[cfg(test)] @@ -176,7 +176,6 @@ pub(crate) use crate::model::ModelSet; /// use promptforge_core::model::ModelCatalog; /// use promptforge_core::observe::NullObserver; /// use promptforge_core::parser::Prompt; -/// use promptforge_core::store::StoreRef; /// use promptforge_core::tools::ToolCatalog; /// use promptforge_tool_picker::{Catalog, Config, ToolPicker}; /// @@ -198,7 +197,7 @@ pub(crate) use crate::model::ModelSet; /// &prompt, /// "", /// ResolutionContext::new(&picker, &models, &tools), -/// &StoreRef::memory(), +/// &promptforge_vfs::empty(), /// RunConfig::new("doc-example"), /// ))?; /// assert_eq!(output, "hello"); @@ -218,7 +217,7 @@ pub async fn run( prompt: &Prompt, args: &str, resolution: ResolutionContext<'_>, - store: &StoreRef, + vfs: &VfsRef, config: RunConfig, ) -> std::result::Result { match prompt.frontmatter().promptforge() { @@ -241,7 +240,23 @@ pub async fn run( crate::lua::LuaProgram::empty().map_err(|error| RunError::from(Error::from(error)))? } }; - let ctx = RunContext::new(prompt, args, store, shared, &config); + // The stock handle carries the store mount; a hand-built router lacking + // it gets a fresh memory store overlaid as a defensive fallback, so a + // run never fails for want of the mount. A mounted-but-failing backend + // is never shadowed by the throwaway overlay: its error fails the run. + let fallback; + let vfs = match store_mount_present(vfs) { + Ok(true) => vfs, + Ok(false) => { + fallback = vfs.overlay( + promptforge_vfs::STORE_MOUNT, + shared_vfs::MemoryBackend::new(), + ); + &fallback + } + Err(error) => return Err(RunError::from(Error::Store(error))), + }; + let ctx = RunContext::new(prompt, args, vfs, shared, &config); let RunConfig { execution, @@ -282,5 +297,21 @@ pub async fn run( result.map_err(RunError::from) } +/// Whether the handle already serves the store mount. The probe stats the +/// mount root through a throwaway capability: a mounted backend answers +/// (the memory backend's root always exists), an unmounted path is +/// `NotFound`. Only `NotFound` means "mount absent": any other error is the +/// mounted backend's own failure and propagates, so a loud backend failure +/// is never converted into the run silently reading and writing a +/// throwaway overlay. The probe's identity and claim release with the +/// access. +fn store_mount_present(vfs: &VfsRef) -> std::result::Result { + match vfs.acquire().stat(promptforge_vfs::STORE_MOUNT) { + Ok(_) => Ok(true), + Err(shared_vfs::VfsError::NotFound(_)) => Ok(false), + Err(error) => Err(error), + } +} + #[cfg(test)] mod tests; diff --git a/crates/promptforge-core/src/execute/context.rs b/crates/promptforge-core/src/execute/context.rs index 4e955f824..6dc169276 100644 --- a/crates/promptforge-core/src/execute/context.rs +++ b/crates/promptforge-core/src/execute/context.rs @@ -17,7 +17,7 @@ use crate::lua::{LuaProgram, ToolSet, ToolView}; use crate::model::{ModelSet, ModelView}; use crate::observe::Observer; use crate::parser::Prompt; -use crate::store::{StoreRef, WriteScope}; +use crate::store::{Access, VfsRef}; use crate::untrusted::GuardNonce; use super::config::{RunConfig, RunLimits}; @@ -40,8 +40,10 @@ pub(crate) struct RunContext { /// The untrusted-envelope nonce, minted once here so every wrap in the /// run shares it. nonce: GuardNonce, - /// The run-scoped store backing every section's Lua `store` table. - store: StoreRef, + /// The run's VFS handle: carries the store mount backing every + /// section's Lua `store` table. Chain steps acquire or spawn their + /// access capabilities from it. + vfs: VfsRef, /// The execution identifier every observation carries. execution: Arc, /// The run's argument string for `{{ args }}` substitution. @@ -104,7 +106,7 @@ impl RunContext { pub(crate) fn new( prompt: &Prompt, args: &str, - store: &StoreRef, + vfs: &VfsRef, shared: LuaProgram, config: &RunConfig, ) -> Self { @@ -113,7 +115,7 @@ impl RunContext { Self { prompt: Arc::new(prompt.clone()), nonce: GuardNonce::fresh(), - store: store.clone(), + vfs: vfs.clone(), execution: Arc::from(config.execution.as_str()), args: Arc::from(args), limits: config.limits, @@ -143,9 +145,9 @@ impl RunContext { &self.nonce } - /// The run-scoped store. - pub(crate) fn store(&self) -> &StoreRef { - &self.store + /// The run's VFS handle. + pub(crate) fn vfs(&self) -> &VfsRef { + &self.vfs } /// The execution identifier every observation carries. @@ -307,23 +309,23 @@ impl RunContext { } /// The borrowed VM-setup inputs both engine drivers share, sourcing the - /// run-wide slots (`args`, `store`, `observer`, `shared`) from this + /// run-wide slots (`args`, `observer`, `shared`) from this /// context; the driver supplies only its own deltas: the `sys` JSON, - /// the seed, the store-write scope (a fanout arm's - /// identity; `None` on the walk), and the section name. + /// the seed, the chain step's access capability (the walk's own, a + /// call chain's borrowed parent capability, a fanout arm's spawned + /// one), and the section name. pub(crate) fn vm_setup<'a>( &'a self, sys: &'a serde_json::Value, seed: VmSeed<'a>, - write_scope: Option, + access: &'a Arc, section_name: &'a str, ) -> SectionVmSetup<'a> { SectionVmSetup { args: &self.args, sys, - store: &self.store, + access, seed, - write_scope, observer_arc: &self.observer, section_name, shared: &self.shared, @@ -356,7 +358,7 @@ impl fmt::Debug for RunContext { f.debug_struct("RunContext") .field("prompt", &self.prompt) .field("nonce", &self.nonce) - .field("store", &"") + .field("vfs", &"") .field("execution", &self.execution) .field("args", &self.args) .field("limits", &self.limits) @@ -395,7 +397,7 @@ mod tests { RunContext::new( prompt, "", - &StoreRef::memory(), + &promptforge_vfs::empty(), LuaProgram::empty().expect("the empty chunk compiles"), &RunConfig::new("run-context-test"), ) diff --git a/crates/promptforge-core/src/execute/error.rs b/crates/promptforge-core/src/execute/error.rs index 7c930f0ec..3c9fcb858 100644 --- a/crates/promptforge-core/src/execute/error.rs +++ b/crates/promptforge-core/src/execute/error.rs @@ -85,6 +85,7 @@ impl RunError { | Error::UnboundToolCall { .. } | Error::Tool { .. } => RunErrorKind::Tool, Error::Internal(_) | Error::TimestampFormat(_) => RunErrorKind::Internal, + Error::Store(_) => RunErrorKind::Store, Error::Bind { .. } | Error::BindSchema { .. } | Error::BindQuery { .. } diff --git a/crates/promptforge-core/src/execute/scheduler.rs b/crates/promptforge-core/src/execute/scheduler.rs index 04eeaacdb..2ea25a712 100644 --- a/crates/promptforge-core/src/execute/scheduler.rs +++ b/crates/promptforge-core/src/execute/scheduler.rs @@ -46,9 +46,9 @@ //! scheduling, a fatal arm error aborts the sibling arms (each aborted //! arm's finalizer reports `FANOUT_ARM_CANCELLED`, so exactly one terminal //! observation fires per arm), [`Error::ToolLoopExhausted`] soft-degrades -//! its arm to the incomplete stub, and two arms of one fanout writing the -//! same store path fail with the store's write-write race error while -//! `append` stays legal with unspecified order. A received `mcp` request +//! its arm to the incomplete stub, and two live arms of one fanout touching +//! the same store path with at least one write fail the second with the +//! claims model's write-write race error. A received `mcp` request //! is the protocol's typed reserved error. use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; @@ -73,6 +73,7 @@ use crate::model::ModelBinding; use crate::observe::{Observer, detail}; use crate::parser::{Block, Section}; use crate::resolve::RuntimeResolution; +use crate::store::Access; use crate::tools::{Tool, ToolId}; use crate::{Error, Result, cancel, subst}; @@ -153,10 +154,11 @@ struct ArmTemplate<'a> { /// a chain never crosses) with a fresh turn counter, so arm turns count /// against the fanout's own cap. ctx: RunContext, - /// This fanout's store-write token: one per fanout, so the store's - /// write registry can tell two arms of this fanout (a write-write race) - /// from a later fanout's write (legal). - write_token: u64, + /// The fanout caller's access capability: each arm spawns its own + /// capability from it at dispatch, so the spawn retires the caller's + /// claims (the happens-before edge) and two live arms touching one + /// path meet the claims model's conflict rule. + access: Arc, /// The caller's `var` snapshot; each arm seeds from its own clone and /// its writes never reach the caller. var: serde_json::Value, @@ -177,13 +179,11 @@ struct ArmState<'a> { /// The join this arm reports to. fanout: FanoutId, /// The arm's 0-based collection index: its result slot and (plus one) - /// its `sys.index` and write-scope arm id. + /// its `sys.index`. item_index: usize, /// The arm's collection member: the `item` global and `{{ item }}` /// substitution seed for the worker entry. item: serde_json::Value, - /// This fanout's store-write token. - write_token: u64, /// True while the worker is the chain's current section: the worker /// entry gets the arm seeds, and a control transfer out of the worker /// resolves over the arm's visible set. Cleared by the first jump. @@ -319,6 +319,16 @@ struct Chain<'a> { /// The chain's fork of the run context: the run's own for the root /// chain, `with_args` for a call chain's input override. ctx: RunContext, + /// The chain's VFS access capability, installed into each section VM + /// the chain enters: the walk and the live H1 pass acquire their own, + /// a call chain borrows its parent's (a blocking child is the same + /// serial thread - no new identity, no false conflicts), and a fanout + /// arm spawns its own from the fanout caller's. `None` only after the + /// chain ends: the arena is append-only, so `finish` and + /// `abort_subtree` take the slot to release the identity's claims at + /// chain end rather than at scheduler drop - a fanout's join merge + /// must not meet a finished arm's lingering claims. + access: Option>, /// The per-section frame (VM, `sys`, conversation, counts): `Some` /// while a section is entered, `None` before the first entry and /// between sections. @@ -380,6 +390,19 @@ struct Chain<'a> { } impl Chain<'_> { + /// The chain's access capability for section-VM installation. A live + /// chain always holds one; `finish` and `abort_subtree` take it at + /// chain end. + /// + /// # Errors + /// Returns [`Error::Internal`] when the chain's capability is gone, + /// which only the chain-end paths do - a live chain always holds it. + fn access(&self) -> Result<&Arc> { + self.access + .as_ref() + .ok_or(Error::Internal("a live chain holds its access capability")) + } + /// The chain's current block sequence: the live H1 pass's blocks, or /// the current section's blocks on the walk. fn blocks(&self) -> &[Block] { @@ -650,6 +673,7 @@ impl<'a> Scheduler<'a> { ); self.chains.push(Chain { ctx, + access: None, frame: None, slice, index, @@ -676,15 +700,23 @@ impl<'a> Scheduler<'a> { /// Returns [`Error::Internal`] when the run's chain count exceeds `u32`. fn start_root_walk(&mut self, sections: &'a [Section], var: &serde_json::Value) -> Result<()> { let root = self.start_chain(self.ctx.clone(), sections, 0, None, var, 0, None)?; - // Seed the root chain's client slot from the run's configured - // client, as the legacy walk's slot is seeded from run()'s client: - // a prose block before any infer must use it rather than fall back - // to building an environment client. - self.chains[root.index()].client = self.client.ready().cloned(); + self.install_root_slots(root); self.ready.push_back(root); Ok(()) } + /// Seeds a fresh root walk chain's slots: its own access capability - + /// the walk is its own serial thread of execution, and a fresh acquire + /// (the H1 pass's identity ended with its chain) means nothing the pass + /// touched can false-conflict with the walk - and its client slot from + /// the run's configured client, as the legacy walk's slot is seeded + /// from run()'s client: a prose block before any infer must use it + /// rather than fall back to building an environment client. + fn install_root_slots(&mut self, root: ChainId) { + self.chains[root.index()].access = Some(Arc::new(self.ctx.vfs().acquire())); + self.chains[root.index()].client = self.client.ready().cloned(); + } + /// Starts the live H1 pass as the driver loop's first chain: the /// prompt's H1 blocks under its title, driven through the same /// coroutine machinery as any section under the live pass's rules. @@ -701,6 +733,7 @@ impl<'a> Scheduler<'a> { let client = self.client.ready().cloned(); self.chains.push(Chain { ctx: self.ctx.clone(), + access: Some(Arc::new(self.ctx.vfs().acquire())), frame: None, slice: &[], index: 0, @@ -737,6 +770,9 @@ impl<'a> Scheduler<'a> { }; let var = frame.read_var()?; drop(frame); + // The pass's chain ends here: release its capability (and with it + // the identity's claims) before the walk acquires its own. + chain.access = None; let sections = self.ctx.prompt().sections(); if sections.is_empty() { *root_result = Some(Ok(GENERIC_COMPLETION.to_owned())); @@ -747,7 +783,7 @@ impl<'a> Scheduler<'a> { let when = now_rfc3339_checked()?; let walk_ctx = self.ctx.with_walk_state(&when); let root = self.start_chain(walk_ctx, sections, 0, None, &var, 0, None)?; - self.chains[root.index()].client = self.client.ready().cloned(); + self.install_root_slots(root); self.ready.push_back(root); Ok(()) } @@ -769,7 +805,7 @@ impl<'a> Scheduler<'a> { // The live H1 pass enters its frame exactly once: id 0 under // the prompt's title, the control-stub surface, the shim base, // and no SECTION_STARTED - the pass is not a walked section. - let frame = SectionContext::new_live_h1(&chain.ctx)?; + let frame = SectionContext::new_live_h1(&chain.ctx, chain.access()?)?; chain.frame = Some(frame); chain.block = 0; return Ok(true); @@ -784,18 +820,17 @@ impl<'a> Scheduler<'a> { { let (worker_slice, worker_index) = (arm.worker_slice, arm.worker_index); let (caller_slice, caller_index) = (arm.caller_slice, arm.caller_index); - let (item_index, item, write_token) = - (arm.item_index, arm.item.clone(), arm.write_token); + let (item_index, item) = (arm.item_index, arm.item.clone()); let worker = &worker_slice[worker_index]; let caller = &caller_slice[caller_index]; let home = home_without(&visible_sections(caller_slice, caller), worker); let frame = SectionContext::new_fanout_arm( &chain.ctx, + chain.access()?, worker, &home, item_index, item, - write_token, &chain.var, )?; chain.frame = Some(frame); @@ -811,6 +846,7 @@ impl<'a> Scheduler<'a> { let slice = chain.slice; let frame = SectionContext::new( &chain.ctx, + chain.access()?, &slice[index], slice, next_id(chain.ctx.ids()), @@ -1935,6 +1971,10 @@ impl<'a> Scheduler<'a> { let args = input.unwrap_or_else(|| chain.ctx.args()).to_owned(); let child_ctx = chain.ctx.with_args(&args); let client = chain.client.clone(); + // A call chain is a blocking child: it borrows the caller's access + // capability (the same serial thread of execution), so the caller's + // standing claims never false-conflict with the child's ops. + let access = chain.access.clone(); // `chain`'s arena borrow ends here; the resolution borrows the // prompt tree, so the target's slice outlives it. let target_section = self.resolve_chain_target(id, target)?; @@ -1950,6 +1990,7 @@ impl<'a> Scheduler<'a> { // The child inherits the caller's client slot: an already-resolved // client is shared, an unresolved one stays lazy. self.chains[child.index()].client = client; + self.chains[child.index()].access = access; Ok(child) } @@ -2020,6 +2061,12 @@ impl<'a> Scheduler<'a> { }; let ctx = chain.ctx.clone(); let client = chain.client.clone(); + // The caller's capability: each arm spawns its own from it, so the + // spawn is the happens-before edge that retires the caller's claims. + let access = chain + .access + .clone() + .ok_or(Error::Internal("a live chain holds its access capability"))?; // `chain`'s arena borrow ends here; the resolution borrows the // prompt tree, so the worker's slice outlives it. let target = self.resolve_chain_target(id, worker_name)?; @@ -2057,7 +2104,7 @@ impl<'a> Scheduler<'a> { ctx.debug().cloned(), Arc::new(AtomicU32::new(0)), ), - write_token: ctx.store().next_write_token(), + access, var: var.clone(), call_depth: depth, client, @@ -2120,7 +2167,6 @@ impl<'a> Scheduler<'a> { fanout, item_index: index, item, - write_token: template.write_token, at_worker: true, caller_slice: template.caller_slice, caller_index: template.caller_index, @@ -2142,6 +2188,12 @@ impl<'a> Scheduler<'a> { template.call_depth, Some(arm), )?; + // The arm is a new concurrent thread of execution: its + // capability spawns from the fanout caller's, retiring the + // caller's claims (the happens-before edge), and drops with + // the chain so a finished arm's claims never linger into the + // join's merge. + self.chains[chain.index()].access = Some(Arc::new(template.access.spawn())); // The arm inherits the caller's client slot: an // already-resolved client is shared, an unresolved one stays // lazy. @@ -2334,6 +2386,7 @@ impl<'a> Scheduler<'a> { chain.coroutine = None; chain.incoming = None; chain.frame = None; + chain.access = None; chain.arm = None; } @@ -2356,6 +2409,12 @@ impl<'a> Scheduler<'a> { // `None` when the chain ended by exhausting its slice: the last // section's frame already dropped at the fall-through. let mut frame = chain.frame.take(); + // Taken now, dropped after the frame: the VM's store closures hold + // their own Arc clones of the capability, so the identity's claims + // release only when both are gone - at chain end, before a fanout + // join resumes the parent into its merge. A call chain's slot is a + // borrowed clone, so its drop never releases the parent's identity. + let access = chain.access.take(); // The live H1 pass never arms completion: SECTION_FINISHED is a // walked section's boundary, not the setup pass's. Its completion // paths (fall-through, scalar return) handle the frame themselves; @@ -2388,6 +2447,7 @@ impl<'a> Scheduler<'a> { }); // The frame drops here: the single teardown boundary. drop(frame); + drop(access); if let Some(arm) = arm { self.complete_arm(arm, outcome); return; diff --git a/crates/promptforge-core/src/execute/section_context.rs b/crates/promptforge-core/src/execute/section_context.rs index dd09dfa95..daa8b0951 100644 --- a/crates/promptforge-core/src/execute/section_context.rs +++ b/crates/promptforge-core/src/execute/section_context.rs @@ -26,7 +26,7 @@ use crate::debug::DebugCapture; use crate::lua::{ProseState, SectionVm, ToolBinding, ToolCallCounts, install_live_h1_shim_base}; use crate::observe::{Observer, detail}; use crate::parser::Section; -use crate::store::WriteScope; +use crate::store::Access; use crate::{Error, Result, subst}; use super::context::RunContext; @@ -113,6 +113,7 @@ impl SectionContext { /// the teardown boundary still fires exactly once on that path. pub(crate) fn new( ctx: &RunContext, + access: &Arc, section: &Section, siblings: &[Section], section_id: u64, @@ -152,9 +153,7 @@ impl SectionContext { var: Some(var), item: None, }, - // Walk-section store writes are untracked; only fanout arms - // carry a write scope. - None, + access, section.name(), ); // Setup runs on the bare VM so a failure tears it down here: the @@ -184,8 +183,7 @@ impl SectionContext { /// control-global stubs, and the live H1 shim base. /// /// H1 is the level-1 section: it runs first and is never re-entered, so - /// the frame seeds an empty `var`, no item, and no write - /// scope. The scheduler answers the pass's `models.infer` yields (with + /// the frame seeds an empty `var` and no item. The scheduler answers the pass's `models.infer` yields (with /// or without a leading handle) through its driver, so the shim base /// keeps the control stubs, /// which raise before anything structural can yield. @@ -195,7 +193,7 @@ impl SectionContext { /// construction or limits failure propagates bare, before any teardown /// observation exists; a setup failure tears the fresh VM down first, so /// the teardown boundary still fires exactly once on that path. - pub(crate) fn new_live_h1(ctx: &RunContext) -> Result { + pub(crate) fn new_live_h1(ctx: &RunContext, access: &Arc) -> Result { let title = ctx.prompt().title(); let now = now_rfc3339_checked()?; let sys = sys_json( @@ -215,7 +213,7 @@ impl SectionContext { )?; // Setup runs on the bare VM so a failure tears it down here: the // frame does not exist yet, so its `Drop` cannot own this path. - if let Err(error) = setup_live_h1(&mut vm, ctx, &sys, title) + if let Err(error) = setup_live_h1(&mut vm, ctx, access, &sys, title) .and_then(|()| install_live_h1_shim_base(vm.lua()).map_err(Error::from)) { vm.teardown(ctx.observer().as_ref(), title); @@ -243,9 +241,9 @@ impl SectionContext { /// visible set: its home slice plus its children; plus the yield /// shims), and the shared setup half. /// - /// The seed is the fanout's own: the collection `item`, the store-write - /// scope (this fanout's token plus the arm's index, matching - /// `sys.index`), and the caller's cloned `var`. The + /// The seed is the fanout's own: the collection `item`, the arm's + /// spawned access capability (its claims-model identity), and the + /// caller's cloned `var`. The /// effective reporting handles /// are the fanout's too: the run's own observer and debug sink with the /// fanout's fresh turn counter arrive through the context's fanout fork, @@ -261,11 +259,11 @@ impl SectionContext { /// once. pub(crate) fn new_fanout_arm( ctx: &RunContext, + access: &Arc, worker: &Section, home: &[Section], index: usize, item: serde_json::Value, - write_token: u64, var: &serde_json::Value, ) -> Result { let tool_set = ctx.tool_set_snapshot()?; @@ -304,9 +302,6 @@ impl SectionContext { } }; let item = Some(item); - // The arm's store-write identity: this fanout's token plus the - // arm's 1-based index, matching `sys.index`. - let write_scope = Some(WriteScope::new(write_token, index + 1)); // The `list_from_section` callback resolves over the worker's // visible set (its home slice plus its children); the suspending // calls are the yield shims the setup half installs. @@ -320,7 +315,7 @@ impl SectionContext { var: Some(var), item: item.as_ref(), }, - write_scope, + access, worker.name(), ); // Setup runs on the bare VM so a failure tears it down here: the @@ -497,10 +492,11 @@ fn install_section_scope( fn setup_live_h1( vm: &mut SectionVm, ctx: &RunContext, + access: &Arc, sys: &serde_json::Value, title: &str, ) -> Result<()> { - vm.inject_host(ctx.args(), sys, ctx.store())?; + vm.inject_host(ctx.args(), sys, access)?; vm.install_host_apis(ctx.observer(), title)?; vm.install_h1_control_stubs().map_err(Error::from) } diff --git a/crates/promptforge-core/src/execute/section_vm.rs b/crates/promptforge-core/src/execute/section_vm.rs index 4e58c0b8e..54180f612 100644 --- a/crates/promptforge-core/src/execute/section_vm.rs +++ b/crates/promptforge-core/src/execute/section_vm.rs @@ -25,7 +25,7 @@ use std::sync::Arc; use crate::lua::{LuaProgram, SectionVm}; use crate::observe::Observer; -use crate::store::{StoreRef, WriteScope}; +use crate::store::Access; use crate::{Error, Result}; /// What a section VM is seeded with beyond the shared host contract. @@ -53,14 +53,14 @@ pub(crate) struct SectionVmSetup<'a> { pub(crate) args: &'a str, /// The `sys` JSON the driver built for this section or arm. pub(crate) sys: &'a serde_json::Value, - /// The run-scoped store backing the Lua `store` table. - pub(crate) store: &'a StoreRef, + /// The chain step's VFS access capability backing the Lua `store` + /// table: the walk's own, a call chain's borrowed parent capability, or + /// a fanout arm's spawned one. The store closures share it, so every + /// store op is attributed to the chain step's identity. + pub(crate) access: &'a Arc, /// The driver-specific seed: the walk's `var`, plus the collection /// `item` for an arm. pub(crate) seed: VmSeed<'a>, - /// The fanout arm's store-write identity; `None` on the walk, whose - /// `store.write` calls stay untracked. - pub(crate) write_scope: Option, /// The observer `Arc`: the persistent host APIs (`log`, `store`) capture /// it, and the shared-library replay reports through it. pub(crate) observer_arc: &'a Arc, @@ -107,13 +107,7 @@ where if setup.ui.is_some() { vm.allow_raw_model_ids(); } - vm.inject_host_with_var( - setup.args, - setup.sys, - setup.store, - setup.seed.var, - setup.write_scope, - )?; + vm.inject_host_with_var(setup.args, setup.sys, setup.access, setup.seed.var)?; vm.install_host_apis(setup.observer_arc, setup.section_name)?; if let Some(provider) = setup.ui { crate::lua::install_ui(vm.lua(), Arc::clone(provider))?; diff --git a/crates/promptforge-core/src/execute/tests/debug_and_counts.rs b/crates/promptforge-core/src/execute/tests/debug_and_counts.rs index fec61495f..7b1ac826f 100644 --- a/crates/promptforge-core/src/execute/tests/debug_and_counts.rs +++ b/crates/promptforge-core/src/execute/tests/debug_and_counts.rs @@ -12,7 +12,7 @@ async fn debug_capture_receives_request_and_response_when_set() { &bound_for_model(md), "", &[], - &StoreRef::memory(), + &TestStore::new(), gatewayed_with_debug(addr, Arc::clone(&capture) as Arc), ) .await @@ -98,7 +98,7 @@ async fn nested_model_infer_capture_reaches_the_debug_sink() { &prompt, "", &[], - &StoreRef::memory(), + &TestStore::new(), gatewayed_with_debug(addr, Arc::clone(&capture) as Arc), ) .await @@ -148,7 +148,7 @@ async fn fanout_arm_debug_events_reach_the_run_sink() { &prompt, "", &[], - &StoreRef::memory(), + &TestStore::new(), gatewayed_with_debug(addr, Arc::clone(&capture) as Arc), ) .await @@ -184,7 +184,7 @@ async fn debug_capture_none_changes_nothing() { &bound_for_model(md), "", &[], - &StoreRef::memory(), + &TestStore::new(), gatewayed(addr), ) .await @@ -217,7 +217,7 @@ async fn tool_calls_count_increments_on_successful_dispatch() { &prompt, "", &[Arc::clone(&tool) as Arc], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await @@ -309,7 +309,7 @@ async fn tool_calls_count_zero_for_uncalled_alias_fails_epilog_assert() { Arc::new(search) as Arc, Arc::new(other) as Arc, ], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await @@ -338,7 +338,7 @@ async fn tool_calls_typo_alias_is_a_hard_error_with_seeded_set() { &prompt, "", &[Arc::new(tool) as Arc], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await @@ -543,7 +543,7 @@ async fn handle_infer_returns_text_without_touching_reply_or_sys() { return text\n\ ```\n"; let prompt = bound_with_tools(md, Vec::new()); - let out = run(&prompt, "", &[], &StoreRef::memory(), gatewayed(addr)) + let out = run(&prompt, "", &[], &TestStore::new(), gatewayed(addr)) .await .expect("handle-form infer must return text"); assert_eq!(out, "pong"); diff --git a/crates/promptforge-core/src/execute/tests/exec_flow.rs b/crates/promptforge-core/src/execute/tests/exec_flow.rs index 6f04310d8..c96d66264 100644 --- a/crates/promptforge-core/src/execute/tests/exec_flow.rs +++ b/crates/promptforge-core/src/execute/tests/exec_flow.rs @@ -29,7 +29,7 @@ First ask.\n\n\ Final ask.\n\n\ ```lua\nstore.append('order.txt', 'lua3\\n')\nreturn models.infer(prose)\n```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&bound_for_model(md), "", &[], &store, gatewayed(addr)) .await .expect("alternating blocks must execute"); @@ -67,7 +67,7 @@ store.write('evidence.md', answer)\n\ return answer\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&bound_for_model(md), "topic", &[], &store, gatewayed(addr)) .await .expect("call must run named section as subroutine"); @@ -95,7 +95,7 @@ return r\n\ Args: {{ args }}\n\n\ ```lua\nreturn models.infer(prose)\n```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run( &bound_for_model(md), "run-args", @@ -158,7 +158,7 @@ store.write('seen.txt', 'should-not-run')\n\ return 'helped:' .. store.read('seen.txt')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("jump must transfer control"); @@ -233,7 +233,7 @@ store.append('order.txt', 'S2\\n')\n\ return 's2-result'\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("the call chain must jump, fall through, and return its final text"); @@ -268,7 +268,7 @@ store.append('order.txt', 'S2\\n')\n\ return 's2-reply'\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("the chain must run the addressed off-walk target and fall through"); @@ -301,7 +301,7 @@ store.append('order.txt', 'Tail\\n')\n\ return 'tail-reply'\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("a jump inside the chain must move within the chain"); @@ -335,7 +335,7 @@ store.append('order.txt', 'Peer\\n')\n\ return 'p'\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("the outer walk must resume at the section after the caller"); @@ -368,7 +368,7 @@ return 'sub-reply'\n\ error('a return must end the chain before fall-through')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("a return must end the chain, not the run"); @@ -403,7 +403,7 @@ assert(sys.id == 3, 'the chain fall-through takes the next global id')\n\ return 'tail-reply'\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("a call chain must continue the global sys.id sequence"); @@ -590,7 +590,7 @@ Do work.\n\n\ - alpha\n\ - beta\n" ); - let out = run(&fixture(md), "", &[], &StoreRef::memory(), silent()) + let out = run(&fixture(md), "", &[], &TestStore::new(), silent()) .await .expect("fanout must return structured results"); assert_eq!(out, "ok"); @@ -642,7 +642,7 @@ store.append('order.txt', 'B\\n')\n\ return store.read('order.txt')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("a jump to a child must start the child-level walk"); @@ -684,7 +684,7 @@ store.append('order.txt', 'B\\n')\n\ return store.read('order.txt')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("the child-level rule must recurse to H4"); @@ -719,7 +719,7 @@ store.append('order.txt', 'Y\\n')\n\ return store.read('order.txt')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("a jump to an off-walk child must run it"); @@ -748,7 +748,7 @@ store.append('order.txt', 'After\\n')\n\ return 'after-reply'\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("call to a child must start a contained chain"); @@ -776,7 +776,7 @@ store.append('order.txt', 'B\\n')\n\ return store.read('order.txt')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("the walk must never descend into children"); @@ -812,7 +812,7 @@ store.append('order.txt', 'Y\\n')\n\ return store.read('order.txt')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("a running child must address its own siblings and children"); @@ -899,7 +899,7 @@ store.append('ids.txt', tostring(sys.id) .. '\\n')\n\ return store.read('ids.txt')\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("sys.id must count sections entered run-wide"); @@ -1257,7 +1257,7 @@ return 'tail-reply'\n\ ```\n", ] .concat(); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(&md), "", &[], &store, silent()) .await .expect("a jump inside an arm must drive a child walk"); @@ -1295,7 +1295,7 @@ return 'child-tail-reply'\n\ ```\n", ] .concat(); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(&md), "", &[], &store, silent()) .await .expect("a jump to a worker child must drive the child slice"); @@ -1323,7 +1323,7 @@ store.append('order.txt', 'Target\\n')\n\ ```\n", ] .concat(); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(&md), "", &[], &store, silent()) .await .expect("a jump to a silent chain must succeed with empty text"); @@ -1497,7 +1497,7 @@ return models.infer(models.get('writer'), 'ping about ' .. item)\n\ &bound_for_model(&md), "", &[], - &StoreRef::memory(), + &TestStore::new(), gatewayed(addr), ) .await @@ -1535,15 +1535,9 @@ return models.infer(models.get('writer'), 'ping about ' .. item)\n\ ```\n", ] .concat(); - let error = run( - &bound_for_model(&md), - "", - &[], - &StoreRef::memory(), - silent(), - ) - .await - .expect_err("handle infer in an arm with no client must surface the lazy error"); + let error = run(&bound_for_model(&md), "", &[], &TestStore::new(), silent()) + .await + .expect_err("handle infer in an arm with no client must surface the lazy error"); let rendered = error.to_string(); assert!( rendered.contains("missing environment variable: PROMPTFORGE_GATEWAY"), @@ -1562,15 +1556,9 @@ return models.infer(models.get('ghost'), 'ping')\n\ ```\n", ] .concat(); - let error = run( - &bound_for_model(&md), - "", - &[], - &StoreRef::memory(), - silent(), - ) - .await - .expect_err("an unknown model alias inside an arm must fail loudly"); + let error = run(&bound_for_model(&md), "", &[], &TestStore::new(), silent()) + .await + .expect_err("an unknown model alias inside an arm must fail loudly"); let rendered = error.to_string(); assert!( rendered.contains("models.get alias \"ghost\" was not declared"), @@ -1672,7 +1660,7 @@ store.append('order.txt', 'Sub3\\n')\n\ return 'sub3-reply'\n\ ```\n" ); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()) .await .expect("call to a later child must run the child slice from that index"); @@ -2297,3 +2285,42 @@ fn now_rfc3339_checked_produces_a_parseable_timestamp() { "RFC 3339 UTC has a zone designator: {now}" ); } + +#[tokio::test] +async fn a_mount_less_handle_runs_on_the_defensive_store_overlay() { + // The defensive fallback in `run`: a hand-built VfsRef lacking the store + // mount gets a fresh memory store overlaid for the run, so the run's + // store writes land on the overlay (readable across sections) instead of + // failing for want of the mount, and the caller's backend stays + // untouched. + let md = flow_prompt!( + "# Test prompt\n\n\ + ## First\n\n```lua\nstore.write('overlay.txt', 'overlaid')\n```\n\n\ + ## Second\n\n```lua\nreturn store.read('overlay.txt')\n```\n" + ); + let test = fixture(md); + let vfs = VfsRef::new(shared_vfs::MemoryBackend::new()); + let picker = empty_test_picker(); + let out = crate::execute::run( + &test.prompt, + "", + ResolutionContext::new(&picker, &test.models, &ToolCatalog::default()), + &vfs, + RunConfig::new(EXECUTION), + ) + .await + .expect("a mount-less handle gets the defensive memory-store overlay"); + assert_eq!( + out, "overlaid", + "the first section's write must be readable from the overlaid store" + ); + // The overlay is throwaway: the caller's backend never gains the mount + // or the run's writes. + assert!( + matches!( + vfs.acquire().stat(promptforge_vfs::STORE_MOUNT), + Err(shared_vfs::VfsError::NotFound(_)) + ), + "the run's writes must land on the overlay, not the caller's backend" + ); +} diff --git a/crates/promptforge-core/src/execute/tests/exit_rules.rs b/crates/promptforge-core/src/execute/tests/exit_rules.rs index cf76a16d0..0e681c016 100644 --- a/crates/promptforge-core/src/execute/tests/exit_rules.rs +++ b/crates/promptforge-core/src/execute/tests/exit_rules.rs @@ -121,7 +121,7 @@ async fn store_persists_across_sections() { let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ ## Writer\n\n```lua\nstore.write('note.txt', 'carried across')\n```\n\n\ ## Reader\n\n```lua\nvar.seen = store.read('note.txt')\nreturn var.seen\n```\n"; - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()).await.unwrap(); assert_eq!( out, "carried across", diff --git a/crates/promptforge-core/src/execute/tests/input.rs b/crates/promptforge-core/src/execute/tests/input.rs index 842b07649..e74f455f7 100644 --- a/crates/promptforge-core/src/execute/tests/input.rs +++ b/crates/promptforge-core/src/execute/tests/input.rs @@ -37,7 +37,7 @@ fn input_context(prompt: &Prompt, tools: ToolSet, config: &RunConfig) -> RunCont let ctx = RunContext::new( prompt, "", - &StoreRef::memory(), + &TestStore::new(), LuaProgram::empty().expect("the empty chunk compiles"), config, ); diff --git a/crates/promptforge-core/src/execute/tests/live_infer.rs b/crates/promptforge-core/src/execute/tests/live_infer.rs index 9690b5810..cd8167000 100644 --- a/crates/promptforge-core/src/execute/tests/live_infer.rs +++ b/crates/promptforge-core/src/execute/tests/live_infer.rs @@ -21,7 +21,7 @@ async fn live_h1_infer_runs_once() { &prompt, "", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), to_config(gatewayed(addr)), ) .await @@ -43,7 +43,7 @@ async fn unread_h1_prose_stays_inert_and_explicit_infer_requires_a_model() { {{ var.omit }}\n\n\ ## Result\n\n\ ```lua\nreturn 'ok'\n```\n"; - let out = super::run(&fixture(unread), "", &[], &StoreRef::memory(), silent()) + let out = super::run(&fixture(unread), "", &[], &TestStore::new(), silent()) .await .expect("unread H1 prose must not require a model"); assert_eq!(out, "ok"); @@ -52,7 +52,7 @@ async fn unread_h1_prose_stays_inert_and_explicit_infer_requires_a_model() { # Read H1\n\n\ ask\n\n\ ```lua\nreturn models.infer(prose)\n```\n"; - let error = super::run(&fixture(reading), "", &[], &StoreRef::memory(), silent()) + let error = super::run(&fixture(reading), "", &[], &TestStore::new(), silent()) .await .expect_err("an explicit infer of H1 prose with no binding must fail"); assert!( @@ -72,7 +72,7 @@ async fn caught_h1_callback_error_stops_before_a_later_block() { ```lua\nstore.write('later.txt', 'ran')\n```\n\n\ ## Result\n\n\ ```lua\nreturn 'unexpected'\n```\n"; - let store = StoreRef::memory(); + let store = TestStore::new(); let error = super::run(&fixture(source), "", &[], &store, silent()) .await .expect_err("a caught resolver callback error must fail its own block"); @@ -102,7 +102,7 @@ async fn shared_function_resolves_host_globals_when_called() { &prompt, "later host value", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), to_config(silent()), ) .await @@ -118,7 +118,7 @@ async fn shared_library_calls_host_apis_at_load_time() { // `log`, and `args` at load. let picker = empty_test_picker(); let models = test_model_catalog(); - let store = StoreRef::memory(); + let store = TestStore::new(); let source = "---\nname: shared-host-load\ndescription: d\npromptforge: 0\n---\n\n\ # Shared Host Load\n\n\ ```lua shared\n\ @@ -191,7 +191,7 @@ async fn captured_bindings_reach_section_call_and_fanout_vms() { &prompt, "", ResolutionContext::new(&picker, &models, &catalog), - &StoreRef::memory(), + &TestStore::new(), to_config(silent()), ) .await @@ -225,7 +225,7 @@ async fn live_h1_models_infer_resolves_the_default_model_without_touching_sys() &prompt, "", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), to_config(gatewayed(gateway.addr())), ) .await @@ -275,7 +275,7 @@ async fn nested_lua_infer_emits_a_model_turn_observation() { &prompt, "", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), to_config(RunOptions { execution: EXECUTION, observer: Arc::clone(&recorder) as Arc, @@ -339,7 +339,7 @@ async fn cancelled_nested_infer_does_not_report_model_turn_failed() { &prompt, "", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), RunConfig::new(EXECUTION) .observer(Arc::clone(&recorder) as Arc) .client(gateway_client(gateway.addr())) @@ -379,7 +379,7 @@ async fn handle_infer_tool_call_violation_uses_entry_point_neutral_wording() { &bound_for_model(source), "", &[], - &StoreRef::memory(), + &TestStore::new(), gatewayed(gateway.addr()), ) .await @@ -423,7 +423,7 @@ async fn live_h1_prose_infers_explicitly_and_var_accumulates_into_the_walk() { &prompt, "", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), to_config(gatewayed(addr)), ) .await @@ -459,7 +459,7 @@ async fn h1_and_h2_prose_each_infer_explicitly_in_source_order() { &prompt, "", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), to_config(gatewayed(gateway.addr())), ) .await @@ -509,7 +509,7 @@ async fn live_h1_chunk_keeps_sys_id_zero_and_the_first_walked_section_takes_one( &prompt, "", ResolutionContext::new(&picker, &models, &ToolCatalog::default()), - &StoreRef::memory(), + &TestStore::new(), to_config(silent()), ) .await diff --git a/crates/promptforge-core/src/execute/tests/local_tools.rs b/crates/promptforge-core/src/execute/tests/local_tools.rs index 813ec0631..79f3c980f 100644 --- a/crates/promptforge-core/src/execute/tests/local_tools.rs +++ b/crates/promptforge-core/src/execute/tests/local_tools.rs @@ -248,7 +248,7 @@ tools.add_local('grab', 'Local grab', {}, function() return 'local' end)\n\ &prompt, "", &[tool as Arc], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await diff --git a/crates/promptforge-core/src/execute/tests/mod.rs b/crates/promptforge-core/src/execute/tests/mod.rs index b4b924e00..190021129 100644 --- a/crates/promptforge-core/src/execute/tests/mod.rs +++ b/crates/promptforge-core/src/execute/tests/mod.rs @@ -21,9 +21,16 @@ use crate::debug::DebugCapture; use crate::lua::{LuaProgram, current_tool_bindings}; use crate::model::{CompletionOptions, ModelCatalog, ModelDescriptor, ModelId, ThinkingMode}; use crate::observe::{NullObserver, Observation, detail}; +use crate::store::{Access, StoreError, StoreExt, VfsRef}; use crate::tools::{Tool, ToolCatalog, ToolError, ToolErrorKind, ToolId, ToolOutput}; use crate::untrusted::GuardNonce; +/// A fresh stock handle's access capability, for tests that inject host +/// values into a standalone VM. +fn fresh_access() -> Arc { + Arc::new(promptforge_vfs::empty().acquire()) +} + const EXECUTION: &str = "execute-test"; /// F10: compile-time proof that the public execution types are thread-safe. @@ -185,6 +192,43 @@ struct RunOptions { debug: Option>, } +/// The test stand-in for the old `StoreRef::memory()`: a stock VFS handle +/// (the store mount preinstalled) whose `read`/`write` helpers each go +/// through a fresh, immediately dropped access. A short-lived access per +/// call is what keeps seeding and post-run assertions conflict-free: the +/// claims model attributes every operation to a live identity, so a held +/// seeder access would meet the run's own identities as a false race. +struct TestStore(VfsRef); + +impl std::ops::Deref for TestStore { + type Target = VfsRef; + + fn deref(&self) -> &VfsRef { + &self.0 + } +} + +impl TestStore { + fn new() -> TestStore { + TestStore(promptforge_vfs::empty()) + } + + /// The handle the run and the context builders take. + fn vfs(&self) -> &VfsRef { + &self.0 + } + + fn read(&self, path: &str) -> std::result::Result { + let access = self.0.acquire(); + self.0.store(&access).read(path) + } + + fn glob(&self, pattern: &str) -> std::result::Result, StoreError> { + let access = self.0.acquire(); + self.0.store(&access).glob(pattern) + } +} + /// Builds a [`RunConfig`] from the test-local [`RunOptions`], for the tests that /// call [`super::run`] directly with a custom picker and model catalog. fn to_config(opts: RunOptions) -> RunConfig { @@ -248,14 +292,14 @@ fn gateway_env_is_unset() -> bool { /// in-memory store created for the run - the ergonomic path for the /// Lua-only tests that do not care about the store's contents. async fn run_offline(md: &str) -> Result { - run(&fixture(md), "", &[], &StoreRef::memory(), silent()).await + run(&fixture(md), "", &[], &TestStore::new(), silent()).await } async fn run( test: &TestPrompt, args: &str, tools: &[Arc], - store: &StoreRef, + store: &TestStore, opts: RunOptions, ) -> Result { let catalog = test.picker_catalog.clone().unwrap_or_else(|| { @@ -290,7 +334,7 @@ async fn run( &test.prompt, args, ResolutionContext::new(&picker, &test.models, &tool_catalog), - store, + store.vfs(), run_config, ) .await @@ -339,7 +383,7 @@ async fn run_with_config( &test.prompt, "", ResolutionContext::new(&picker, &test.models, &ToolCatalog::default()), - &StoreRef::memory(), + TestStore::new().vfs(), configure(RunConfig::new(EXECUTION)), ) .await @@ -387,7 +431,7 @@ async fn run_recorded(md: &str) -> (Result, Vec<(String, String, String) &fixture(md), "", &[], - &StoreRef::memory(), + &TestStore::new(), RunOptions { execution: EXECUTION, observer: Arc::clone(&recorder) as Arc, @@ -1086,7 +1130,7 @@ fn tool_description_override_appears_in_model_schema() { .expect("captured bindings must install"); vm.install_captured_bindings() .expect("alias globals must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); // tools.add(alias) with no override keeps the bound tool's catalog text. @@ -1160,7 +1204,7 @@ fn bind_override_reaches_the_schema_and_add_beats_bind() { .expect("captured bindings must install"); vm.install_captured_bindings() .expect("alias globals must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); let add_plain = LuaProgram::compile( @@ -1679,7 +1723,7 @@ async fn untrusted_nonce_differs_across_runs() { &bound_with_tools(md, Vec::new()), "", &[Arc::new(UntrustedEchoTool) as Arc], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await diff --git a/crates/promptforge-core/src/execute/tests/model_and_reply.rs b/crates/promptforge-core/src/execute/tests/model_and_reply.rs index 0af574a4d..66b856e7d 100644 --- a/crates/promptforge-core/src/execute/tests/model_and_reply.rs +++ b/crates/promptforge-core/src/execute/tests/model_and_reply.rs @@ -32,7 +32,7 @@ Ask the model.\n\n\ picker_catalog: None, }; - let out = run(&prompt, "", &[], &StoreRef::memory(), gatewayed(addr)) + let out = run(&prompt, "", &[], &TestStore::new(), gatewayed(addr)) .await .unwrap(); assert_eq!(out, "hello from the mock"); @@ -60,7 +60,7 @@ async fn an_explicit_client_is_used_instead_of_the_environment() { &bound_for_model(md), "", &[], - &StoreRef::memory(), + &TestStore::new(), RunOptions { execution: EXECUTION, observer: Arc::clone(&recorder) as Arc, @@ -132,7 +132,7 @@ return 'epilog result'\n\ assert!(entry.epilog().is_some()); let recorder = Arc::new(Recorder::default()); - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run( &prompt, "", @@ -205,7 +205,7 @@ async fn add_without_h1_bindings_fails_the_run_loudly() { # Test prompt\n\n\ ## Only\n\n```lua\ntools.add('web_search')\n```\n\nThis prose must not reach a model.\n"; let prompt = fixture(md); - let error = run(&prompt, "", &[], &StoreRef::memory(), silent()) + let error = run(&prompt, "", &[], &TestStore::new(), silent()) .await .expect_err("an undeclared alias must fail the run"); assert!( @@ -222,7 +222,7 @@ async fn add_with_an_empty_shared_library_fails_the_run_loudly() { # Test prompt\n\n\ ```lua\nfunction helper() return 'no declarations' end\n```\n\n\ ## Only\n\n```lua\ntools.add('web_search')\n```\n\nThis prose must not reach a model.\n"; - let error = run(&fixture(md), "", &[], &StoreRef::memory(), silent()) + let error = run(&fixture(md), "", &[], &TestStore::new(), silent()) .await .expect_err("an undeclared alias must fail the run"); assert!( @@ -238,7 +238,7 @@ async fn prologue_return_skips_model_and_epilog() { ## Only\n\n```lua\nreturn 'early'\n```\n\n\ This prose must not reach a model.\n\n\ ```lua\nstore.write('epilog-ran.txt', 'yes')\nreturn 'late'\n```\n"; - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run(&fixture(md), "", &[], &store, silent()).await.unwrap(); assert_eq!(out, "early"); @@ -260,7 +260,7 @@ Ask using {{ var.question }}.\n\n\ &bound_for_model(md), "input", &[], - &StoreRef::memory(), + &TestStore::new(), RunOptions { execution: EXECUTION, observer: Arc::clone(&recorder) as Arc, @@ -325,7 +325,7 @@ async fn empty_prose_skips_model_but_runs_epilog_with_nil_reply() { ```lua\nif reply ~= nil then error('empty prose must not bind a reply') end\nreturn var.phase .. '-epilog'\n```\n"; assert_eq!( - run(&fixture(md), "", &[], &StoreRef::memory(), silent()) + run(&fixture(md), "", &[], &TestStore::new(), silent()) .await .unwrap(), "prologue-epilog" @@ -339,7 +339,7 @@ async fn whitespace_only_prose_skips_model_without_binding() { ## Only\n\n```lua\n-- prologue\n```\n\n \n\t\n\n\ ```lua\nif reply ~= nil then error('whitespace prose must not bind a reply') end\nreturn 'ok'\n```\n"; assert_eq!( - run(&fixture(md), "", &[], &StoreRef::memory(), silent()) + run(&fixture(md), "", &[], &TestStore::new(), silent()) .await .unwrap(), "ok" @@ -352,7 +352,7 @@ async fn model_required_when_infer_has_no_binding() { // of it does. let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ ## Only\n\nAsk the model.\n\n```lua\nreturn models.infer(prose)\n```\n"; - let error = run(&fixture(md), "", &[], &StoreRef::memory(), silent()) + let error = run(&fixture(md), "", &[], &TestStore::new(), silent()) .await .expect_err("an explicit infer without a model binding must fail"); assert!( @@ -374,7 +374,7 @@ async fn shared_function_sees_sys_model_unknown_before_scope_close() { ```lua\nmodels.default('writer', 'A general model for tests')\n```\n\n\ ```lua shared\nfunction read_sys_model()\n return sys.model\nend\n```\n\n\ ## Only\n\n```lua\nreturn read_sys_model()\n```\n\nprose\n"; - let error = run(&bound_for_model(md), "", &[], &StoreRef::memory(), silent()) + let error = run(&bound_for_model(md), "", &[], &TestStore::new(), silent()) .await .expect_err("shared function must not read sys.model before scope close"); assert!( @@ -387,7 +387,7 @@ async fn shared_function_sees_sys_model_unknown_before_scope_close() { async fn prologue_sys_model_unknown_before_scope_close() { let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ ## Only\n\n```lua\nreturn sys.model\n```\n\nprose\n"; - let error = run(&bound_for_model(md), "", &[], &StoreRef::memory(), silent()) + let error = run(&bound_for_model(md), "", &[], &TestStore::new(), silent()) .await .expect_err("prologue must not read sys.model before scope close"); assert!( @@ -412,7 +412,7 @@ models.default('writer', 'A general model for tests')\n```\n\n\ &prompt, "", &[Arc::new(EchoTool) as Arc], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await @@ -432,7 +432,7 @@ models.default('writer', 'A general model for tests')\n```\n\n\ &prompt, "", &[Arc::new(EchoTool) as Arc], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await @@ -455,7 +455,7 @@ models.default('writer', 'A general model for tests')\n```\n\n\ &prompt, "", &[Arc::new(EchoTool) as Arc], - &StoreRef::memory(), + &TestStore::new(), silent(), ) .await @@ -477,7 +477,7 @@ async fn fanout_item_substitution_renders_a_table_member_as_compact_json() { &bound_for_model(md), "", &[], - &StoreRef::memory(), + &TestStore::new(), gatewayed(addr), ) .await @@ -557,7 +557,11 @@ fn analyst_only_catalog() -> ModelCatalog { } /// Run a parsed prompt against a scripted gateway with no external tools. -async fn run_with_gateway(test: &TestPrompt, addr: SocketAddr, store: &StoreRef) -> Result { +async fn run_with_gateway( + test: &TestPrompt, + addr: SocketAddr, + store: &TestStore, +) -> Result { run(test, "", &[], store, gatewayed(addr)).await } @@ -580,7 +584,7 @@ Ask the model.\n\n\ models: writer_and_analyst_catalog(), picker_catalog: None, }; - let store = StoreRef::memory(); + let store = TestStore::new(); let out = run_with_gateway(&prompt, addr, &store).await.unwrap(); assert_eq!(out, "hello from the mock"); @@ -606,7 +610,7 @@ async fn models_infer_uses_the_section_model_without_touching_reply() { ## Only\n\n\ ```lua\nvar.r = models.infer('ping')\n```\n\n\ ```lua\nreturn var.r .. ':' .. tostring(reply)\n```\n"; - let out = run_with_gateway(&bound_for_model(md), addr, &StoreRef::memory()) + let out = run_with_gateway(&bound_for_model(md), addr, &TestStore::new()) .await .unwrap(); assert_eq!( @@ -646,7 +650,7 @@ models.bind('analyst', 'A careful analysis model')\n\ models: writer_and_analyst_catalog(), picker_catalog: None, }; - let out = run_with_gateway(&prompt, addr, &StoreRef::memory()) + let out = run_with_gateway(&prompt, addr, &TestStore::new()) .await .unwrap(); assert_eq!(out, "pong"); @@ -681,7 +685,7 @@ return models.infer('ping')\n\ models: writer_and_analyst_catalog(), picker_catalog: None, }; - let out = run_with_gateway(&prompt, addr, &StoreRef::memory()) + let out = run_with_gateway(&prompt, addr, &TestStore::new()) .await .expect("re-selection within a section must succeed"); assert_eq!(out, "second"); @@ -713,7 +717,7 @@ async fn models_infer_without_use_or_default_errors() { models: analyst_only_catalog(), picker_catalog: None, }; - let error = run(&prompt, "", &[], &StoreRef::memory(), silent()) + let error = run(&prompt, "", &[], &TestStore::new(), silent()) .await .expect_err("models.infer with no current model must fail"); assert!( @@ -738,7 +742,7 @@ async fn models_get_infer_works_without_any_section_model() { models: analyst_only_catalog(), picker_catalog: None, }; - let out = run_with_gateway(&prompt, addr, &StoreRef::memory()) + let out = run_with_gateway(&prompt, addr, &TestStore::new()) .await .unwrap(); assert_eq!(out, "pong"); diff --git a/crates/promptforge-core/src/execute/tests/models_loop.rs b/crates/promptforge-core/src/execute/tests/models_loop.rs index e813609e6..00be0b043 100644 --- a/crates/promptforge-core/src/execute/tests/models_loop.rs +++ b/crates/promptforge-core/src/execute/tests/models_loop.rs @@ -44,7 +44,7 @@ fn loop_context(prompt: &Prompt, tools: ToolSet) -> RunContext { let ctx = RunContext::new( prompt, "", - &StoreRef::memory(), + &TestStore::new(), LuaProgram::empty().expect("the empty chunk compiles"), &RunConfig::new(EXECUTION), ); diff --git a/crates/promptforge-core/src/execute/tests/observations.rs b/crates/promptforge-core/src/execute/tests/observations.rs index 0615c40ad..b6220d5e4 100644 --- a/crates/promptforge-core/src/execute/tests/observations.rs +++ b/crates/promptforge-core/src/execute/tests/observations.rs @@ -77,7 +77,7 @@ async fn a_two_section_run_reports_the_exact_observation_sequence() { #[tokio::test] async fn recording_and_null_observers_produce_the_same_result_and_store_state() { let prompt = fixture(STORE_SECTIONS); - let recorded_store = StoreRef::memory(); + let recorded_store = TestStore::new(); let sink = Arc::new(Recorder::default()); let observed_result = run( &prompt, @@ -92,7 +92,7 @@ async fn recording_and_null_observers_produce_the_same_result_and_store_state() }, ) .await; - let null_store = StoreRef::memory(); + let null_store = TestStore::new(); let null_result = run(&prompt, "", &[], &null_store, silent()).await; assert_eq!(observed_result.unwrap(), null_result.unwrap()); @@ -113,7 +113,7 @@ async fn recording_and_null_observers_produce_the_same_result_and_store_state() &failing, "", &[], - &StoreRef::memory(), + &TestStore::new(), RunOptions { execution: EXECUTION, observer: Arc::clone(&sink) as Arc, @@ -123,7 +123,7 @@ async fn recording_and_null_observers_produce_the_same_result_and_store_state() ) .await .expect_err("the prologue fails"); - let null_error = run(&failing, "", &[], &StoreRef::memory(), silent()) + let null_error = run(&failing, "", &[], &TestStore::new(), silent()) .await .expect_err("the prologue fails"); assert_eq!( @@ -314,7 +314,7 @@ async fn one_execution_id_spans_parse_and_the_complete_runtime_lifecycle() { models: test_model_catalog(), picker_catalog: Some(Catalog::new(vec![descriptor])), }; - let store = StoreRef::memory(); + let store = TestStore::new(); let result = run( &prompt, diff --git a/crates/promptforge-core/src/execute/tests/scheduler.rs b/crates/promptforge-core/src/execute/tests/scheduler.rs index a40555589..f7dfa8acf 100644 --- a/crates/promptforge-core/src/execute/tests/scheduler.rs +++ b/crates/promptforge-core/src/execute/tests/scheduler.rs @@ -7,8 +7,9 @@ //! and child descents with the parent resuming after the jumper), the //! scalar return's chain scoping, and the section-boundary observations. //! The fanout coverage mirrors the legacy engine's mechanics (ordering, -//! the concurrency window, interleaving) and its failure semantics (the -//! store write-write race as a hard error, unordered-legal appends, the +//! the concurrency window, interleaving) and its failure semantics under +//! the claims model (a live cross-arm write conflict as a hard error, +//! sequential-arm appends staying legal, the //! fatal-arm sibling abort, the pre-scheduling guards, and cancellation //! while suspended in an arm). @@ -42,24 +43,20 @@ fn writer_models() -> ModelSet { /// Builds the run context for a scheduler test: the parsed prompt, an empty /// shared library, and the model set pre-filled. fn scheduler_context(prompt: &Prompt) -> RunContext { - scheduler_context_on( - prompt, - &StoreRef::memory(), - Arc::new(NullObserver::default()), - ) + scheduler_context_on(prompt, &TestStore::new(), Arc::new(NullObserver::default())) } /// Builds the run context on the given store and observer, so a walk test /// can inspect the store's contents and the observation stream afterward. fn scheduler_context_on( prompt: &Prompt, - store: &StoreRef, + store: &TestStore, observer: Arc, ) -> RunContext { let ctx = RunContext::new( prompt, "", - store, + store.vfs(), LuaProgram::empty().expect("the empty chunk compiles"), &RunConfig::new(EXECUTION).observer(observer), ); @@ -255,7 +252,7 @@ async fn sections_run_in_fall_through_order() { // Mirror of the legacy `falls_through_to_next_section`, strengthened // with an order log: a section without a return falls through to the // next section in document order. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: walk\ndescription: d\npromptforge: 0\n---\n\n\ # Walk\n\n\ ## First\n\n\ @@ -317,7 +314,7 @@ async fn call_chain_over_off_walk_siblings_returns_to_the_caller() { // S1, which runs because it is addressed; the chain falls through to // S2, and S2's reply returns to A. The main walk ends at B and never // runs S1 or S2. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Siblings\n\n\ ## A\n\n\ @@ -410,7 +407,7 @@ async fn a_call_chain_continues_the_global_sys_id_sequence() { // Mirror of the legacy case of the same name: the contained chain's // entries take the next run-global ids, and the outer walk resumes the // same sequence when the chain ends. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Sequence\n\n\ ## Main\n\n\ @@ -481,7 +478,7 @@ async fn fall_through_fires_section_finished_before_the_next_section_starts() { ## Two\n\n\ ```lua\nreturn 'two-ran'\n```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); let out = Scheduler::new(&ctx, None) .drive() .await @@ -516,7 +513,7 @@ async fn jump_transfer_skips_the_jumpers_remaining_blocks() { // `jump_target_sees_no_prior_reply_and_transfer_skips_remaining_blocks`: // the jump transfers control and the jumper's remaining blocks never // run. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Jump\n\n\ ## Check\n\n\ @@ -638,7 +635,7 @@ async fn a_jump_fires_section_finished_for_the_jumper_before_the_target_starts() ## B\n\n\ ```lua\nreturn 'b-ran'\n```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); let out = Scheduler::new(&ctx, None) .drive() .await @@ -674,7 +671,7 @@ async fn an_erroring_section_reports_started_but_not_finished() { ## Only\n\n\ ```lua\nerror('expected failure')\n```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); let result = Scheduler::new(&ctx, None).drive().await; assert!(result.is_err()); @@ -697,7 +694,7 @@ async fn jump_to_a_child_starts_the_child_level_walk() { // starts a child-level walk at the target, which falls through to the // target's following siblings; when the level exhausts, the parent walk // resumes after the jumper. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Descend\n\n\ ## A\n\n\ @@ -730,7 +727,7 @@ async fn child_walk_recurses_to_h4() { // recurses - a jump from an H3 child to an H4 grandchild starts an // H4-level walk, and each level's exhaustion resumes its parent after // the jumper. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Recurse\n\n\ ## A\n\n\ @@ -769,7 +766,7 @@ async fn jump_to_an_off_walk_child_runs_it() { // Mirror of the legacy case of the same name: an off-walk child stays // addressable - a jump to it runs it, and the fall-through that follows // skips nothing addressed. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # OffChild\n\n\ ## A\n\n\ @@ -798,7 +795,7 @@ async fn running_child_addresses_its_own_siblings_and_children() { // Mirror of the legacy case of the same name: a running child's visible // set is its own siblings plus its own children - it can execute a // child and jump to a sibling. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Visible\n\n\ ## A\n\n\ @@ -883,7 +880,7 @@ async fn sys_id_counts_sections_entered_run_wide() { // Mirror of the legacy case of the same name: `sys.id` counts the // sections the walk has entered run-wide - the detour into a child // level continues the count rather than restarting it. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Ids\n\n\ ## A\n\n\ @@ -968,7 +965,7 @@ async fn jump_inside_a_call_chain_moves_within_the_chain() { // `call()` chain to a sibling moves within the contained chain - the // walk continues from the jump target under the normal rules, and the // chain's final reply is the call's return value. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Move\n\n\ ## A\n\n\ @@ -1004,7 +1001,7 @@ async fn call_chain_jumps_to_a_child_and_returns_the_chain_result() { // starting a child-level walk that falls through to S2; S2's return is // the chain's final text back to A, and the outer walk continues at B, // never having moved. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Chain\n\n\ ## A\n\n\ @@ -1046,7 +1043,7 @@ async fn the_outer_walk_never_moves_during_a_contained_chain() { // Mirror of the legacy case of the same name: the outer walk never // moves while a contained chain runs - wherever the chain ends, the // outer walk resumes at the section after the caller. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Outer\n\n\ ## A\n\n\ @@ -1082,7 +1079,7 @@ async fn a_return_inside_a_chain_ends_the_chain_not_the_run() { // contained chain ends the chain, not the run - the returned value is // the call's return, the chain's remaining sections do not run, and // the outer walk continues. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Scoped\n\n\ ## A\n\n\ @@ -1115,7 +1112,7 @@ async fn call_to_a_child_starts_a_contained_chain() { // starts a contained chain at the target - the chain falls through to // the target's following siblings under the same rules as any walk, and // the chain's final reply is the call's return value. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # ChildExecute\n\n\ ## Main\n\n\ @@ -1150,7 +1147,7 @@ async fn a_jump_descent_does_not_consume_call_depth() { // the ninth nested call would run (depth 9 > 8), after exactly nine // section entries - a descent that wrongly consumed depth would trip // the cap one entry earlier. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Depth\n\n\ ## Main\n\n\ @@ -1189,7 +1186,7 @@ async fn walk_never_descends_into_children() { // a section's children do not run unless addressed. This is the // negative half of the child-descent rule: a fall-through that // descended would run the child and trip its error. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # NoDescent\n\n\ ## A\n\n\ @@ -1223,7 +1220,7 @@ async fn a_failed_jump_resolution_still_finishes_the_jumper() { ## A\n\n\ ```lua\njump('## Missing')\n```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); let result = Scheduler::new(&ctx, None).drive().await; let error = result.expect_err("an unresolvable jump target must fail the run"); @@ -1244,21 +1241,17 @@ async fn a_failed_jump_resolution_still_finishes_the_jumper() { /// set starts empty - the live H1 pass under test records its own /// bindings, exactly as the legacy run's H1 hand-off leaves them. fn h1_context(prompt: &Prompt) -> RunContext { - h1_context_on( - prompt, - &StoreRef::memory(), - Arc::new(NullObserver::default()), - ) + h1_context_on(prompt, &TestStore::new(), Arc::new(NullObserver::default())) } /// Builds the H1 run context on the given store and observer, so a pass /// test can inspect the store's contents and the observation stream /// afterward. -fn h1_context_on(prompt: &Prompt, store: &StoreRef, observer: Arc) -> RunContext { +fn h1_context_on(prompt: &Prompt, store: &TestStore, observer: Arc) -> RunContext { RunContext::new( prompt, "", - store, + store.vfs(), LuaProgram::empty().expect("the empty chunk compiles"), &RunConfig::new(EXECUTION).observer(observer), ) @@ -1410,7 +1403,7 @@ async fn caught_h1_callback_error_stops_before_a_later_block() { ## Result\n\n\ ```lua\nreturn 'unexpected'\n```\n"; let prompt = parse(md); - let store = StoreRef::memory(); + let store = TestStore::new(); let ctx = h1_context_on(&prompt, &store, Arc::new(NullObserver::default())); let resolution = H1Resolution::empty(); let error = Scheduler::new(&ctx, None) @@ -1444,7 +1437,7 @@ async fn a_caught_h1_callback_error_reports_the_chunk_succeeded() { assert(not ok)\n\ ```\n"; let prompt = parse(md); - let ctx = h1_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = h1_context_on(&prompt, &TestStore::new(), recorder.clone()); let resolution = H1Resolution::empty(); let error = Scheduler::new(&ctx, None) .with_live_h1(resolution.context()) @@ -1818,7 +1811,7 @@ async fn the_live_h1_pass_fires_no_section_boundaries() { ## Only\n\n\ ```lua\nreturn 'done-now'\n```\n"; let prompt = parse(md); - let ctx = h1_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = h1_context_on(&prompt, &TestStore::new(), recorder.clone()); let resolution = H1Resolution::empty(); let out = Scheduler::new(&ctx, None) .with_live_h1(resolution.context()) @@ -1865,7 +1858,7 @@ fn scheduler_context_with_limits(prompt: &Prompt, limits: RunLimits) -> RunConte let ctx = RunContext::new( prompt, "", - &StoreRef::memory(), + &TestStore::new(), LuaProgram::empty().expect("the empty chunk compiles"), &RunConfig::new(EXECUTION).limits(limits), ); @@ -2024,7 +2017,7 @@ async fn fanout_arms_take_global_ids_per_fanout_index_and_structured_results() { // each arm entry takes the next run-global id, `sys.index` is the // 1-based per-fanout position, and the packed sequence carries `.ok` // and `.item` with `__tostring` driving `table.concat`. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ ## Parent\n\n\ @@ -2132,7 +2125,7 @@ async fn model_required_when_arm_infer_has_no_binding() { let ctx = RunContext::new( &prompt, "", - &StoreRef::memory(), + &TestStore::new(), shared, &RunConfig::new(EXECUTION), ); @@ -2182,7 +2175,7 @@ async fn the_shared_replay_sees_the_arm_item() { let ctx = RunContext::new( &prompt, "", - &StoreRef::memory(), + &TestStore::new(), shared, &RunConfig::new(EXECUTION), ); @@ -2229,7 +2222,7 @@ async fn a_jump_inside_a_fanout_arm_drives_a_child_walk() { store.append('order.txt', 'Tail\\n')\n\ return 'tail-reply'\n\ ```\n"; - let store = StoreRef::memory(); + let store = TestStore::new(); let prompt = parse(md); let ctx = scheduler_context_on(&prompt, &store, Arc::new(NullObserver::default())); let out = Scheduler::new(&ctx, None) @@ -2276,7 +2269,7 @@ async fn a_jump_from_an_arm_to_a_worker_child_walks_the_child_slice() { store.append('order.txt', 'ChildTail\\n')\n\ return 'child-tail-reply'\n\ ```\n"; - let store = StoreRef::memory(); + let store = TestStore::new(); let prompt = parse(md); let ctx = scheduler_context_on(&prompt, &store, Arc::new(NullObserver::default())); let out = Scheduler::new(&ctx, None) @@ -2314,7 +2307,7 @@ async fn fanout_empty_collection_errors_before_any_scheduling() { // `an_empty_collection_is_rejected_before_any_scheduling`: the fanout // errors before any arm is created - no STARTED observation, and the // worker's store tripwire never fires. - let store = StoreRef::memory(); + let store = TestStore::new(); let recorder = Arc::new(Recorder::default()); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ @@ -2408,12 +2401,14 @@ async fn fanout_depth_cap_reads_the_chain_field() { #[tokio::test(flavor = "current_thread")] async fn two_arms_writing_one_path_fail_with_a_write_race() { - // Mirror of the legacy `two_arms_writing_one_path_fail_with_a_write_race`: - // two arms of one fanout calling `store.write` on the same path is a - // hard write-write race; the store's registry is the semantic guard - // (one thread runs everything, so no locks are involved), and the race - // is fatal to the arm and fails the fanout. + // Mirror of the legacy `two_arms_writing_one_path_fail_with_a_write_race`, + // restructured for the claims model: the registry is gone, so the race + // needs both arms live at once - each arm writes, then suspends on an + // infer, so the second arm's write meets the first arm's standing write + // claim. The conflict is fatal to the second arm and fails the fanout; + // the first arm, still parked on its infer, is aborted as the sibling. let recorder = Arc::new(Recorder::default()); + let gateway = ScriptedGateway::start(vec![resp_text("p1"), resp_text("p2")]).await; let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ ## Parent\n\n\ @@ -2424,38 +2419,73 @@ async fn two_arms_writing_one_path_fail_with_a_write_race() { ### Worker\n\n\ ```lua\n\ store.write('shared.txt', item)\n\ + models.infer('pause ' .. item)\n\ return item\n\ ```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); - let error = Scheduler::new(&ctx, None) + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); + let error = Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))) .drive() .await - .expect_err("two arms writing one path must fail the fanout"); + .expect_err("two live arms writing one path must fail the fanout"); let text = error.to_string(); assert!(text.contains("write-write race"), "error was: {text}"); assert!(text.contains("shared.txt"), "error was: {text}"); assert_eq!( - terminal_count(&recorder, &detail::FANOUT_ARM_SUCCEEDED), + terminal_count(&recorder, &detail::FANOUT_ARM_FAILED), 1, - "the first arm's write landed: {:?}", + "the second arm's write raced: {:?}", recorder.events() ); assert_eq!( - terminal_count(&recorder, &detail::FANOUT_ARM_FAILED), + terminal_count(&recorder, &detail::FANOUT_ARM_CANCELLED), 1, - "the second arm's write raced: {:?}", + "the first arm, parked on its infer, is aborted as the sibling: {:?}", recorder.events() ); } +#[tokio::test(flavor = "current_thread")] +async fn two_live_arms_appending_one_path_fail_with_a_write_race() { + // The papergate case the WriteScope registry never caught: `append` + // claims write intent now, so two live arms appending to one path + // conflict exactly as two writes do. The arms interleave because each + // appends before suspending on its infer. + let gateway = ScriptedGateway::start(vec![resp_text("p1"), resp_text("p2")]).await; + let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ + # Fanout\n\n\ + ## Parent\n\n\ + ```lua\n\ + local r = fanout('### Worker', {'alpha', 'beta'})\n\ + return r[1].text\n\ + ```\n\n\ + ### Worker\n\n\ + ```lua\n\ + store.append('evidence.md', item .. '\\n')\n\ + models.infer('pause ' .. item)\n\ + return item\n\ + ```\n"; + let prompt = parse(md); + let ctx = scheduler_context(&prompt); + let error = Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))) + .drive() + .await + .expect_err("two live arms appending one path must fail the fanout"); + + let text = error.to_string(); + assert!(text.contains("write-write race"), "error was: {text}"); + assert!(text.contains("evidence.md"), "error was: {text}"); +} + #[tokio::test(flavor = "current_thread")] async fn two_arms_appending_one_path_succeed() { - // Mirror of the legacy case of the same name: `append` is untracked, so - // concurrent appends to one path are legal; only the relative order is - // unspecified. - let store = StoreRef::memory(); + // Mirror of the legacy case of the same name, with the claims-model + // rationale: arms that never suspend at I/O run one at a time, so each + // arm's claims release at its end and the next arm's append meets no + // live claimant. Only the relative order is unspecified - and with no + // interleaving points it is the collection order. + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ ## Parent\n\n\ @@ -2486,7 +2516,7 @@ async fn an_arm_rewriting_its_own_path_succeeds() { // Mirror of the legacy case of the same name: the registry records // (fanout token, arm index), so the same arm writing the same path // again is a rewrite, not a race. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ ## Parent\n\n\ @@ -2516,7 +2546,7 @@ async fn sequential_fanouts_may_write_one_path() { // Mirror of the legacy case of the same name: a later fanout carries a // fresh write token, so its write overwrites the earlier fanout's // registry record instead of racing against it. - let store = StoreRef::memory(); + let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ ## Parent\n\n\ @@ -2548,7 +2578,7 @@ async fn fatal_arm_aborts_queued_siblings() { // arm fails fatally they are never created - proven by the store // side-channel only the fatal arm ever wrote to, and by the terminal // observations: one FAILED, nothing else. - let store = StoreRef::memory(); + let store = TestStore::new(); let recorder = Arc::new(Recorder::default()); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ @@ -2634,7 +2664,7 @@ async fn fatal_arm_aborts_an_in_flight_sibling() { return a\n\ ```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); let result = tokio::time::timeout( std::time::Duration::from_secs(10), Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))).drive(), @@ -2756,7 +2786,7 @@ async fn cancellation_while_suspended_in_a_fanout_arm_interrupts_the_run() { ### Worker\n\n\ ```lua\nreturn models.infer('hang ' .. item)\n```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); let cancel = CancelHandle::new(); let canceller = cancel.clone(); let calls = Arc::clone(&gateway.calls); @@ -2832,7 +2862,7 @@ async fn a_mid_refill_arm_start_failure_tears_down_the_join() { ### Worker\n\n\ ```lua\nreturn 'worked:' .. item\n```\n"; let prompt = parse(md); - let ctx = scheduler_context_on(&prompt, &StoreRef::memory(), recorder.clone()); + let ctx = scheduler_context_on(&prompt, &TestStore::new(), recorder.clone()); let mut scheduler = Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))); // The root walk chain is id 0 and the first arm id 1; the second arm's // start trips the bound. diff --git a/crates/promptforge-core/src/execute/tests/tool_scoping.rs b/crates/promptforge-core/src/execute/tests/tool_scoping.rs index 8c3fd5117..b616f3a75 100644 --- a/crates/promptforge-core/src/execute/tests/tool_scoping.rs +++ b/crates/promptforge-core/src/execute/tests/tool_scoping.rs @@ -124,7 +124,7 @@ async fn h2_add_scopes_an_alias_and_dispatches_the_concrete_tool() { .expect("captured bindings must install"); vm.install_captured_bindings() .expect("alias globals must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); // The H2 `tools.add` lands in the section's tool runtime; the scope diff --git a/crates/promptforge-core/src/execute/tests/unified_pipeline.rs b/crates/promptforge-core/src/execute/tests/unified_pipeline.rs index e753797ef..42212c191 100644 --- a/crates/promptforge-core/src/execute/tests/unified_pipeline.rs +++ b/crates/promptforge-core/src/execute/tests/unified_pipeline.rs @@ -48,7 +48,7 @@ async fn finite_pipeline_runs_the_unified_surface_end_to_end() { &test, "quantum", &[Arc::new(EchoTool) as Arc], - &StoreRef::memory(), + &TestStore::new(), gatewayed(addr), ) .await diff --git a/crates/promptforge-core/src/lib.rs b/crates/promptforge-core/src/lib.rs index 042699560..cae32bf33 100644 --- a/crates/promptforge-core/src/lib.rs +++ b/crates/promptforge-core/src/lib.rs @@ -39,14 +39,13 @@ //! //! Executing a parsed prompt goes through [`run`] with a [`RunConfig`], a //! [`ResolutionContext`] (picker, model catalog, and tool catalog), and a -//! store; that path can perform gateway I/O, so it is shown as `no_run`: +//! VFS handle; that path can perform gateway I/O, so it is shown as `no_run`: //! //! ```no_run //! # async fn example() -> Result<(), Box> { //! use promptforge_core::{Prompt, ResolutionContext, RunConfig, run}; //! use promptforge_core::model::ModelCatalog; //! use promptforge_core::observe::NullObserver; -//! use promptforge_core::store::StoreRef; //! use promptforge_core::tools::ToolCatalog; //! use promptforge_tool_picker::{Catalog, Config, ToolPicker}; //! @@ -60,7 +59,7 @@ //! &prompt, //! "", //! ResolutionContext::new(&picker, &models, &tools), -//! &StoreRef::memory(), +//! &promptforge_vfs::empty(), //! RunConfig::new("run-example"), //! ) //! .await?; diff --git a/crates/promptforge-core/src/lua/coro_tests.rs b/crates/promptforge-core/src/lua/coro_tests.rs index 591c41bc2..71af09cb9 100644 --- a/crates/promptforge-core/src/lua/coro_tests.rs +++ b/crates/promptforge-core/src/lua/coro_tests.rs @@ -19,7 +19,6 @@ use crate::execute::section_vm::{SectionVmSetup, VmSeed, setup_section_vm}; use crate::lua::{CoroStep, LuaBlockResult, LuaProgram, SectionVm, ToolBinding, ToolSet}; use crate::model::{ModelBinding, ModelId, ModelInvocation, ModelSet}; use crate::observe::{NullObserver, Observer}; -use crate::store::StoreRef; use crate::tools::{Tool, ToolError, ToolId, ToolOutput}; use crate::untrusted::GuardNonce; @@ -113,13 +112,12 @@ fn scheduler_vm_with_tools( .expect("the section VM builds"); let shared = LuaProgram::empty().expect("the empty shared program compiles"); let sys = json!({}); - let store = StoreRef::memory(); + let access = Arc::new(promptforge_vfs::empty().acquire()); let setup = SectionVmSetup { args: "", sys: &sys, - store: &store, + access: &access, seed: VmSeed { var, item: None }, - write_scope: None, observer_arc: &observer, section_name: "Test", shared: &shared, diff --git a/crates/promptforge-core/src/model/tests/always.rs b/crates/promptforge-core/src/model/tests/always.rs index a7fc65ec5..90a75b5f6 100644 --- a/crates/promptforge-core/src/model/tests/always.rs +++ b/crates/promptforge-core/src/model/tests/always.rs @@ -107,7 +107,7 @@ fn models_always_installs_exactly() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let model = resolve_section_model(&vm).unwrap(); assert_eq!(model.as_ref().map(ModelBinding::alias), Some("writer")); vm.teardown(&NullObserver::default(), "Section"); @@ -128,7 +128,7 @@ fn models_always_provides_completion_options_without_use() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let model = resolve_section_model(&vm).unwrap(); let opts = model.as_ref().map(ModelBinding::completion_options); let expected = CompletionOptions::new("small") @@ -150,7 +150,7 @@ fn models_always_from_h2_prologue_fails() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let prologue = crate::lua::LuaProgram::compile( r#"models.default("writer")"#, "prologue", @@ -201,7 +201,7 @@ fn models_always_multi_arg_provides_completion_options() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let model = resolve_section_model(&vm).unwrap(); let opts = model.as_ref().map(ModelBinding::completion_options); let expected = CompletionOptions::new("small") @@ -225,7 +225,7 @@ fn models_always_multi_arg_installs_exactly() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let model = resolve_section_model(&vm).unwrap(); assert_eq!(model.as_ref().map(ModelBinding::alias), Some("writer")); vm.teardown(&NullObserver::default(), "Section"); diff --git a/crates/promptforge-core/src/model/tests/integration.rs b/crates/promptforge-core/src/model/tests/integration.rs index 2ef0548d9..b4494c9f0 100644 --- a/crates/promptforge-core/src/model/tests/integration.rs +++ b/crates/promptforge-core/src/model/tests/integration.rs @@ -43,7 +43,7 @@ fn models_bind_resolves_and_use_selects_section_binding() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let prologue = crate::lua::LuaProgram::compile( r#"models.use("analyst")"#, "prologue", @@ -71,7 +71,7 @@ fn no_models_use_or_always_leaves_section_unbound() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let model = resolve_section_model(&vm).unwrap(); assert!(model.is_none()); vm.teardown(&NullObserver::default(), "Section"); @@ -96,7 +96,7 @@ fn undeclared_models_use_fails_loudly() { "Section", ) .unwrap(); - vm.inject_host("", &json!({}), &StoreRef::memory()).unwrap(); + vm.inject_host("", &json!({}), &fresh_access()).unwrap(); let prologue = crate::lua::LuaProgram::compile( r#"models.use("missing")"#, "prologue", diff --git a/crates/promptforge-core/src/model/tests/mod.rs b/crates/promptforge-core/src/model/tests/mod.rs index 2ea9bd599..35f3cdcfc 100644 --- a/crates/promptforge-core/src/model/tests/mod.rs +++ b/crates/promptforge-core/src/model/tests/mod.rs @@ -8,7 +8,7 @@ use crate::lua::{ LiveBindingProducer, LuaProgram, SectionVm, ToolResolver, ToolSet, resolve_model_binding, }; use crate::observe::NullObserver; -use crate::store::StoreRef; +use crate::store::Access; use crate::tools::ToolCatalog; use crate::untrusted::GuardNonce; use crate::{Error, Result}; @@ -17,6 +17,12 @@ use serde_json::json; const EXECUTION: &str = "model-bind-test"; +/// A fresh stock handle's access capability, for tests that inject host +/// values into a standalone VM. +fn fresh_access() -> Arc { + Arc::new(promptforge_vfs::empty().acquire()) +} + fn ctx(window: u32) -> NonZeroU32 { NonZeroU32::new(window).expect("test context window is non-zero") } diff --git a/crates/promptforge-core/src/store.rs b/crates/promptforge-core/src/store.rs index d5045ebc5..4efabacd1 100644 --- a/crates/promptforge-core/src/store.rs +++ b/crates/promptforge-core/src/store.rs @@ -1,24 +1,23 @@ //! Run-scoped virtual files, shared by Lua and the model. //! //! A prompt run keeps its bulk state in virtual files addressed by logical -//! string paths. [`Store`] is the backend contract, [`MemStore`] is an -//! in-memory backend, and [`StoreRef`] is the cheaply cloneable, thread-safe -//! handle the runtime hands to both the Lua VM and (later) the model's file -//! tools. [`StoreRef::read`] returns verbatim contents for trusted handoff, -//! [`StoreRef::read_range`] slices a 1-based inclusive line range out of the -//! same verbatim contents, and [`StoreRef::read_range_numbered`] numbers such -//! a slice absolutely (with no bounds it numbers the whole file from 1). For -//! model-facing re-injection the caller wraps a verbatim read in an -//! untrusted guard envelope (the `untrusted` Lua global). -//! Edits are anchor-based ([`Store::str_replace`]) rather than offset-based, -//! the shape that works for a model. +//! string paths. The run's [`VfsRef`] handle carries the store mount; the +//! [`Store`] facade (behind the [`StoreExt`] extension trait's +//! `vfs.store(&access)` call shape) scopes logical paths onto it, and every +//! operation is attributed to the [`Access`] capability's identity, so a +//! conflicting operation by a second live identity surfaces as +//! [`StoreError::WriteRace`]. [`Store::read`] returns verbatim contents for +//! trusted handoff, [`Store::read_range`] slices a 1-based inclusive line +//! range out of the same verbatim contents, and +//! [`Store::read_range_numbered`] numbers such a slice absolutely (with no +//! bounds it numbers the whole file from 1). For model-facing re-injection +//! the caller wraps a verbatim read in an untrusted guard envelope (the +//! `untrusted` Lua global). Edits are anchor-based ([`Store::str_replace`]) +//! rather than offset-based, the shape that works for a model. //! //! The implementation lives in the `promptforge-store` crate and is -//! re-exported here unchanged, so existing `promptforge_core::store::*` paths -//! keep working. +//! re-exported here unchanged, so existing `promptforge_core::store::*` +//! paths keep working. -pub use promptforge_store::{ - FileStore, MemStore, PathReason, Store, StoreError, StoreErrorKind, StoreRef, -}; - -pub(crate) use promptforge_store::WriteScope; +pub use promptforge_store::{PathReason, Store, StoreError, StoreErrorKind, StoreExt}; +pub use shared_vfs::{Access, VfsRef}; diff --git a/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md b/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md index f8ec811a7..223738b70 100644 --- a/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md +++ b/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md @@ -17,16 +17,12 @@ return tostring(#files) .. ":" .. table.concat(replies, ",") ### Worker ```lua --- Rendezvous: both arms must be live before either writes its reply path. --- Each poll iteration yields through `call`, giving the sibling arm its --- I/O points: under the scheduler "concurrent" means interleaving at yield --- points, not preemption. A sequential driver (arm 2 starting only after --- arm 1 finishes) never reaches two ready files, and the loop spins --- forever: no instruction ceiling ends it; the test's timeout bounds it. -store.write("ready-" .. sys.index .. ".md", "1") -while #store.glob("ready-*.md") < 2 do - call("## Yield") -end +-- Arm-scoped writes: the pattern the claims model teaches. Each arm writes +-- only its own path, so no two live identities ever claim one path, and the +-- parent's post-join glob reads the merged state after every arm's claims +-- released at its end. (The old ready-*.md rendezvous - polling a sibling +-- arm's files while that arm is live - is exactly the cross-arm +-- read-while-written pattern the claims model rejects.) store.write("arm-" .. sys.index .. ".md", item) return item ``` @@ -37,9 +33,3 @@ Write to store. - alpha - beta - -## Yield - -```lua -return "yielded" -``` diff --git a/crates/promptforge-core/tests/suite/execution.rs b/crates/promptforge-core/tests/suite/execution.rs index 18808ff31..435ce474b 100644 --- a/crates/promptforge-core/tests/suite/execution.rs +++ b/crates/promptforge-core/tests/suite/execution.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use promptforge_core::execute::RunErrorKind; use promptforge_core::observe::Observer; -use promptforge_core::store::StoreRef; use super::support::{Record, Recorder, RunOptions, parse_execution_fixture, run, run_fixture}; @@ -182,7 +181,7 @@ async fn concurrent_runs_keep_execution_ids_separate() { first_prompt.as_ref(), "first result", &[], - &StoreRef::memory(), + &promptforge_vfs::empty(), RunOptions { execution: FIRST, observer: Arc::clone(&first_recorder) as Arc, @@ -198,7 +197,7 @@ async fn concurrent_runs_keep_execution_ids_separate() { second_prompt.as_ref(), "second result", &[], - &StoreRef::memory(), + &promptforge_vfs::empty(), RunOptions { execution: SECOND, observer: Arc::clone(&second_recorder) as Arc, diff --git a/crates/promptforge-core/tests/suite/fanout.rs b/crates/promptforge-core/tests/suite/fanout.rs index 1ea68b2c3..9de0b5aac 100644 --- a/crates/promptforge-core/tests/suite/fanout.rs +++ b/crates/promptforge-core/tests/suite/fanout.rs @@ -97,15 +97,14 @@ async fn fanout_epilog_two_items() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn fanout_store_writes_persist_across_arms() { - // The arms rendezvous by writing and polling ready-*.md, so concurrency is - // proven by both ready markers and both arm writes existing. Each poll - // iteration yields through `call` on the nop `## Yield` section: under - // the scheduler "concurrent" means interleaving at I/O points, not - // preemption, so the rendezvous completes only if the sibling arm gets the - // driver's thread while the poller is suspended. A sequential driver never - // reaches two ready files and the poll spins forever - no instruction - // ceiling ends it - so the timeout below is what turns that regression - // into a failure; the asserts, not the timeout, are the pass condition. + // Arm-scoped writes under the claims model: each arm writes only its + // own path, so no two live identities ever claim one path, and the + // parent's post-join glob sees the merged state because a finished + // arm's claims release at chain end. (The fixture's old ready-*.md + // rendezvous polled a live sibling's writes - precisely the cross-arm + // read-while-written pattern the claims model rejects - so it was + // removed; interleaving coverage lives in the scheduler's + // `fanout_arms_interleave_at_io_points_on_one_thread`.) let run = tokio::time::timeout( Duration::from_secs(30), run_fixture( @@ -117,7 +116,7 @@ async fn fanout_store_writes_persist_across_arms() { ), ) .await - .expect("concurrent fanout must finish; a sequential regression would hang on the rendezvous"); + .expect("the fanout fixture completes"); let result = run .result .expect("the fanout store fixture must execute offline"); @@ -133,18 +132,6 @@ async fn fanout_store_writes_persist_across_arms() { run.store.read("arm-2.md").expect("arm 2 must write"), "beta" ); - assert_eq!( - run.store - .read("ready-1.md") - .expect("arm 1 rendezvous marker"), - "1" - ); - assert_eq!( - run.store - .read("ready-2.md") - .expect("arm 2 rendezvous marker"), - "1" - ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/crates/promptforge-core/tests/suite/main.rs b/crates/promptforge-core/tests/suite/main.rs index 43a4c2d37..f1cfd8142 100644 --- a/crates/promptforge-core/tests/suite/main.rs +++ b/crates/promptforge-core/tests/suite/main.rs @@ -12,3 +12,4 @@ mod fanout; mod parsing; mod shipped; mod support; +mod vfs; diff --git a/crates/promptforge-core/tests/suite/support.rs b/crates/promptforge-core/tests/suite/support.rs index e91486178..9d72096c5 100644 --- a/crates/promptforge-core/tests/suite/support.rs +++ b/crates/promptforge-core/tests/suite/support.rs @@ -9,7 +9,7 @@ use promptforge_core::execute::{ResolutionContext, RunConfig, RunError, run as r use promptforge_core::model::ModelCatalog; use promptforge_core::observe::{Observation, Observer}; use promptforge_core::parser::Prompt; -use promptforge_core::store::StoreRef; +use promptforge_core::store::{StoreError, StoreExt, VfsRef}; use promptforge_tool_picker::{Catalog, Config, ToolPicker}; use promptforge_tools::{Tool, ToolCatalog}; @@ -45,7 +45,7 @@ pub(super) async fn run( prompt: &Prompt, args: &str, tools: &[Arc], - store: &StoreRef, + vfs: &VfsRef, opts: RunOptions, ) -> Result { let picker = ToolPicker::build_with_model( @@ -61,7 +61,7 @@ pub(super) async fn run( prompt, args, ResolutionContext::new(&picker, &models, &tools), - store, + vfs, RunConfig::new(opts.execution).observer(opts.observer), ) .await @@ -103,31 +103,44 @@ pub(super) fn parse_execution_fixture( .unwrap_or_else(|error| panic!("fixture {name} failed to parse: {error}")) } +/// The run's VFS handle with per-call fresh-access store reads, for +/// post-run assertions: the run's identities dropped with it, so a fresh +/// access never meets a lingering claim. +pub(super) struct FixtureStore(VfsRef); + +impl FixtureStore { + /// Reads a store path through a fresh, immediately dropped access. + pub(super) fn read(&self, path: &str) -> Result { + let access = self.0.acquire(); + self.0.store(&access).read(path) + } +} + /// The parsed prompt run plus the recorder and store an assertion needs. pub(super) struct FixtureRun { pub(super) result: Result, pub(super) recorder: Arc, - pub(super) store: StoreRef, + pub(super) store: FixtureStore, } /// Parses `source` and runs it offline with `args`, no tools, and either the -/// supplied `store` or a fresh in-memory one, returning the result together +/// supplied `vfs` or a fresh stock handle, returning the result together /// with the recorder and store the caller asserts on. pub(super) async fn run_fixture( source: &'static str, name: &'static str, execution: &'static str, args: &str, - store: Option, + vfs: Option, ) -> FixtureRun { let recorder = Arc::new(Recorder::default()); let prompt = parse_execution_fixture(source, name, execution, recorder.as_ref()); - let store = store.unwrap_or_else(StoreRef::memory); + let vfs = vfs.unwrap_or_else(promptforge_vfs::empty); let result = run( &prompt, args, &[], - &store, + &vfs, RunOptions { execution, observer: Arc::clone(&recorder) as Arc, @@ -137,6 +150,6 @@ pub(super) async fn run_fixture( FixtureRun { result, recorder, - store, + store: FixtureStore(vfs), } } diff --git a/crates/promptforge-core/tests/suite/vfs.rs b/crates/promptforge-core/tests/suite/vfs.rs new file mode 100644 index 000000000..7824dd9ed --- /dev/null +++ b/crates/promptforge-core/tests/suite/vfs.rs @@ -0,0 +1,202 @@ +//! The executor's `VfsRef` host contract: an end-to-end run over the stock +//! handle, and the papergate-shaped seed-run-extract round trip a production +//! host drives with no real files - seed the declared input through +//! `vfs.store()`, run, extract the declared output, and charge a missing +//! output to the prompt's promise as an explicit contract error. + +use promptforge_core::parser::Prompt; +use promptforge_core::store::{Store, StoreError, StoreExt}; +use shared_vfs::VfsRef; + +use super::support::{RunOptions, parse_execution_fixture, run}; +use crate::support::Recorder; +use std::sync::Arc; + +const ROUND_TRIP: &str = concat!( + "---\n", + "name: papergate-round-trip\n", + "description: Seed, run, extract\n", + "promptforge: 0\n", + "input:\n", + " path: paper.md\n", + " description: The input paper\n", + "output:\n", + " path: report.md\n", + " description: The output report\n", + "---\n\n", + "# Review\n\n", + "## Summarize\n\n", + "```lua\n", + "local paper = store.read('paper.md')\n", + "store.write('report.md', 'report on: ' .. paper)\n", + "return 'done'\n", + "```\n", +); + +const MISSING_OUTPUT: &str = concat!( + "---\n", + "name: papergate-missing-output\n", + "description: Never writes its promised report\n", + "promptforge: 0\n", + "input:\n", + " path: paper.md\n", + " description: The input paper\n", + "output:\n", + " path: report.md\n", + " description: The output report\n", + "---\n\n", + "# Review\n\n", + "## Summarize\n\n", + "```lua\n", + "local paper = store.read('paper.md')\n", + "return 'read: ' .. paper\n", + "```\n", +); + +/// The production host's extraction rule (pattern: papergate's app.rs): a +/// declared output the run did not leave behind is an explicit contract +/// error naming the prompt's promise, never a bare not-found. +fn extract_declared_output(store: &Store, prompt: &Prompt) -> Result { + let output = prompt + .frontmatter() + .output() + .expect("the fixture declares an output"); + store.read(output.path()).map_err(|error| match error { + StoreError::NotFound { .. } => format!( + "the prompt promised output '{}' ({}) but the run left no such file", + output.path(), + output.description() + ), + other => format!("extracting the declared output failed: {other}"), + }) +} + +/// Seeds the prompt's declared input through the stock handle's store +/// facade. The seeding access drops here - its claims release - so the +/// run's own identity never meets the host's. +fn seed_declared_input(vfs: &VfsRef, prompt: &Prompt, contents: &str) { + let input = prompt + .frontmatter() + .input() + .expect("the fixture declares an input"); + let access = vfs.acquire(); + vfs.store(&access) + .write(input.path(), contents) + .expect("the declared input seeds"); +} + +fn offline_run( + prompt: &Prompt, + vfs: &VfsRef, + execution: &'static str, +) -> impl std::future::Future> { + let recorder = Arc::new(Recorder::default()); + let prompt = prompt.clone(); + let vfs = vfs.clone(); + async move { + run( + &prompt, + "", + &[], + &vfs, + RunOptions { + execution, + observer: recorder, + }, + ) + .await + } +} + +#[tokio::test] +async fn an_end_to_end_run_threads_one_vfs_ref_through_every_section() { + // The store survives the context-clearing section transition: one + // section's write is the next section's read, over the stock handle. + let source = "\ +---\nname: vfs-end-to-end\ndescription: d\npromptforge: 0\n---\n\n\ +# Title\n\n\ +## First\n\n\ +```lua\n\ +store.write('handoff.txt', 'across the reset')\n\ +```\n\n\ +## Second\n\n\ +```lua\n\ +return store.read('handoff.txt')\n\ +```\n"; + let recorder = Arc::new(Recorder::default()); + let prompt = parse_execution_fixture(source, "vfs-end-to-end", "vfs-e2e", recorder.as_ref()); + let vfs = promptforge_vfs::empty(); + let result = run( + &prompt, + "", + &[], + &vfs, + RunOptions { + execution: "vfs-e2e", + observer: recorder, + }, + ) + .await + .expect("the run threads the stock handle through both sections"); + assert_eq!(result, "across the reset"); + // Extraction after the run takes a fresh access: the run's identities + // dropped with it, so nothing the run touched can conflict here. + let access = vfs.acquire(); + assert_eq!( + vfs.store(&access) + .read("handoff.txt") + .expect("the run's write persists on the handle"), + "across the reset" + ); +} + +#[tokio::test] +async fn a_host_seeds_and_extracts_through_the_stock_handle_with_no_real_files() { + let recorder = Arc::new(Recorder::default()); + let prompt = parse_execution_fixture( + ROUND_TRIP, + "papergate-round-trip", + "vfs-round-trip", + recorder.as_ref(), + ); + let vfs = promptforge_vfs::empty(); + seed_declared_input(&vfs, &prompt, "the paper body"); + let result = offline_run(&prompt, &vfs, "vfs-round-trip") + .await + .expect("the seeded run executes offline"); + assert_eq!(result, "done"); + let access = vfs.acquire(); + let report = extract_declared_output(&vfs.store(&access), &prompt) + .expect("the run left its promised output"); + assert_eq!(report, "report on: the paper body"); +} + +#[tokio::test] +async fn a_missing_declared_output_is_a_contract_error_naming_the_prompts_promise() { + let recorder = Arc::new(Recorder::default()); + let prompt = parse_execution_fixture( + MISSING_OUTPUT, + "papergate-missing-output", + "vfs-missing-output", + recorder.as_ref(), + ); + let vfs = promptforge_vfs::empty(); + seed_declared_input(&vfs, &prompt, "the paper body"); + // The executor does not enforce the declaration; the run succeeds and + // the host's extraction is where the broken promise surfaces. + let result = offline_run(&prompt, &vfs, "vfs-missing-output") + .await + .expect("the run itself succeeds"); + assert_eq!(result, "read: the paper body"); + let access = vfs.acquire(); + let error = extract_declared_output(&vfs.store(&access), &prompt) + .expect_err("the missing output is a contract error"); + assert!( + error.contains("report.md"), + "the error names the promised path: {error}" + ); + assert!( + error.contains("The output report"), + "the error names the promise's description: {error}" + ); +} diff --git a/crates/promptforge-lua/Cargo.toml b/crates/promptforge-lua/Cargo.toml index 06fce1ddf..a6a4aca08 100644 --- a/crates/promptforge-lua/Cargo.toml +++ b/crates/promptforge-lua/Cargo.toml @@ -25,6 +25,8 @@ tokio.workspace = true [dev-dependencies] async-trait.workspace = true criterion.workspace = true +promptforge-vfs.workspace = true +shared-vfs.workspace = true tokio = { workspace = true, features = ["macros", "rt-multi-thread", "test-util"] } [[bench]] diff --git a/crates/promptforge-lua/benches/surface.rs b/crates/promptforge-lua/benches/surface.rs index a41066e0e..4894a5507 100644 --- a/crates/promptforge-lua/benches/surface.rs +++ b/crates/promptforge-lua/benches/surface.rs @@ -24,7 +24,6 @@ use promptforge_lua::{ project_messages, }; use promptforge_model_client::model::ModelSet; -use promptforge_store::StoreRef; use serde_json::json; const EXECUTION: &str = "bench"; @@ -42,8 +41,12 @@ fn builder_vm() -> SectionVm { SECTION, ) .expect("the bench VM builds"); - vm.inject_host("", &json!({}), &StoreRef::memory()) - .expect("host injection installs the messages namespace"); + vm.inject_host( + "", + &json!({}), + &std::sync::Arc::new(promptforge_vfs::empty().acquire()), + ) + .expect("host injection installs the messages namespace"); vm } diff --git a/crates/promptforge-lua/src/host.rs b/crates/promptforge-lua/src/host.rs index 839f86f06..93289ce0a 100644 --- a/crates/promptforge-lua/src/host.rs +++ b/crates/promptforge-lua/src/host.rs @@ -1,6 +1,6 @@ use super::{ - Arc, AtomicU32, AtomicUsize, Error, GuardNonce, LUA_LOG_CHARACTER_LIMIT, Lua, LuaSerdeExt, - MultiValue, Observation, Observer, Ordering, Result, StoreRef, Value, WriteScope, detail, + Access, Arc, AtomicU32, AtomicUsize, Error, GuardNonce, LUA_LOG_CHARACTER_LIMIT, Lua, + LuaSerdeExt, MultiValue, Observation, Observer, Ordering, Result, Store, Value, detail, }; /// Shared body of the persistent per-section `log(message)` host callback. @@ -172,10 +172,10 @@ pub(crate) fn observe_store_result( /// /// No `start` reads the whole file; a present `start` slices a 1-based /// inclusive line range. A negative bound converts to 0, which -/// [`StoreRef::read_range`] rejects with the same error a zero bound earns, +/// [`Store::read_range`] rejects with the same error a zero bound earns, /// and an `end` without a `start` is refused rather than silently ignored. fn read_store_bounded( - handle: &StoreRef, + store: &Store, path: &str, start: Option, end: Option, @@ -184,9 +184,9 @@ fn read_store_bounded( match start { None if end.is_none() => { if numbered { - handle.read_range_numbered(path, 1, None) + store.read_range_numbered(path, 1, None) } else { - handle.read(path) + store.read(path) } } None => Err(promptforge_store::StoreError::invalid_range( @@ -197,9 +197,9 @@ fn read_store_bounded( let start = usize::try_from(start).unwrap_or(0); let end = end.map(|line| usize::try_from(line).unwrap_or(0)); if numbered { - handle.read_range_numbered(path, start, end) + store.read_range_numbered(path, start, end) } else { - handle.read_range(path, start, end) + store.read_range(path, start, end) } } } @@ -207,27 +207,28 @@ fn read_store_bounded( /// Shared body of the persistent per-section `store.read` host callback. fn read_store( - handle: &StoreRef, + store: &Store, path: &str, start: Option, end: Option, ) -> std::result::Result { - read_store_bounded(handle, path, start, end, false) + read_store_bounded(store, path, start, end, false) } /// Shared body of the persistent per-section `store.read_numbered` callback. fn read_store_numbered( - handle: &StoreRef, + store: &Store, path: &str, start: Option, end: Option, ) -> std::result::Result { - read_store_bounded(handle, path, start, end, true) + read_store_bounded(store, path, start, end, true) } /// Expose an always-on `store` table whose methods (`write`, `append`, /// `read`, `read_numbered`, `str_replace`, `delete`, -/// `glob`, `exists`) are backed by the run-scoped [`StoreRef`] handle. +/// `glob`, `exists`) are backed by the [`Store`] facade over the caller's +/// VFS access capability. /// Installed once per section with [`Lua::create_function`], so the table /// stays valid across every chunk the VM runs without a live [`mlua::Scope`]. /// @@ -241,14 +242,13 @@ fn read_store_numbered( /// an `mlua` error via [`mlua::Error::external`], so it aborts the chunk and /// surfaces as [`Error::Lua`]. /// -/// The `StoreRef` handle locks a mutex internally per call and is synchronous, so -/// nothing is held across an await. -/// -/// A fanout arm's table carries its [`WriteScope`]: `store.write` goes -/// through [`StoreRef::write_scoped`], so two arms of one fanout writing the -/// same path fail the second writer with a write-write race error. Every -/// other caller (walk sections, H1) installs with `None` and writes -/// untracked. +/// Every closure captures an `Arc` clone of the section's [`Access`] +/// capability and builds the borrowing facade per call. The capability +/// locks the backend per call and is synchronous, so nothing is held across +/// an await. The capability's identity is what the claims model attributes +/// operations to: a fanout arm's access is spawned from the caller's, so +/// two live arms touching one path surface the conflict as +/// [`StoreError::WriteRace`]. /// /// [`StoreError`]: promptforge_store::StoreError /// @@ -262,11 +262,10 @@ fn read_store_numbered( pub(crate) fn install_store_table( lua: &Lua, globals: &mlua::Table, - store: &StoreRef, + access: &Arc, execution: &str, observer: &Arc, section: &str, - write_scope: Option, ) -> Result<()> { let table = lua.create_table().map_err(Error::lua)?; let reporter = Arc::new(StoreReporter { @@ -285,7 +284,7 @@ pub(crate) fn install_store_table( $failure:expr, $operation:block ) => {{ - let $handle = store.clone(); + let $handle = Arc::clone(access); let report = Arc::clone(&reporter); let function = lua .create_function(move |_, $arguments: $argument_type| { @@ -305,12 +304,7 @@ pub(crate) fn install_store_table( (String, String), detail::STORE_WRITE_SUCCEEDED, detail::STORE_WRITE_FAILED, - { - match write_scope { - Some(scope) => handle.write_scoped(&path, &contents, scope), - None => handle.write(&path, &contents), - } - } + { Store::new(&handle).write(&path, &contents) } ); install_reported_store_fn!( "append", @@ -319,7 +313,7 @@ pub(crate) fn install_store_table( (String, String), detail::STORE_APPEND_SUCCEEDED, detail::STORE_APPEND_FAILED, - { handle.append(&path, &contents) } + { Store::new(&handle).append(&path, &contents) } ); install_reported_store_fn!( "read", @@ -328,7 +322,7 @@ pub(crate) fn install_store_table( (String, Option, Option), detail::STORE_READ_SUCCEEDED, detail::STORE_READ_FAILED, - { read_store(&handle, &path, start, end) } + { read_store(&Store::new(&handle), &path, start, end) } ); install_reported_store_fn!( "read_numbered", @@ -337,7 +331,7 @@ pub(crate) fn install_store_table( (String, Option, Option), detail::STORE_READ_NUMBERED_SUCCEEDED, detail::STORE_READ_NUMBERED_FAILED, - { read_store_numbered(&handle, &path, start, end) } + { read_store_numbered(&Store::new(&handle), &path, start, end) } ); install_reported_store_fn!( "str_replace", @@ -346,7 +340,7 @@ pub(crate) fn install_store_table( (String, String, String), detail::STORE_REPLACE_SUCCEEDED, detail::STORE_REPLACE_FAILED, - { handle.str_replace(&path, &old, &new) } + { Store::new(&handle).str_replace(&path, &old, &new) } ); install_reported_store_fn!( "delete", @@ -355,14 +349,14 @@ pub(crate) fn install_store_table( String, detail::STORE_DELETE_SUCCEEDED, detail::STORE_DELETE_FAILED, - { handle.delete(&path) } + { Store::new(&handle).delete(&path) } ); - let handle = store.clone(); + let handle = Arc::clone(access); let report = Arc::clone(&reporter); let glob = lua .create_function(move |lua, pattern: String| { - let result = handle.glob(&pattern); + let result = Store::new(&handle).glob(&pattern); report.report( result.is_ok(), detail::STORE_GLOB_SUCCEEDED, @@ -374,9 +368,13 @@ pub(crate) fn install_store_table( .map_err(Error::lua)?; table.set("glob", glob).map_err(Error::lua)?; - let handle = store.clone(); + let handle = Arc::clone(access); let exists = lua - .create_function(move |_, path: String| handle.exists(&path).map_err(mlua::Error::external)) + .create_function(move |_, path: String| { + Store::new(&handle) + .exists(&path) + .map_err(mlua::Error::external) + }) .map_err(Error::lua)?; table.set("exists", exists).map_err(Error::lua)?; diff --git a/crates/promptforge-lua/src/lib.rs b/crates/promptforge-lua/src/lib.rs index 8991ad6a6..b708e707f 100644 --- a/crates/promptforge-lua/src/lib.rs +++ b/crates/promptforge-lua/src/lib.rs @@ -17,7 +17,8 @@ //! //! The `store` table is a deterministic host capability (like `var`), always //! present and independent of tool scoping. Its methods are backed by the -//! run-scoped [`StoreRef`] handle threaded in from the executor, so every section +//! [`Store`] facade over the run's VFS access capability, threaded in from +//! the executor, so every section //! in a run shares one set of virtual files even though contexts clear on each //! transition. A failed store op raises a Lua error, which surfaces from //! `SectionVm::run_chunk` as [`Error::Lua`]. @@ -46,7 +47,7 @@ pub(crate) use promptforge_core_support::untrusted::GuardNonce; pub(crate) use promptforge_model_client::model::{ ModelBinding, ModelResolver, ModelSet, ModelView, }; -pub(crate) use promptforge_store::{StoreRef, WriteScope}; +pub(crate) use promptforge_store::{Access, Store}; pub(crate) use promptforge_tools::{Tool, ToolCatalog, ToolId}; pub(crate) use crate::compactors::install_compactors; diff --git a/crates/promptforge-lua/src/messages/tests.rs b/crates/promptforge-lua/src/messages/tests.rs index f3c589ce1..b27c04edf 100644 --- a/crates/promptforge-lua/src/messages/tests.rs +++ b/crates/promptforge-lua/src/messages/tests.rs @@ -1,13 +1,17 @@ use mlua::{Lua, LuaSerdeExt, Value}; use promptforge_core_support::observe::NullObserver; use promptforge_core_support::untrusted::GuardNonce; -use promptforge_store::StoreRef; use serde_json::json; use super::install_messages; use crate::protocol::{Answer, MessageRecord, Request, ToolCallRecord, YieldParse}; use crate::{Error, SectionVm}; +/// A fresh stock handle's access capability for a test VM. +fn fresh_access() -> std::sync::Arc { + std::sync::Arc::new(promptforge_vfs::empty().acquire()) +} + fn lua_with_messages() -> Lua { let lua = Lua::new(); let globals = lua.globals(); @@ -198,7 +202,7 @@ fn the_builders_run_under_the_hardened_section_sandbox() { let observer = NullObserver::default(); let mut vm = SectionVm::new(&nonce, "test-run", &observer, "Test") .expect("section VM construction cannot fail"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host injection cannot fail"); let json: serde_json::Value = vm .lua() diff --git a/crates/promptforge-lua/src/models/tests.rs b/crates/promptforge-lua/src/models/tests.rs index 6c0054e30..b6e47957a 100644 --- a/crates/promptforge-lua/src/models/tests.rs +++ b/crates/promptforge-lua/src/models/tests.rs @@ -335,7 +335,7 @@ fn h2_vm(raw_ids: bool) -> crate::SectionVm { vm.inject_host( "", &serde_json::json!({}), - &promptforge_store::StoreRef::memory(), + &std::sync::Arc::new(promptforge_vfs::empty().acquire()), ) .expect("host injection installs the H2 models table"); vm diff --git a/crates/promptforge-lua/src/tests.rs b/crates/promptforge-lua/src/tests.rs index 3b48fc8cd..507dcb00b 100644 --- a/crates/promptforge-lua/src/tests.rs +++ b/crates/promptforge-lua/src/tests.rs @@ -4,12 +4,20 @@ use super::*; use crate::program::map_chunk_line_to_absolute; use crate::vm::LocalTools; use promptforge_core_support::observe::{NullObserver, Observation}; -use promptforge_store::{Store, StoreError}; +use promptforge_store::Store; use promptforge_tools::{Tool, ToolError, ToolOutput}; use serde_json::json; +use shared_vfs::{ExecId, Vfs, VfsAccess, VfsError, VfsPath, VfsRef}; const EXECUTION: &str = "lua-test"; +/// A fresh stock handle's access capability for a test VM: the store mount +/// exists and the vended identity is the test's own, so seeding through the +/// facade and the VM's store ops never meet a second live identity. +fn fresh_access() -> Arc { + Arc::new(promptforge_vfs::empty().acquire()) +} + #[derive(Default)] struct Recorder(Mutex>); @@ -48,61 +56,104 @@ fn lua_error_message(error: &Error) -> &str { } } +/// A backend whose every operation fails. The error is `Backend` rather +/// than `NotFound` so the facade's idempotent-delete mapping (absent is +/// `Ok`) cannot swallow the failure: every op must reach Lua as an error. #[derive(Debug)] -struct FailingStore; +struct FailingBackend; + +impl FailingBackend { + fn error(path: VfsPath) -> VfsError { + VfsError::Backend(format!( + "the failing backend rejects every operation: {path}" + )) + } +} + +impl Vfs for FailingBackend { + fn acquire(&mut self, id: ExecId) -> std::result::Result, VfsError> { + let _ = id; + Ok(Box::new(FailingAccess)) + } -impl FailingStore { - fn error(path: &str) -> StoreError { - StoreError::not_found(path) + fn release(&mut self, id: ExecId) -> std::result::Result<(), VfsError> { + let _ = id; + Ok(()) } } -impl Store for FailingStore { - fn write(&mut self, path: &str, _contents: &str) -> std::result::Result<(), StoreError> { - Err(Self::error(path)) +struct FailingAccess; + +impl VfsAccess for FailingAccess { + fn read(&self, path: &VfsPath) -> std::result::Result, VfsError> { + Err(FailingBackend::error(*path)) + } + + fn write(&mut self, path: &VfsPath, _contents: &[u8]) -> std::result::Result<(), VfsError> { + Err(FailingBackend::error(*path)) + } + + fn append(&mut self, path: &VfsPath, _contents: &[u8]) -> std::result::Result<(), VfsError> { + Err(FailingBackend::error(*path)) } - fn append(&mut self, path: &str, _contents: &str) -> std::result::Result<(), StoreError> { - Err(Self::error(path)) + fn remove(&mut self, path: &VfsPath, _recursive: bool) -> std::result::Result<(), VfsError> { + Err(FailingBackend::error(*path)) } - fn read(&self, path: &str) -> std::result::Result { - Err(Self::error(path)) + fn exists(&self, path: &VfsPath) -> std::result::Result { + Err(FailingBackend::error(*path)) } - fn str_replace( - &mut self, - path: &str, - _old: &str, - _new: &str, - ) -> std::result::Result<(), StoreError> { - Err(Self::error(path)) + fn glob(&self, pattern: &str) -> std::result::Result, VfsError> { + Err(VfsError::Backend(format!( + "the failing backend rejects every operation: {pattern}" + ))) } - fn delete(&mut self, path: &str) -> std::result::Result<(), StoreError> { - Err(Self::error(path)) + fn list(&self, path: &VfsPath) -> std::result::Result, VfsError> { + Err(FailingBackend::error(*path)) } - fn glob(&self, pattern: &str) -> std::result::Result, StoreError> { - Err(Self::error(pattern)) + fn stat(&self, path: &VfsPath) -> std::result::Result { + Err(FailingBackend::error(*path)) } - fn exists(&self, path: &str) -> std::result::Result { - Err(Self::error(path)) + fn mkdir(&mut self, path: &VfsPath, _recursive: bool) -> std::result::Result<(), VfsError> { + Err(FailingBackend::error(*path)) } + + fn rename(&mut self, from: &VfsPath, _to: &VfsPath) -> std::result::Result<(), VfsError> { + Err(FailingBackend::error(*from)) + } + + fn copy(&mut self, from: &VfsPath, _to: &VfsPath) -> std::result::Result<(), VfsError> { + Err(FailingBackend::error(*from)) + } +} + +/// The access a failing backend vends, for tests driving the error path. +fn failing_access() -> Arc { + Arc::new(VfsRef::new(FailingBackend).acquire()) } struct BoundaryRecorder { - store: StoreRef, + access: Arc, snapshots: Mutex>>, } impl Observer for BoundaryRecorder { fn observe(&self, _execution: &str, _section: &str, _event: Observation) { + // The recorder shares the VM's identity, so its glob never meets a + // second live identity's claims. self.snapshots .lock() .expect("the snapshot mutex must not be poisoned") - .push(self.store.glob("**").expect("the memory store can glob")); + .push( + Store::new(&self.access) + .glob("**") + .expect("the memory store can glob"), + ); } } @@ -111,7 +162,7 @@ fn run(source: &str, args: &str) -> Result { source, args, &json!({ "id": 1, "when": "t" }), - &StoreRef::memory(), + &fresh_access(), EXECUTION, &null_observer(), "Test", @@ -125,14 +176,14 @@ fn test_nonce() -> GuardNonce { GuardNonce::fresh() } -/// Run a chunk against a caller-supplied store, so a test can inspect the -/// store after the chunk has run. -fn run_with(source: &str, store: &StoreRef) -> Result { +/// Run a chunk against a caller-supplied access, so a test can inspect the +/// store through the same identity after the chunk has run. +fn run_with(source: &str, access: &Arc) -> Result { run_chunk( source, "", &json!({ "id": 1, "when": "t" }), - store, + access, EXECUTION, &null_observer(), "Test", @@ -257,12 +308,12 @@ fn section_vm_with_bindings( fn section_vm_with_shared( shared: &LuaProgram, args: &str, - store: &StoreRef, + access: &Arc, observer: &Arc, section: &str, ) -> Result { let mut vm = SectionVm::new(&test_nonce(), EXECUTION, observer.as_ref(), section)?; - vm.inject_host(args, &json!({}), store)?; + vm.inject_host(args, &json!({}), access)?; vm.install_host_apis(observer, section)?; vm.replay_shared(shared, observer.as_ref(), section)?; Ok(vm) @@ -294,14 +345,9 @@ fn fixture_bindings(source: &str) -> ToolSet { #[test] fn direct_output_is_absent_in_every_executable_lua_vm() { let library = program("assert(print == nil); assert(warn == nil); log('library load')"); - let library_vm = section_vm_with_shared( - &library, - "", - &StoreRef::memory(), - &null_observer(), - "Section", - ) - .expect("library VM must not expose direct output"); + let library_vm = + section_vm_with_shared(&library, "", &fresh_access(), &null_observer(), "Section") + .expect("library VM must not expose direct output"); library_vm.teardown(&NullObserver::default(), "Section"); let shared = program( @@ -321,7 +367,7 @@ fn direct_output_is_absent_in_every_executable_lua_vm() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("section VM must not expose direct output"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); run_scalar( &vm, @@ -361,7 +407,7 @@ fn logs_are_correlated_and_ordered_across_chunks() { ); let mut vm = section_vm_with_bindings(&bindings, EXECUTION, recorder.as_ref(), "Gather") .expect("section VM must install captured bindings"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); let observer: Arc = recorder.clone(); vm.install_host_apis(&observer, "Gather") @@ -439,7 +485,7 @@ fn compatibility_chunk_logs_interleave_with_host_operations() { log('after write')", "", &json!({}), - &StoreRef::memory(), + &fresh_access(), "compatibility-run", &observer, "Compatibility", @@ -498,7 +544,7 @@ fn log_accepts_exactly_one_bounded_control_free_utf8_string() { source, "", &json!({}), - &StoreRef::memory(), + &fresh_access(), EXECUTION, &observer, "Validation", @@ -537,7 +583,7 @@ fn log_accepts_exactly_one_bounded_control_free_utf8_string() { &source, "", &json!({}), - &StoreRef::memory(), + &fresh_access(), EXECUTION, &observer, "Validation", @@ -564,7 +610,7 @@ fn log_cumulative_byte_budget_is_enforced_before_the_event_budget() { .expect("VM builds"); vm.apply_lua_limits(DEFAULT_LUA_MEMORY_BYTES, 4) .expect("limits apply"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host injects"); let recorder = Arc::new(Recorder::default()); let observer: Arc = recorder.clone(); @@ -607,25 +653,27 @@ fn logging_does_not_change_results_or_store_effects_with_null_observer() { var.answer = args\n\ store.write('answer.txt', args)\n\ return var.answer"; - let recorded_store = StoreRef::memory(); + let recorded_access = fresh_access(); + let recorded_store = Store::new(&recorded_access); let recorder = Arc::new(Recorder::default()); let observer: Arc = recorder.clone(); let observed_outcome = run_chunk( source, "same", &json!({}), - &recorded_store, + &recorded_access, EXECUTION, &observer, "Equivalence", ) .expect("recorded execution must succeed"); - let null_store = StoreRef::memory(); + let null_access = fresh_access(); + let null_store = Store::new(&null_access); let silent = run_chunk( source, "same", &json!({}), - &null_store, + &null_access, EXECUTION, &null_observer(), "Equivalence", @@ -657,7 +705,7 @@ fn installed_log_persists_across_chunks() { "Section", ) .expect("VM must construct"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); vm.install_host_apis(&observer, "Section") .expect("host APIs must install"); @@ -698,7 +746,7 @@ fn concurrent_logs_keep_execution_ids_and_local_order() { "log('first'); log('second')", "", &json!({}), - &StoreRef::memory(), + &fresh_access(), execution, &observer, "Concurrent", @@ -773,7 +821,7 @@ fn tool_handles_are_frozen() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); let error = run_scalar( &vm, @@ -927,7 +975,7 @@ fn captured_bindings_do_not_execute_h1_source() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install without executing H1"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); run_scalar( &vm, @@ -949,7 +997,7 @@ fn h2_recording_closes_to_always_then_added_scope() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); run_scalar(&vm, &prologue, &NullObserver::default(), "Section") .expect("H2 additions must record"); @@ -1005,7 +1053,7 @@ fn h2_add_accepts_tool_objects_and_arrays() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); run_scalar(&vm, &prologue, &NullObserver::default(), "Section") .expect("tools.add must accept Tool objects, strings, and arrays"); @@ -1034,7 +1082,7 @@ fn empty_add_is_a_no_op_and_failed_bulk_add_is_atomic() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); run_scalar(&vm, &prologue, &NullObserver::default(), "Section") .expect("caught failed add must not poison recording"); @@ -1069,7 +1117,7 @@ fn add_rejects_misshapen_override_arguments() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); run_scalar(&vm, &prologue, &NullObserver::default(), "Section") .expect("rejected override forms must not poison recording"); @@ -1095,7 +1143,7 @@ fn tool_operations_enforce_their_lifecycle_phase_even_when_captured() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); let error = run_scalar( @@ -1118,7 +1166,7 @@ fn unknown_h2_alias_fails_before_scope_closure() { let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Section") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); let error = run_scalar( &vm, @@ -1143,7 +1191,7 @@ fn captured_bindings_are_installed_without_payload_reports() { let recorder = Recorder::default(); let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &recorder, "Section") .expect("captured binding installation must succeed"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host must inject"); let trace = format!("{:?}", recorder.observations()); assert!(!trace.contains("private_alias")); @@ -1175,13 +1223,14 @@ fn section_vm_preserves_one_environment_across_all_phases() { let epilog = program( "return decorate(phase_marker) .. ':' .. shared_saw_args .. ':' .. shared_saw_store", ); - let store = StoreRef::memory(); + let access = fresh_access(); + let store = Store::new(&access); store .write("seed.txt", "seeded") .expect("the memory store can seed a file"); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("input", &json!({ "id": 7 }), &store) + vm.inject_host("input", &json!({ "id": 7 }), &access) .expect("host values must inject"); let null_observer: Arc = Arc::new(NullObserver::default()); vm.install_host_apis(&null_observer, "Test") @@ -1218,7 +1267,7 @@ fn section_vm_preserves_one_environment_across_all_phases() { #[test] fn section_vm_requires_delayed_single_host_injection() { let no_op = program("return args"); - let store = StoreRef::memory(); + let access = fresh_access(); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); @@ -1226,10 +1275,10 @@ fn section_vm_requires_delayed_single_host_injection() { .expect_err("programs cannot run before host injection"); assert!(error.to_string().contains("not been injected")); - vm.inject_host("first", &json!({}), &store) + vm.inject_host("first", &json!({}), &access) .expect("first injection must succeed"); let error = vm - .inject_host("second", &json!({}), &store) + .inject_host("second", &json!({}), &access) .expect_err("host values cannot be replaced"); assert!(error.to_string().contains("already injected")); } @@ -1263,7 +1312,7 @@ fn section_vm_host_injection_bypasses_shared_global_metatables() { "Test", ) .expect("VM must build"); - vm.inject_host("private input", &json!({}), &StoreRef::memory()) + vm.inject_host("private input", &json!({}), &fresh_access()) .expect("host values must inject"); let observer = null_observer(); vm.install_host_apis(&observer, "Test") @@ -1288,7 +1337,7 @@ fn section_vm_reports_store_operations_in_each_chunk() { let recorder = Arc::new(Recorder::default()); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Gather") .expect("VM must build"); - vm.inject_host("private input", &json!({}), &StoreRef::memory()) + vm.inject_host("private input", &json!({}), &fresh_access()) .expect("host values must inject"); let observer: Arc = recorder.clone(); vm.install_host_apis(&observer, "Gather") @@ -1318,7 +1367,7 @@ fn section_vm_reports_store_operations_in_each_chunk() { #[test] fn section_vm_accepts_only_scalar_top_level_returns() { - let store = StoreRef::memory(); + let access = fresh_access(); for (source, expected) in [ ("return 'text'", Some("text")), ("return 42", Some("42")), @@ -1328,7 +1377,7 @@ fn section_vm_accepts_only_scalar_top_level_returns() { ] { let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &store) + vm.inject_host("", &json!({}), &access) .expect("host values must inject"); assert_eq!( run_scalar(&vm, &program(source), &NullObserver::default(), "Test") @@ -1340,7 +1389,7 @@ fn section_vm_accepts_only_scalar_top_level_returns() { let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &store) + vm.inject_host("", &json!({}), &access) .expect("host values must inject"); let error = run_scalar(&vm, &program("return {}"), &NullObserver::default(), "Test") .expect_err("table returns must be refused"); @@ -1351,10 +1400,10 @@ fn section_vm_accepts_only_scalar_top_level_returns() { fn section_vms_isolate_mutated_shared_globals() { let shared = program("counter = 0"); let increment = program("counter = counter + 1; return counter"); - let store = StoreRef::memory(); - let first = section_vm_with_shared(&shared, "", &store, &null_observer(), "First") + let access = fresh_access(); + let first = section_vm_with_shared(&shared, "", &access, &null_observer(), "First") .expect("first VM must build"); - let second = section_vm_with_shared(&shared, "", &store, &null_observer(), "Second") + let second = section_vm_with_shared(&shared, "", &access, &null_observer(), "Second") .expect("second VM must build"); assert_eq!( @@ -1399,7 +1448,7 @@ fn shared_replay_consumes_the_configured_log_budget() { .expect("VM builds"); vm.apply_lua_limits(DEFAULT_LUA_MEMORY_BYTES, 1) .expect("limits apply"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host injects"); let observer = null_observer(); vm.install_host_apis(&observer, "Budget") @@ -1431,7 +1480,7 @@ fn the_memory_budget_error_stays_reachable() { .expect("VM builds"); vm.apply_lua_limits(4 * 1024 * 1024, DEFAULT_LUA_LOG_EVENTS) .expect("limits apply"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host injects"); let observer = null_observer(); vm.install_host_apis(&observer, "Budget") @@ -1459,7 +1508,7 @@ fn jump_during_shared_replay_is_a_hard_error() { let shared = program("jump('## Anywhere')"); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let observer = null_observer(); vm.install_host_apis(&observer, "Test") @@ -1493,7 +1542,7 @@ fn call_with_a_non_string_target_errors() { // heading, and the error says so. let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let observer = null_observer(); vm.install_host_apis(&observer, "Test") @@ -1550,7 +1599,7 @@ fn shared_replay_sees_the_tables_but_not_the_bare_alias_globals() { "Test", ) .expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let observer = null_observer(); vm.install_host_apis(&observer, "Test") @@ -1608,7 +1657,7 @@ fn shared_functions_resolve_host_globals_when_called_from_a_later_chunk() { "Test", ) .expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let observer = null_observer(); vm.install_host_apis(&observer, "Test") @@ -1644,7 +1693,7 @@ fn absent_shared_library_replays_an_empty_chunk_on_the_same_path() { let recorder = Arc::new(Recorder::default()); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, recorder.as_ref(), "Test").expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let observer: Arc = recorder.clone(); vm.install_host_apis(&observer, "Test") @@ -1683,7 +1732,7 @@ fn section_lifecycle_reports_are_ordered_exact_and_payload_free() { let recorder = Arc::new(Recorder::default()); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, recorder.as_ref(), "Gather") .expect("VM must build"); - vm.inject_host("private input", &json!({}), &StoreRef::memory()) + vm.inject_host("private input", &json!({}), &fresh_access()) .expect("host values must inject"); let observer: Arc = recorder.clone(); vm.install_host_apis(&observer, "Gather") @@ -1722,7 +1771,7 @@ fn section_lifecycle_failures_report_their_phase() { let failing_shared = program("error('private shared failure')"); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, recorder.as_ref(), "Shared") .expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let observer: Arc = recorder.clone(); vm.install_host_apis(&observer, "Shared") @@ -2288,7 +2337,7 @@ async fn a_pre_cancelled_run_aborts_a_tight_loop_promptly() { tokio::task::block_in_place(|| { let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Loop")?; - vm.inject_host("", &json!({}), &StoreRef::memory())?; + vm.inject_host("", &json!({}), &fresh_access())?; let observer = null_observer(); vm.install_host_apis(&observer, "Loop")?; let result = run_scalar( @@ -2330,7 +2379,7 @@ fn add_without_declarations_fails_as_undeclared_in_a_chunk() { fn add_without_declarations_fails_in_a_prologue_without_a_shared_library() { let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let error = run_scalar( &vm, @@ -2363,7 +2412,7 @@ fn add_with_empty_frozen_bindings_fails_as_undeclared() { assert!(bindings.bindings().is_empty()); let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Test") .expect("empty captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let error = run_scalar( &vm, @@ -2384,7 +2433,7 @@ fn add_with_an_override_argument_records_the_model_description() { let bindings = fixture_bindings("tools.bind('search', 'search the web')"); let mut vm = section_vm_with_bindings(&bindings, EXECUTION, &NullObserver::default(), "Test") .expect("captured bindings must install"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); run_scalar( &vm, @@ -2407,7 +2456,7 @@ fn add_with_an_override_argument_records_the_model_description() { fn a_section_vm_without_declarations_snapshots_to_an_empty_scope() { let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host values must inject"); let (bindings, runtime) = vm.tool_bag_handles(); let scope = current_tool_bindings(&bindings, &runtime).expect("an empty scope must snapshot"); @@ -2419,9 +2468,10 @@ fn a_section_vm_without_declarations_snapshots_to_an_empty_scope() { #[test] fn store_exists_returns_boolean() { - let store = StoreRef::memory(); + let access = fresh_access(); + let store = Store::new(&access); assert_eq!( - run_with("return tostring(store.exists('missing.txt'))", &store) + run_with("return tostring(store.exists('missing.txt'))", &access) .unwrap() .returned .as_deref(), @@ -2429,7 +2479,7 @@ fn store_exists_returns_boolean() { ); store.write("a.txt", "hi").expect("write"); assert_eq!( - run_with("return tostring(store.exists('a.txt'))", &store) + run_with("return tostring(store.exists('a.txt'))", &access) .unwrap() .returned .as_deref(), @@ -2438,7 +2488,7 @@ fn store_exists_returns_boolean() { assert_eq!( run_with( "store.delete('a.txt')\nreturn tostring(store.exists('a.txt'))", - &store, + &access, ) .unwrap() .returned @@ -2604,10 +2654,10 @@ fn store_read_end_without_start_raises() { #[test] fn store_read_numbered_without_bounds_numbers_from_one() { - let store = StoreRef::memory(); + let access = fresh_access(); let out = run_with( "store.write('a.txt', 'first\\nsecond')\nreturn store.read_numbered('a.txt')", - &store, + &access, ) .unwrap(); assert_eq!(out.returned.as_deref(), Some("1| first\n2| second")); @@ -2615,27 +2665,29 @@ fn store_read_numbered_without_bounds_numbers_from_one() { #[test] fn store_read_numbered_numbers_a_slice_absolutely() { - let store = StoreRef::memory(); + let access = fresh_access(); + let store = Store::new(&access); let mut body = String::new(); for n in 1..=85 { use std::fmt::Write as _; let _ = writeln!(body, "line{n}"); } store.write("a.txt", &body).expect("write"); - let out = run_with("return store.read_numbered('a.txt', 84, 85)", &store).unwrap(); + let out = run_with("return store.read_numbered('a.txt', 84, 85)", &access).unwrap(); assert_eq!(out.returned.as_deref(), Some("84| line84\n85| line85")); } #[test] fn store_read_numbered_pads_across_the_hundred_boundary() { - let store = StoreRef::memory(); + let access = fresh_access(); + let store = Store::new(&access); let mut body = String::new(); for n in 1..=100 { use std::fmt::Write as _; let _ = writeln!(body, "line{n}"); } store.write("a.txt", &body).expect("write"); - let out = run_with("return store.read_numbered('a.txt', 99, 100)", &store).unwrap(); + let out = run_with("return store.read_numbered('a.txt', 99, 100)", &access).unwrap(); assert_eq!(out.returned.as_deref(), Some(" 99| line99\n100| line100")); } @@ -2704,13 +2756,14 @@ fn store_read_numbered_end_without_start_raises() { #[test] fn installed_store_read_honors_line_bounds() { - let store = StoreRef::memory(); + let access = fresh_access(); + let store = Store::new(&access); store .write("a.txt", "one\ntwo\nthree\n") .expect("the memory store can prepare a file"); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &store) + vm.inject_host("", &json!({}), &access) .expect("host values must inject"); let observer: Arc = Arc::new(NullObserver::default()); vm.install_host_apis(&observer, "Test") @@ -2740,13 +2793,14 @@ fn installed_store_read_honors_line_bounds() { #[test] fn installed_store_read_numbered_honors_line_bounds() { - let store = StoreRef::memory(); + let access = fresh_access(); + let store = Store::new(&access); store .write("a.txt", "one\ntwo\nthree\n") .expect("the memory store can prepare a file"); let mut vm = SectionVm::new(&test_nonce(), EXECUTION, &NullObserver::default(), "Test") .expect("VM must build"); - vm.inject_host("", &json!({}), &store) + vm.inject_host("", &json!({}), &access) .expect("host values must inject"); let observer: Arc = Arc::new(NullObserver::default()); vm.install_host_apis(&observer, "Test") @@ -2829,8 +2883,9 @@ fn lua_runtime_error_preserves_its_mlua_source() { fn store_writes_are_visible_on_the_shared_handle() { // The table is backed by the caller's handle, so a write from Lua is // observable through a clone of that same handle after the chunk ends. - let store = StoreRef::memory(); - run_with("store.write('shared.txt', 'from lua')", &store).unwrap(); + let access = fresh_access(); + let store = Store::new(&access); + run_with("store.write('shared.txt', 'from lua')", &access).unwrap(); assert_eq!( store.read("shared.txt").expect("read"), "from lua", @@ -2842,7 +2897,7 @@ fn store_writes_are_visible_on_the_shared_handle() { fn store_reports_are_ordered_exact_and_payload_free_on_failure() { let recorder = Arc::new(Recorder::default()); let observer: Arc = recorder.clone(); - let store = StoreRef::memory(); + let access = fresh_access(); let source = "store.write('secret/path.txt', 'private contents')\n\ store.read('secret/path.txt')\n\ store.str_replace('secret/path.txt', 'missing secret', 'replacement')"; @@ -2850,7 +2905,7 @@ fn store_reports_are_ordered_exact_and_payload_free_on_failure() { source, "private input", &json!({ "id": 1, "when": "t" }), - &store, + &access, EXECUTION, &observer, "Gather", @@ -2892,13 +2947,13 @@ fn every_store_operation_reports_its_exact_success_and_failure() { source: &'static str, success: Observation, failure: Observation, - prepare: fn(&StoreRef), + prepare: fn(&Arc), } - fn empty(_store: &StoreRef) {} + fn empty(_access: &Arc) {} - fn existing(store: &StoreRef) { - store + fn existing(access: &Arc) { + Store::new(access) .write("a.txt", "old") .expect("the memory store can prepare a file"); } @@ -2961,44 +3016,44 @@ fn every_store_operation_reports_its_exact_success_and_failure() { ]; for case in cases { - let store = StoreRef::memory(); - (case.prepare)(&store); + let access = fresh_access(); + (case.prepare)(&access); let recorder = Arc::new(Recorder::default()); let observer: Arc = recorder.clone(); run_chunk( case.source, "", &json!({}), - &store, + &access, EXECUTION, &observer, - "StoreRef", + "Store", ) .expect("the memory store operation succeeds"); assert_eq!( recorder.observations(), - vec![("StoreRef".to_owned(), case.success.clone())], + vec![("Store".to_owned(), case.success.clone())], "wrong success observation for {}", case.source ); - let store = StoreRef::new(Box::new(FailingStore)); + let access = failing_access(); let recorder = Arc::new(Recorder::default()); let observer: Arc = recorder.clone(); let error = run_chunk( case.source, "", &json!({}), - &store, + &access, EXECUTION, &observer, - "StoreRef", + "Store", ) .expect_err("the failing backend rejects every operation"); assert!(matches!(error, Error::Lua(_) | Error::LuaRuntime { .. })); assert_eq!( recorder.observations(), - vec![("StoreRef".to_owned(), case.failure.clone())], + vec![("Store".to_owned(), case.failure.clone())], "wrong failure observation for {}", case.source ); @@ -3007,9 +3062,9 @@ fn every_store_operation_reports_its_exact_success_and_failure() { #[test] fn store_observations_happen_before_later_lua_side_effects() { - let store = StoreRef::memory(); + let access = fresh_access(); let recorder = Arc::new(BoundaryRecorder { - store: store.clone(), + access: Arc::clone(&access), snapshots: Mutex::new(Vec::new()), }); let observer: Arc = recorder.clone(); @@ -3018,10 +3073,10 @@ fn store_observations_happen_before_later_lua_side_effects() { "store.write('first.txt', '')\nstore.write('second.txt', '')", "", &json!({}), - &store, + &access, EXECUTION, &observer, - "StoreRef", + "Store", ) .expect("both writes succeed"); diff --git a/crates/promptforge-lua/src/tools/tests.rs b/crates/promptforge-lua/src/tools/tests.rs index cecbfc236..5bdc3767a 100644 --- a/crates/promptforge-lua/src/tools/tests.rs +++ b/crates/promptforge-lua/src/tools/tests.rs @@ -1,7 +1,6 @@ use mlua::{Lua, Value, Variadic}; use promptforge_core_support::observe::NullObserver; use promptforge_core_support::untrusted::GuardNonce; -use promptforge_store::StoreRef; use serde_json::json; use super::decode::{add_local_params_schema, collect_tools_add_entries, tool_alias}; @@ -13,6 +12,11 @@ use crate::{SectionVm, ToolBinding}; use promptforge_tools::ToolId; use std::sync::{Arc, Mutex}; +/// A fresh stock handle's access capability for a test VM. +fn fresh_access() -> Arc { + Arc::new(promptforge_vfs::empty().acquire()) +} + fn echo_handle() -> LuaToolHandle { LuaToolHandle::from_binding( "echo", @@ -189,7 +193,7 @@ fn the_shim_prelude_installs_tools_call_and_no_bare_global() { let observer = NullObserver::default(); let mut vm = SectionVm::new(&nonce, "test-run", &observer, "Test") .expect("section VM construction cannot fail"); - vm.inject_host("", &json!({}), &StoreRef::memory()) + vm.inject_host("", &json!({}), &fresh_access()) .expect("host injection cannot fail"); vm.install_coro_shims().expect("the shim prelude installs"); let (call_is_function, bare_is_nil): (bool, bool) = vm diff --git a/crates/promptforge-lua/src/vm.rs b/crates/promptforge-lua/src/vm.rs index d08964391..7dc9a3ade 100644 --- a/crates/promptforge-lua/src/vm.rs +++ b/crates/promptforge-lua/src/vm.rs @@ -1,13 +1,13 @@ #[cfg(test)] use super::LuaFanoutResult; use super::{ - Arc, AtomicU32, AtomicUsize, BTreeMap, DEFAULT_LUA_LOG_EVENTS, DEFAULT_LUA_MEMORY_BYTES, Error, - Function, GuardNonce, InstructionBudget, IntoLuaMulti, Json, Lua, LuaBlockResult, - LuaModelHandle, LuaOptions, LuaProgram, LuaSerdeExt, LuaToolHandle, ModelBinding, ModelRuntime, - ModelSet, ModelView, ModelsInferHook, MultiValue, Mutex, Observer, Ordering, ProseState, - Result, StdLib, StoreRef, Thread, ThreadStatus, ToolBinding, ToolCallCounts, ToolRuntime, - ToolSet, Value, WriteScope, detail, guarded_var, harden, install_compactors, install_h2_models, - install_h2_tools, install_instruction_budget, install_log, install_messages, + Access, Arc, AtomicU32, AtomicUsize, BTreeMap, DEFAULT_LUA_LOG_EVENTS, + DEFAULT_LUA_MEMORY_BYTES, Error, Function, GuardNonce, InstructionBudget, IntoLuaMulti, Json, + Lua, LuaBlockResult, LuaModelHandle, LuaOptions, LuaProgram, LuaSerdeExt, LuaToolHandle, + ModelBinding, ModelRuntime, ModelSet, ModelView, ModelsInferHook, MultiValue, Mutex, Observer, + Ordering, ProseState, Result, StdLib, Thread, ThreadStatus, ToolBinding, ToolCallCounts, + ToolRuntime, ToolSet, Value, detail, guarded_var, harden, install_compactors, + install_h2_models, install_h2_tools, install_instruction_budget, install_log, install_messages, install_shim_prelude, install_store_table, install_tool_call_counts as install_tool_call_counts_impl, install_untrusted, log_byte_budget, resolve_section_target, scalar_return, seal_sys, var_to_json, @@ -74,10 +74,10 @@ pub struct SectionVm { /// Live sealed `sys` JSON, mirrored for [`current_sys`](Self::current_sys) /// snapshots. sys_live: Arc>>, - store: Option, - /// The fanout arm's write scope for `store.write`; `None` outside an arm - /// leaves walk-section writes untracked. - write_scope: Option, + /// The section's VFS access capability: the `store` table's closures + /// share it, so every store op is attributed to the identity the + /// executor installed for this chain step. + access: Option>, host_injected: bool, /// Remaining `log()` events this VM may emit before the budget is exhausted. log_budget: Arc, @@ -262,8 +262,7 @@ impl SectionVm { model_runtime: Arc::new(Mutex::new(ModelRuntime::new())), jump_slot: Arc::new(Mutex::new(None)), sys_live: Arc::new(Mutex::new(None)), - store: None, - write_scope: None, + access: None, host_injected: false, log_budget: Arc::new(AtomicU32::new(DEFAULT_LUA_LOG_EVENTS)), log_byte_budget: Arc::new(AtomicUsize::new(log_byte_budget(DEFAULT_LUA_LOG_EVENTS))), @@ -409,17 +408,18 @@ impl SectionVm { /// ```text /// use promptforge_lua::SectionVm; /// use promptforge_core_support::observe::NullObserver; - /// use promptforge_store::StoreRef; /// use promptforge_core_support::untrusted::GuardNonce; /// /// let nonce = GuardNonce::fresh(); + /// let vfs = promptforge_vfs::empty(); + /// let access = std::sync::Arc::new(vfs.acquire()); /// let mut vm = SectionVm::new(&nonce, "example-run", &NullObserver::default(), "Example")?; - /// vm.inject_host("input", &serde_json::json!({ "id": 1 }), &StoreRef::memory())?; + /// vm.inject_host("input", &serde_json::json!({ "id": 1 }), &access)?; /// vm.teardown(&NullObserver::default(), "Example"); /// # Ok::<(), promptforge_lua::Error>(()) /// ``` - pub fn inject_host(&mut self, args: &str, sys: &Json, store: &StoreRef) -> Result<()> { - self.inject_host_with_var(args, sys, store, None, None) + pub fn inject_host(&mut self, args: &str, sys: &Json, access: &Arc) -> Result<()> { + self.inject_host_with_var(args, sys, access, None) } /// Installs host values while seeding `var` from an earlier VM. @@ -427,8 +427,10 @@ impl SectionVm { /// The `var` global is a guarded proxy (see [`guarded_var`]): writes are /// validated for JSON-representability at the assigning line, and the /// hidden data table behind it is what [`var`](Self::var) reads back. - /// `write_scope` is the fanout arm's store-write identity; it is `None` - /// for every other driver, leaving `store.write` untracked. + /// `access` is the chain step's VFS capability: the `store` table's + /// closures share it, so a fanout arm's store ops carry the arm's + /// spawned identity and a conflicting second live identity surfaces as + /// a write race. /// /// # Errors /// Returns [`Error::Lua`] if host values cannot be bridged or were already @@ -437,9 +439,8 @@ impl SectionVm { &mut self, args: &str, sys: &Json, - store: &StoreRef, + access: &Arc, initial_var: Option<&Json>, - write_scope: Option, ) -> Result<()> { if self.host_injected { return Err(Error::Lua( @@ -476,8 +477,7 @@ impl SectionVm { )?; install_messages(&self.lua, &globals)?; install_compactors(&self.lua, &globals)?; - self.store = Some(store.clone()); - self.write_scope = write_scope; + self.access = Some(Arc::clone(access)); self.host_injected = true; Ok(()) } @@ -494,7 +494,7 @@ impl SectionVm { /// Returns [`Error::Lua`] if host values have not been injected or the /// globals cannot be installed. pub fn install_host_apis(&self, observer: &Arc, section: &str) -> Result<()> { - let store = self.store.as_ref().ok_or_else(|| { + let access = self.access.as_ref().ok_or_else(|| { Error::Lua("section VM host values have not been injected".to_owned()) })?; install_log( @@ -508,11 +508,10 @@ impl SectionVm { install_store_table( &self.lua, &self.lua.globals(), - store, + access, &self.execution, observer, section, - self.write_scope, ) } @@ -731,7 +730,7 @@ impl SectionVm { /// /// This is the legacy engine's path for running a section's Lua blocks; /// the scheduler drives blocks through - /// [`start_block_coro`](Self::start_block_coro) instead. StoreRef and + /// [`start_block_coro`](Self::start_block_coro) instead. Store and /// `log` reports go to the observer captured by /// [`install_host_apis`](Self::install_host_apis); a nil or absent /// top-level return produces [`LuaBlockResult::Returned`]`(None)`. When @@ -782,12 +781,13 @@ impl SectionVm { /// ```text /// use promptforge_lua::SectionVm; /// use promptforge_core_support::observe::NullObserver; - /// use promptforge_store::StoreRef; /// use promptforge_core_support::untrusted::GuardNonce; /// /// let nonce = GuardNonce::fresh(); + /// let vfs = promptforge_vfs::empty(); + /// let access = std::sync::Arc::new(vfs.acquire()); /// let mut vm = SectionVm::new(&nonce, "example-run", &NullObserver::default(), "Example")?; - /// vm.inject_host("", &serde_json::json!({}), &StoreRef::memory())?; + /// vm.inject_host("", &serde_json::json!({}), &access)?; /// assert_eq!(vm.var()?, serde_json::json!({})); /// vm.teardown(&NullObserver::default(), "Example"); /// # Ok::<(), promptforge_lua::Error>(()) @@ -1210,13 +1210,13 @@ pub(crate) fn run_chunk( source: &str, args: &str, sys: &Json, - store: &StoreRef, + access: &Arc, execution: &str, observer: &Arc, section: &str, ) -> Result { let mut vm = SectionVm::new(&GuardNonce::fresh(), execution, observer.as_ref(), section)?; - vm.inject_host(args, sys, store)?; + vm.inject_host(args, sys, access)?; vm.install_host_apis(observer, section)?; let returned: MultiValue = vm.lua.load(source).eval().map_err(Error::lua)?; let returned = scalar_return(returned)?; diff --git a/crates/promptforge-store/README.md b/crates/promptforge-store/README.md index 6e02309ab..1454d271d 100644 --- a/crates/promptforge-store/README.md +++ b/crates/promptforge-store/README.md @@ -1,10 +1,13 @@ # promptforge-store -The PromptForge run-scoped virtual filesystem. A prompt run keeps its bulk -state in virtual files addressed by logical string paths: `Store` is the -backend contract, `MemStore` and `FileStore` are the in-memory and -filesystem backends, and `StoreRef` is the cheaply cloneable, thread-safe -handle the runtime shares between the Lua VM and the model's file tools. +The PromptForge run-scoped virtual filesystem facade. A prompt run keeps its +bulk state in virtual files addressed by logical string paths: `Store` is a +concrete facade over a prefix-scoped VFS access capability from `shared-vfs` +(the run's `VfsRef` carries the store mount, installed by +`promptforge-vfs`'s stock constructors), exposed as `vfs.store(&access)` +through the prelude-exported `StoreExt` extension trait. Every operation is +attributed to the access's identity, so a conflicting operation by a second +live identity surfaces as `StoreError::WriteRace`. Reads are verbatim, ranged reads slice 1-based inclusive line ranges (plain or absolutely numbered), edits are anchor-based (`Store::str_replace`), and diff --git a/crates/promptforge-store/src/error.rs b/crates/promptforge-store/src/error.rs index 5db7f37e7..8c1965c09 100644 --- a/crates/promptforge-store/src/error.rs +++ b/crates/promptforge-store/src/error.rs @@ -2,8 +2,8 @@ /// Why a logical store path was rejected before any backend saw it. /// -/// `StoreRef` validates every caller-supplied path into one canonical form -/// before dispatch; this names the rule the path broke. +/// The `Store` facade validates every caller-supplied path into one +/// canonical form before dispatch; this names the rule the path broke. #[derive(Debug, Clone, Copy, PartialEq, Eq)] #[non_exhaustive] pub enum PathReason { diff --git a/crates/promptforge-store/src/lib.rs b/crates/promptforge-store/src/lib.rs index bdbb58d3a..e3eea767c 100644 --- a/crates/promptforge-store/src/lib.rs +++ b/crates/promptforge-store/src/lib.rs @@ -25,7 +25,9 @@ mod path; use std::fmt::Write as _; use promptforge_vfs::STORE_MOUNT; -use shared_vfs::{Access, FileType, VfsError, VfsRef}; +use shared_vfs::{FileType, VfsError, VfsRef}; + +pub use shared_vfs::Access; pub use error::{PathReason, StoreError, StoreErrorKind}; use path::StorePath; @@ -61,6 +63,20 @@ pub struct Store<'a> { access: &'a Access, } +impl<'a> Store<'a> { + /// Returns the facade over one identity's capability, scoped to the + /// stock store mount. + /// + /// This is the constructor for holders that own the [`Access`] - the + /// Lua VM's store closures build a facade per call over their shared + /// `Arc`. Callers holding a `VfsRef` prefer the + /// [`StoreExt::store`] shape. + #[must_use] + pub fn new(access: &'a Access) -> Store<'a> { + Store { access } + } +} + impl Store<'_> { /// Creates or overwrites the file at `path`. /// diff --git a/crates/promptforge-store/src/path.rs b/crates/promptforge-store/src/path.rs index 2da1b77b2..525c31d9b 100644 --- a/crates/promptforge-store/src/path.rs +++ b/crates/promptforge-store/src/path.rs @@ -1,6 +1,7 @@ //! Logical store-path validation and canonicalization. //! -//! `StoreRef` parses every caller-supplied `&str` into a [`StorePath`] before +//! The `Store` facade parses every caller-supplied `&str` into a +//! [`StorePath`] before //! dispatch, so a backend never sees an empty, absolute, traversing, //! control-bearing, backslash-bearing, platform-reserved, or over-long path //! (STORE-003). @@ -36,9 +37,9 @@ fn is_numbered_device(name: &str, prefix: &str) -> bool { /// A validated logical store path in one canonical form. /// -/// `StoreRef` parses every caller-supplied `&str` into this before dispatch, so +/// The facade parses every caller-supplied `&str` into this before dispatch, so /// a backend never sees an empty, absolute, traversing, control-bearing, or -/// empty-segment path. The trait boundary keeps `&str`; this type is internal. +/// empty-segment path. The facade boundary keeps `&str`; this type is internal. #[derive(Debug, Clone, PartialEq, Eq)] pub(crate) struct StorePath(String); diff --git a/crates/workshop-server/Cargo.toml b/crates/workshop-server/Cargo.toml index 3fa5c5366..f8402481a 100644 --- a/crates/workshop-server/Cargo.toml +++ b/crates/workshop-server/Cargo.toml @@ -26,7 +26,7 @@ promptforge-core-support.workspace = true promptforge-model-client.workspace = true promptforge-tool-picker.workspace = true shared-progress.workspace = true -promptforge-store.workspace = true +promptforge-vfs.workspace = true promptforge-tools.workspace = true rand.workspace = true reqwest.workspace = true @@ -35,6 +35,7 @@ serde.workspace = true serde_json.workspace = true shared-loopback.workspace = true shared-sidecar.workspace = true +shared-vfs.workspace = true socket2.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/workshop-server/src/session_agents.rs b/crates/workshop-server/src/session_agents.rs index 91e0539b3..76fdb65d3 100644 --- a/crates/workshop-server/src/session_agents.rs +++ b/crates/workshop-server/src/session_agents.rs @@ -1071,7 +1071,7 @@ mod tests { .expect("the empty picker builds"); let models = ModelCatalog::empty(); let tools = promptforge_tools::ToolCatalog::new(&[]).expect("an empty catalog is valid"); - let store = promptforge_store::StoreRef::memory(); + let store = promptforge_vfs::empty(); let mut config = RunConfig::new("chat-unit").observer(observer); if let Some(broker) = broker { config = config.input_broker(broker); diff --git a/crates/workshop-server/src/session_agents/supervisor/effects.rs b/crates/workshop-server/src/session_agents/supervisor/effects.rs index 072cc7098..a1be766b6 100644 --- a/crates/workshop-server/src/session_agents/supervisor/effects.rs +++ b/crates/workshop-server/src/session_agents/supervisor/effects.rs @@ -8,9 +8,9 @@ use promptforge_core::{Prompt, ResolutionContext, RunConfig}; use promptforge_core_support::observe::Observer; use promptforge_model_client::client::{GatewayClient as ModelClient, StreamDelta}; use promptforge_model_client::model::ModelCatalog; -use promptforge_store::StoreRef; use promptforge_tool_picker::{Config, ToolPicker}; use promptforge_tools::ToolCatalog; +use shared_vfs::VfsRef; use crate::catalog::ChatCatalog; use crate::gateway_binding::GatewaySnapshot; @@ -38,7 +38,7 @@ pub(super) enum EffectOutcome { struct RunFactory { session: Arc, tools: ToolCatalog, - store: StoreRef, + vfs: VfsRef, observer: Arc, on_delta: Arc, ui: Arc serde_json::Value + Send + Sync>, @@ -68,7 +68,7 @@ impl RunFactory { ui: ui_provider(&host.menu, &host.workspace), session, tools, - store: StoreRef::memory(), + vfs: promptforge_vfs::empty(), observer, picker, } @@ -92,7 +92,7 @@ impl RunFactory { ) -> RunFuture { let tools = self.tools.clone(); let models = build_model_catalog(Some(models)); - let store = self.store.clone(); + let vfs = self.vfs.clone(); let config = AgentConfig { name: self.session.agent.clone(), execution: self.session.id.clone(), @@ -105,7 +105,7 @@ impl RunFactory { }; Box::pin(async move { let result = - run_agent_with_client(&source, &tools, &models, &store, config, Some(client)).await; + run_agent_with_client(&source, &tools, &models, &vfs, config, Some(client)).await; (run, result) }) } @@ -117,7 +117,7 @@ impl RunFactory { observer: Arc::clone(&self.observer), ui: Arc::clone(&self.ui), on_delta: Arc::clone(&self.on_delta), - store: self.store.clone(), + vfs: self.vfs.clone(), picker: Arc::clone( self.picker .as_ref() @@ -138,7 +138,7 @@ struct MarkdownRunParts { observer: Arc, ui: Arc serde_json::Value + Send + Sync>, on_delta: Arc, - store: StoreRef, + vfs: VfsRef, picker: Arc, } @@ -158,7 +158,7 @@ async fn run_markdown_agent( observer, ui, on_delta, - store, + vfs, picker, } = parts; let prompt = Prompt::parse(source, &session.id, observer.as_ref()).map_err(|error| { @@ -185,7 +185,7 @@ async fn run_markdown_agent( &prompt, "", ResolutionContext::new(picker.as_ref(), &models, &tools), - &store, + &vfs, config, ) .await diff --git a/crates/workshop-server/tests/it/chat_gate.rs b/crates/workshop-server/tests/it/chat_gate.rs index 12b182e52..de4f7fe6d 100644 --- a/crates/workshop-server/tests/it/chat_gate.rs +++ b/crates/workshop-server/tests/it/chat_gate.rs @@ -37,7 +37,6 @@ use promptforge_model_client::client::{ GatewayClient as ModelClient, GatewayEndpoint, SecretString, }; use promptforge_model_client::model::ModelCatalog; -use promptforge_store::StoreRef; use promptforge_tool_picker::{Catalog as PickerCatalog, Config as PickerConfig, ToolPicker}; use promptforge_tools::ToolCatalog; use workshop_server::fixtures::{gateway_updater, replace_gateway, state_with_gateway}; @@ -404,7 +403,7 @@ fn spawn_restored_chat( .expect("the embedded chat prompt parses"); let models = ModelCatalog::empty(); let tools = ToolCatalog::new(&[]).expect("an empty tool catalog is valid"); - let store = StoreRef::memory(); + let store = promptforge_vfs::empty(); promptforge_core::run( &prompt, "", diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index a7f5299e6..1ee86518f 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -644,7 +644,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 9: executor API pivot to VfsRef +### Step 9: executor API pivot to VfsRef [completed] - Component: promptforge-core - Change `execute::run(prompt, args, resolution, vfs: &VfsRef, config)`; `RunContext::new` takes the VfsRef and builds the Store facade internally; run() overlays a fresh memory store only as a defensive fallback for hand-built routers lacking the mount; the scheduler installs the current Access per chain step. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index cc9725cee..21d5a1fb3 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -115,3 +115,10 @@ - Decision: glob delegates matching to the backend and stat-filters to files only | Falsifier: glob latency complaints on large trees. - Decision: backslash glob rejection lives in the facade because the router canonicalizes patterns before the backend sees them | Falsifier: router forwarding verbatim patterns. - Decision: `GlobSpyStore`, `with_files`, and `FileStore` tests dropped; their mechanisms no longer exist (matching moved to the backend, `with_files` had no external callers, `FileStore` superseded by `HostBackend`) | Falsifier: a host depending on file-backed store behavior through this crate. +- Step 8 deferred leg closed: workspace clippy `-D warnings` passed clean during step 9's coding run after the caller migration - the falsifier did not fire. +- Step 9: executor API pivot to VfsRef - FOCUSED verify: `cargo build`, `cargo nextest run -p promptforge-core execute::` - pass (314/314); coding run additionally passed 843 tests across shared-vfs, promptforge-vfs, promptforge-store, promptforge-lua, promptforge-agent, promptforge-core, 351 workshop-server tests, doctests including the run() VfsRef doc example, and workspace clippy/fmt. Review: 1 Important (missing mount-less-handle fallback test), 1 Minor (store-mount probe swallowed backend errors), both closed; probe now matches NotFound specifically and other errors fail the run via a new `Error::Store` variant mapped to the previously unreachable `RunErrorKind::Store`. + - Decision: access lives in the Chain (arena is append-only), taken at finish/abort, so a finished arm's claims never block the join's merge | Falsifier: a parent merge conflicting with a dead arm's claims. + - Decision: call chains Arc-borrow the parent's access; arms spawn from the fanout caller | Falsifier: a false conflict between a caller's claims and its blocking child. + - Decision: `run_agent` takes `&VfsRef` - the census missed promptforge-agent; StoreRef's deletion forced it | Falsifier: the four-crates API criterion read strictly. + - Decision: the fanout-store-writes fixture's ready-*.md rendezvous was removed - polling a live sibling's writes is exactly the cross-arm read-while-written pattern claims reject; interleaving coverage stays in fanout_arms_interleave_at_io_points_on_one_thread | Falsifier: a requirement that this fixture prove arm concurrency. + - Decision: run() probes stat(STORE_MOUNT) and overlays a fresh memory store only when absent | Falsifier: a router whose mount exists but stats error getting a shadowing overlay. From 6d2e089c110c800a9dfb6b3cecfd91f70a1a06eb Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 20:56:43 -0700 Subject: [PATCH 10/26] Make store ops leaf yields with fatal determinism conflicts Every store operation a script makes now suspends its block as a leaf yield that the driver answers on the blocking pool against the sync store, uniformly for memory and host backends with no inline fast path, so a run's interleaving behavior can no longer depend on which backend serves the mount. A claims conflict between two live execution identities is now fatal to the whole run: the driver intercepts it at the answer boundary and ends the run on the spot rather than resuming it into the script, where author error handling could catch it. Agent VMs keep their direct store closures, since their driver is a single-identity loop with no interleaving for the claims model to govern. - `install_store_shims` replaces a VM's direct store closures with the yield shims, and section setup installs it only after the shared replay: the shared chunk runs as a main chunk that cannot yield, so its load-time store calls must still reach the direct closures. - `run_store_op` is the single operation implementation behind both the legacy direct closures and the yield dispatch, so the two paths cannot drift. - `dispatch_store` runs the operation with spawn_blocking because the store is sync, and fires the operation's observation before posting the answer so the event stream keeps the legacy closure path's ordering. - `classify_store_failure` maps a claims conflict to the fatal `Error::Determinism`; every other store failure rides back as the call's answer classified `Lua`, exactly as the legacy closures surfaced it. - `Answer::Store` interception in the driver loop returns the determinism error instead of resuming a chain; parked fanout arms drop unarmed and report cancelled, exactly as on the cancellation path. - `StoreError::WriteRace` gains a `detail` field carrying the claims model's full conflict diagnosis, exposed through `conflict_detail` and carried verbatim into the fatal violation. - `parse_store` validates the operation name and author arguments once at the protocol boundary; a wrong shape resumes as the call's error, catchable by an author `pcall` like the legacy callback's conversion failures. - `two_arms_appending_one_path_boom_without_any_other_suspension` inverts the old premise: the store operation alone is now the interleaving point, so cross-arm same-path appends boom with no other suspension, and exactly one arm's append lands. - `store_observations` maps each operation to its succeeded/failed observation pair and reports nothing for `exists`, matching the legacy closures event for event. Design: extends dispatch-on-tag @ crates/promptforge-lua/src/protocol.rs::Request::from_yield Design: new temporal-coupling @ crates/promptforge-lua/src/coro.rs::install_store_shims deps: Lua boundary: pub Design: new pure-function @ crates/promptforge-lua/src/protocol.rs::call_optional_line deps: mlua::Table,str Design: new pure-function @ crates/promptforge-lua/src/protocol.rs::parse_store deps: mlua::Table Design: new pure-function @ crates/promptforge-core/src/execute/scheduler.rs::store_observations deps: StoreOp Design: new pure-function @ crates/promptforge-core/src/execute/scheduler.rs::classify_store_failure deps: StoreError Design: extends surface-growth @ crates/promptforge-core/src/execute/error.rs::RunErrorKind boundary: pub Design: extends surface-growth @ crates/promptforge-store/src/error.rs::StoreError boundary: pub Repairs: claims conflicts are fatal and uncatchable @ crates/promptforge-core/src/execute/scheduler.rs - a claims conflict resumed into Lua as a catchable answer an author pcall could swallow Plan: vibe/2026-09-11-3-vfs-foundation.md --- crates/promptforge-agent/src/agent.rs | 3 + crates/promptforge-core/src/error.rs | 9 + crates/promptforge-core/src/execute.rs | 2 + crates/promptforge-core/src/execute/error.rs | 4 + .../promptforge-core/src/execute/protocol.rs | 2 +- .../promptforge-core/src/execute/scheduler.rs | 120 ++++++++++- .../src/execute/section_context.rs | 6 +- .../src/execute/section_vm.rs | 7 + .../src/execute/tests/scheduler.rs | 121 +++++++---- crates/promptforge-core/src/lua.rs | 4 +- .../execution/fanout-cross-arm-append.md | 25 +++ .../prompts/execution/fanout-store-writes.md | 4 + crates/promptforge-core/tests/suite/fanout.rs | 57 +++++ crates/promptforge-core/tests/suite/vfs.rs | 89 +++++++- crates/promptforge-lua/src/__impl_coro.lua | 57 +++++ crates/promptforge-lua/src/coro.rs | 42 ++++ crates/promptforge-lua/src/host.rs | 40 ++++ crates/promptforge-lua/src/lib.rs | 9 +- crates/promptforge-lua/src/protocol.rs | 198 +++++++++++++++++- crates/promptforge-store/src/error.rs | 18 +- crates/promptforge-store/src/lib.rs | 3 +- vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 4 + 23 files changed, 764 insertions(+), 62 deletions(-) create mode 100644 crates/promptforge-core/tests/prompts/execution/fanout-cross-arm-append.md diff --git a/crates/promptforge-agent/src/agent.rs b/crates/promptforge-agent/src/agent.rs index 8f8c9fa3b..aa5ffda3c 100644 --- a/crates/promptforge-agent/src/agent.rs +++ b/crates/promptforge-agent/src/agent.rs @@ -533,6 +533,9 @@ async fn dispatch(run: &AgentRun<'_>, request: Request) -> Result Err(AgentError::Internal( "an agent VM cannot yield a fanout request: the shim is never installed", )), + Request::Store { .. } => Err(AgentError::Internal( + "an agent VM cannot yield a store request: the store yield shims are never installed", + )), Request::Mcp { .. } => Err(AgentError::Internal( "an agent VM cannot yield an mcp request: no shim produces one", )), diff --git a/crates/promptforge-core/src/error.rs b/crates/promptforge-core/src/error.rs index aaa58e072..7bba44848 100644 --- a/crates/promptforge-core/src/error.rs +++ b/crates/promptforge-core/src/error.rs @@ -536,6 +536,15 @@ pub(crate) enum Error { #[error("store operation failed: {0}")] Store(#[source] shared_vfs::VfsError), + /// Two live execution identities claimed one store path: the claims + /// model's conflict, mapped from the store's write-race vocabulary at + /// the yield-answer boundary. Fatal to the run on the spot and never + /// resumed into Lua, so no author `pcall` can catch it; the message is + /// the claims model's whole diagnosis, naming the canonical path, both + /// identities, and both claim kinds. + #[error("store determinism violation: {0}")] + Determinism(String), + /// Rendering the current time as an RFC 3339 string failed. /// /// Retains the [`time::error::Format`] failure as the private `#[source]` diff --git a/crates/promptforge-core/src/execute.rs b/crates/promptforge-core/src/execute.rs index d15670e3c..4a33a7c55 100644 --- a/crates/promptforge-core/src/execute.rs +++ b/crates/promptforge-core/src/execute.rs @@ -164,6 +164,8 @@ pub(crate) use crate::model::ModelSet; /// request. /// - [`RunErrorKind::Substitution`] - a `{{ }}` prose substitution failed. /// - [`RunErrorKind::Store`] - a run-scoped store operation failed. +/// - [`RunErrorKind::Determinism`] - two live execution identities claimed +/// one store path; the run terminated on the spot, uncatchably from Lua. /// - [`RunErrorKind::Cancelled`] - the host cancelled the run. /// - [`RunErrorKind::Internal`] - an internal invariant failed. /// diff --git a/crates/promptforge-core/src/execute/error.rs b/crates/promptforge-core/src/execute/error.rs index 3c9fcb858..0b63dfff5 100644 --- a/crates/promptforge-core/src/execute/error.rs +++ b/crates/promptforge-core/src/execute/error.rs @@ -24,6 +24,9 @@ pub enum RunErrorKind { Tool, /// A run-scoped store operation failed. Store, + /// Two live execution identities claimed one store path: the claims + /// model terminated the run to keep interleaving deterministic. + Determinism, /// A section's Lua phase failed to run or return a usable value. Lua, /// A Lua host resource quota (log events, log bytes, or instructions) was @@ -86,6 +89,7 @@ impl RunError { | Error::Tool { .. } => RunErrorKind::Tool, Error::Internal(_) | Error::TimestampFormat(_) => RunErrorKind::Internal, Error::Store(_) => RunErrorKind::Store, + Error::Determinism(_) => RunErrorKind::Determinism, Error::Bind { .. } | Error::BindSchema { .. } | Error::BindQuery { .. } diff --git a/crates/promptforge-core/src/execute/protocol.rs b/crates/promptforge-core/src/execute/protocol.rs index 7d61fc9a7..3168520c1 100644 --- a/crates/promptforge-core/src/execute/protocol.rs +++ b/crates/promptforge-core/src/execute/protocol.rs @@ -10,4 +10,4 @@ //! produced by the Lua side) and is re-exported here unchanged, so existing //! `crate::execute::protocol::*` paths keep working. -pub(crate) use promptforge_lua::{Answer, Request, ToolCallOutcome, YieldParse}; +pub(crate) use promptforge_lua::{Answer, Request, StoreOp, ToolCallOutcome, YieldParse}; diff --git a/crates/promptforge-core/src/execute/scheduler.rs b/crates/promptforge-core/src/execute/scheduler.rs index 2ea25a712..d3d7a46ab 100644 --- a/crates/promptforge-core/src/execute/scheduler.rs +++ b/crates/promptforge-core/src/execute/scheduler.rs @@ -47,8 +47,12 @@ //! arm's finalizer reports `FANOUT_ARM_CANCELLED`, so exactly one terminal //! observation fires per arm), [`Error::ToolLoopExhausted`] soft-degrades //! its arm to the incomplete stub, and two live arms of one fanout touching -//! the same store path with at least one write fail the second with the -//! claims model's write-write race error. A received `mcp` request +//! the same store path with at least one write terminate the whole run with +//! the claims model's fatal determinism violation, intercepted at the +//! answer boundary so no author `pcall` can catch it. Every store operation +//! is such a leaf yield, answered on the blocking pool uniformly for all +//! backends - no inline fast path - so interleaving behavior never depends +//! on which backend serves the mount. A received `mcp` request //! is the protocol's typed reserved error. use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; @@ -67,13 +71,13 @@ use crate::lua::{ CoroStep, LuaBlockResult, LuaFanoutResult, LuaProgram, MessageRecord, OverflowReason, ScriptReport, SectionVm, ToolOutputKind, UserInputOutcome, append_message_record, current_tool_bindings, dispatch_tool, invoke_selected, project_messages, resolve_model_binding, - shim_live_h1_models, + run_store_op, shim_live_h1_models, }; use crate::model::ModelBinding; -use crate::observe::{Observer, detail}; +use crate::observe::{Observation, Observer, detail}; use crate::parser::{Block, Section}; use crate::resolve::RuntimeResolution; -use crate::store::Access; +use crate::store::{Access, Store, StoreError}; use crate::tools::{Tool, ToolId}; use crate::{Error, Result, cancel, subst}; @@ -82,7 +86,7 @@ use super::engine::{ JumpTarget, home_without, resolve_jump_target, section_position, visible_sections, }; use super::gateway::{GatewaySource, ResolutionContext}; -use super::protocol::{Answer, Request, ToolCallOutcome, YieldParse}; +use super::protocol::{Answer, Request, StoreOp, ToolCallOutcome, YieldParse}; use super::scope::prepare_effective_scope; use super::section_context::SectionContext; use super::support::{GENERIC_COMPLETION, MAX_CALL_DEPTH, next_id, now_rfc3339_checked}; @@ -422,6 +426,44 @@ impl Chain<'_> { } } +/// The succeeded/failed observation pair one store operation reports, +/// matching the legacy direct closures event for event; `exists` reported +/// nothing there and reports nothing here. +fn store_observations(op: &StoreOp) -> Option<(Observation, Observation)> { + let pair = match op { + StoreOp::Write { .. } => (detail::STORE_WRITE_SUCCEEDED, detail::STORE_WRITE_FAILED), + StoreOp::Append { .. } => (detail::STORE_APPEND_SUCCEEDED, detail::STORE_APPEND_FAILED), + StoreOp::Read { .. } => (detail::STORE_READ_SUCCEEDED, detail::STORE_READ_FAILED), + StoreOp::ReadNumbered { .. } => ( + detail::STORE_READ_NUMBERED_SUCCEEDED, + detail::STORE_READ_NUMBERED_FAILED, + ), + StoreOp::StrReplace { .. } => ( + detail::STORE_REPLACE_SUCCEEDED, + detail::STORE_REPLACE_FAILED, + ), + StoreOp::Delete { .. } => (detail::STORE_DELETE_SUCCEEDED, detail::STORE_DELETE_FAILED), + StoreOp::Glob { .. } => (detail::STORE_GLOB_SUCCEEDED, detail::STORE_GLOB_FAILED), + StoreOp::Exists { .. } => return None, + }; + Some(pair) +} + +/// Classifies one store operation's failure for the answer channel. A +/// claims-model conflict becomes the fatal determinism violation: the +/// driver intercepts it at the answer boundary and ends the run on the +/// spot rather than resuming it into Lua, so no author `pcall` can catch +/// it. Every other failure rides back as the call's answer carrying the +/// store's own message, exactly as the legacy closure's external error +/// surfaced at the call site (and classified `Lua` if it aborts the chunk +/// uncaught, exactly as then). +fn classify_store_failure(error: &StoreError) -> Error { + if let Some(detail) = error.conflict_detail() { + return Error::Determinism(detail.to_owned()); + } + Error::Lua(error.to_string()) +} + /// The coroutine protocol's driver: the chain arena, ready queue, pending /// table, join table, and answer channel, owned outright by the driver /// loop's stack frame. @@ -635,13 +677,24 @@ impl<'a> Scheduler<'a> { "an answer arrived for a request with no pending entry and no recorded abort", )); }; - self.chains[chain_id.index()].incoming = Some(answer); - self.ready.push_back(chain_id); + match answer { + // A claims-model conflict is fatal: the run ends on + // the spot with the determinism violation rather + // than resuming it into Lua, where an author + // `pcall` could catch it. The suspended chains drop + // unarmed with the scheduler, each fanout arm's + // finalizer reporting its cancelled terminal + // observation, exactly as on the cancellation path. + Answer::Store(Err(error @ Error::Determinism(_))) => return Err(error), + answer => { + self.chains[chain_id.index()].incoming = Some(answer); + self.ready.push_back(chain_id); + } + } } } } } - /// Creates one chain over `slice` from `index` and returns its id. The /// chain enters its first section on its first step. The chain's /// `var` slot seeds from `var` (a call chain's or arm's caller @@ -1434,6 +1487,7 @@ impl<'a> Scheduler<'a> { self.dispatch_user_input(id); Ok(()) } + Request::Store { op } => self.dispatch_store(id, op), // Unreachable: no section VM installs the models.chat shim, and // stripped coroutines make a hand-rolled yield fail validation // before dispatch - the mirror of the agent driver's guards for @@ -1687,6 +1741,54 @@ impl<'a> Scheduler<'a> { self.pending.insert(request_id, id); } + /// Dispatches a `store` request: the chain's access capability runs the + /// operation on the blocking pool and posts the answer to the channel, + /// parking the chain in the pending table exactly as a leaf I/O round + /// does. Every store operation takes this yield path uniformly - + /// memory- and host-backed alike, with no inline fast path - so + /// interleaving behavior never depends on which backend serves the + /// mount. The operation's observation fires before the answer posts, so + /// the event stream keeps the legacy closure path's ordering (the op's + /// outcome precedes the chunk's closing boundary). + /// + /// # Errors + /// Returns [`Error::Internal`] when the live chain's access capability + /// is gone, which only the chain-end paths take. + fn dispatch_store(&mut self, id: ChainId, op: StoreOp) -> Result<()> { + let chain = &self.chains[id.index()]; + let access = Arc::clone(chain.access()?); + let observer = Arc::clone(chain.ctx.observer()); + let execution = chain.ctx.execution().to_owned(); + let section = chain.section_name().to_owned(); + let observations = store_observations(&op); + let request_id = RequestId(self.next_request); + self.next_request += 1; + let tx = self.answer_tx.clone(); + // spawn_blocking, not a plain task: the Vfs is sync by design, and + // the blocking pool keeps a slow host-backend op from stalling the + // driver. Aborting the handle detaches rather than interrupts, so a + // cancelled run's in-flight op completes without delivering. + let task = tokio::task::spawn_blocking(move || { + let result = run_store_op(&Store::new(&access), op); + if let Some((succeeded, failed)) = observations { + observer.observe( + &execution, + §ion, + if result.is_ok() { succeeded } else { failed }, + ); + } + // A send fails only when the driver is gone (a cancelled run); + // the answer is then moot. + let _ = tx.send(( + request_id, + Answer::Store(result.map_err(|e| classify_store_failure(&e))), + )); + }); + self.io_tasks.insert(request_id, task.abort_handle()); + self.pending.insert(request_id, id); + Ok(()) + } + /// Dispatches a `loop` request: runs the Rust-backed model-tool loop on /// the driver thread, then resumes the chain with the nil answer. The /// loop holds the section VM through its append sink and local-tool diff --git a/crates/promptforge-core/src/execute/section_context.rs b/crates/promptforge-core/src/execute/section_context.rs index daa8b0951..b931eb821 100644 --- a/crates/promptforge-core/src/execute/section_context.rs +++ b/crates/promptforge-core/src/execute/section_context.rs @@ -23,7 +23,10 @@ use std::sync::Arc; use std::sync::atomic::AtomicU32; use crate::debug::DebugCapture; -use crate::lua::{ProseState, SectionVm, ToolBinding, ToolCallCounts, install_live_h1_shim_base}; +use crate::lua::{ + ProseState, SectionVm, ToolBinding, ToolCallCounts, install_live_h1_shim_base, + install_store_shims, +}; use crate::observe::{Observer, detail}; use crate::parser::Section; use crate::store::Access; @@ -215,6 +218,7 @@ impl SectionContext { // frame does not exist yet, so its `Drop` cannot own this path. if let Err(error) = setup_live_h1(&mut vm, ctx, access, &sys, title) .and_then(|()| install_live_h1_shim_base(vm.lua()).map_err(Error::from)) + .and_then(|()| install_store_shims(vm.lua()).map_err(Error::from)) { vm.teardown(ctx.observer().as_ref(), title); return Err(error); diff --git a/crates/promptforge-core/src/execute/section_vm.rs b/crates/promptforge-core/src/execute/section_vm.rs index 54180f612..90dfc390b 100644 --- a/crates/promptforge-core/src/execute/section_vm.rs +++ b/crates/promptforge-core/src/execute/section_vm.rs @@ -124,5 +124,12 @@ where setup.observer_arc.as_ref(), setup.section_name, )?; + // The store yield shims install after the shared replay: the shared + // chunk runs as a main chunk, not a coroutine, so load-time store + // calls must hit the direct closures (which capture the same + // Arc, leaving claims attribution unchanged). Installing + // earlier would make a top-level `store.write` yield from outside a + // coroutine. + crate::lua::install_store_shims(vm.lua())?; vm.install_captured_bindings().map_err(Error::from) } diff --git a/crates/promptforge-core/src/execute/tests/scheduler.rs b/crates/promptforge-core/src/execute/tests/scheduler.rs index f7dfa8acf..b6f243804 100644 --- a/crates/promptforge-core/src/execute/tests/scheduler.rs +++ b/crates/promptforge-core/src/execute/tests/scheduler.rs @@ -2016,7 +2016,10 @@ async fn fanout_arms_take_global_ids_per_fanout_index_and_structured_results() { // plus the structured-result shape of `fanout_returns_structured_results`: // each arm entry takes the next run-global id, `sys.index` is the // 1-based per-fanout position, and the packed sequence carries `.ok` - // and `.item` with `__tostring` driving `table.concat`. + // and `.item` with `__tostring` driving `table.concat`. The ids log is + // arm-scoped (the pattern the claims model teaches): every store op is + // a leaf yield now, so two arms appending one path would genuinely race + // and boom; the parent's post-join read merges the arm logs in order. let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ @@ -2030,7 +2033,7 @@ async fn fanout_arms_take_global_ids_per_fanout_index_and_structured_results() { ```\n\n\ ### Worker\n\n\ ```lua\n\ - store.append('ids.txt', sys.id .. ':' .. sys.index .. '\\n')\n\ + store.append('ids-' .. sys.index .. '.txt', sys.id .. ':' .. sys.index .. '\\n')\n\ return item\n\ ```\n"; let prompt = parse(md); @@ -2042,9 +2045,19 @@ async fn fanout_arms_take_global_ids_per_fanout_index_and_structured_results() { assert_eq!(out, "a,b"); assert_eq!( - store.read("ids.txt").expect("the ids log"), - "2:1\n3:2\nparent:1\n", - "the arms take the next run-global ids with their per-fanout index" + store.read("ids-1.txt").expect("arm 1's ids log"), + "2:1\n", + "arm 1 takes the next run-global id with its per-fanout index" + ); + assert_eq!( + store.read("ids-2.txt").expect("arm 2's ids log"), + "3:2\n", + "arm 2 takes the following run-global id with its per-fanout index" + ); + assert_eq!( + store.read("ids.txt").expect("the parent's ids log"), + "parent:1\n", + "the parent keeps the run's first id" ); } @@ -2400,13 +2413,13 @@ async fn fanout_depth_cap_reads_the_chain_field() { } #[tokio::test(flavor = "current_thread")] -async fn two_arms_writing_one_path_fail_with_a_write_race() { - // Mirror of the legacy `two_arms_writing_one_path_fail_with_a_write_race`, - // restructured for the claims model: the registry is gone, so the race - // needs both arms live at once - each arm writes, then suspends on an - // infer, so the second arm's write meets the first arm's standing write - // claim. The conflict is fatal to the second arm and fails the fanout; - // the first arm, still parked on its infer, is aborted as the sibling. +async fn two_arms_writing_one_path_terminate_the_run_with_a_determinism_violation() { + // Restructured for the leaf-yield store path: each arm's write is a + // yield answered from the blocking pool, so two live arms writing one + // path genuinely race and the loser's op booms. The violation is fatal + // to the whole run at the answer boundary - it never resumes into Lua, + // so no author pcall can catch it - and both parked arms drop unarmed, + // reporting cancelled rather than failed. let recorder = Arc::new(Recorder::default()); let gateway = ScriptedGateway::start(vec![resp_text("p1"), resp_text("p2")]).await; let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ @@ -2427,31 +2440,44 @@ async fn two_arms_writing_one_path_fail_with_a_write_race() { let error = Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))) .drive() .await - .expect_err("two live arms writing one path must fail the fanout"); + .expect_err("two live arms writing one path must terminate the run"); - let text = error.to_string(); - assert!(text.contains("write-write race"), "error was: {text}"); - assert!(text.contains("shared.txt"), "error was: {text}"); + match &error { + Error::Determinism(detail) => { + assert!(detail.contains("shared.txt"), "error was: {detail}"); + assert!( + detail.contains("conflicts with"), + "the conflict is named: {detail}" + ); + assert_eq!( + detail.matches("ExecId(").count(), + 2, + "both arms' identities are named: {detail}" + ); + } + other => panic!("expected the fatal determinism violation, got {other:?}"), + } assert_eq!( terminal_count(&recorder, &detail::FANOUT_ARM_FAILED), - 1, - "the second arm's write raced: {:?}", + 0, + "no arm fails on its own; the run ends at the answer boundary: {:?}", recorder.events() ); assert_eq!( terminal_count(&recorder, &detail::FANOUT_ARM_CANCELLED), - 1, - "the first arm, parked on its infer, is aborted as the sibling: {:?}", + 2, + "both parked arms drop unarmed and report cancelled: {:?}", recorder.events() ); } #[tokio::test(flavor = "current_thread")] -async fn two_live_arms_appending_one_path_fail_with_a_write_race() { +async fn two_live_arms_appending_one_path_terminate_with_a_determinism_violation() { // The papergate case the WriteScope registry never caught: `append` // claims write intent now, so two live arms appending to one path - // conflict exactly as two writes do. The arms interleave because each - // appends before suspending on its infer. + // conflict exactly as two writes do - and under the leaf-yield store + // path the conflict is the fatal determinism violation, not a + // per-arm store error. let gateway = ScriptedGateway::start(vec![resp_text("p1"), resp_text("p2")]).await; let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ @@ -2471,20 +2497,28 @@ async fn two_live_arms_appending_one_path_fail_with_a_write_race() { let error = Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))) .drive() .await - .expect_err("two live arms appending one path must fail the fanout"); + .expect_err("two live arms appending one path must terminate the run"); - let text = error.to_string(); - assert!(text.contains("write-write race"), "error was: {text}"); - assert!(text.contains("evidence.md"), "error was: {text}"); + match &error { + Error::Determinism(detail) => { + assert!(detail.contains("evidence.md"), "error was: {detail}"); + assert_eq!( + detail.matches("ExecId(").count(), + 2, + "both arms' identities are named: {detail}" + ); + } + other => panic!("expected the fatal determinism violation, got {other:?}"), + } } #[tokio::test(flavor = "current_thread")] -async fn two_arms_appending_one_path_succeed() { - // Mirror of the legacy case of the same name, with the claims-model - // rationale: arms that never suspend at I/O run one at a time, so each - // arm's claims release at its end and the next arm's append meets no - // live claimant. Only the relative order is unspecified - and with no - // interleaving points it is the collection order. +async fn two_arms_appending_one_path_boom_without_any_other_suspension() { + // The store operation alone is the interleaving point now: every store + // op is a leaf yield, so the arms park live on their appends and the + // second op to execute in the blocking pool meets the first arm's + // standing claim. The old premise - arms that never suspend at I/O run + // one at a time - is gone, and the cross-arm append booms. let store = TestStore::new(); let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ @@ -2500,15 +2534,24 @@ async fn two_arms_appending_one_path_succeed() { ```\n"; let prompt = parse(md); let ctx = scheduler_context_on(&prompt, &store, Arc::new(NullObserver::default())); - let out = Scheduler::new(&ctx, None) + let error = Scheduler::new(&ctx, None) .drive() .await - .expect("concurrent appends to one path must succeed"); + .expect_err("concurrent appends to one path must boom"); - assert_eq!(out, "alpha,beta"); - let log = store.read("log.txt").expect("both arms appended"); - assert!(log.contains("alpha;"), "log was: {log:?}"); - assert!(log.contains("beta;"), "log was: {log:?}"); + match &error { + Error::Determinism(detail) => { + assert!(detail.contains("log.txt"), "error was: {detail}"); + } + other => panic!("expected the fatal determinism violation, got {other:?}"), + } + // The losing arm's append never reached the backend: exactly one arm's + // append landed. + let log = store.read("log.txt").expect("one arm appended"); + assert!( + log == "alpha;" || log == "beta;", + "exactly one arm's append may land: {log:?}" + ); } #[tokio::test(flavor = "current_thread")] diff --git a/crates/promptforge-core/src/lua.rs b/crates/promptforge-core/src/lua.rs index 5f2b981f4..325d3a71c 100644 --- a/crates/promptforge-core/src/lua.rs +++ b/crates/promptforge-core/src/lua.rs @@ -17,8 +17,8 @@ pub(crate) use promptforge_lua::{ ToolCallCounts, ToolCallRecord, ToolResolver, ToolSet, ToolView, UserInputOutcome, append_message_record, current_tool_bindings, dispatch_tool, enrich_sys_model, install_live_h1_shim_base, install_section_loop_shim, install_section_user_input_shim, - install_ui, invoke_selected, is_context_overflow, precheck, project_messages, - resolve_model_binding, shim_live_h1_models, + install_store_shims, install_ui, invoke_selected, is_context_overflow, precheck, + project_messages, resolve_model_binding, run_store_op, shim_live_h1_models, }; // The typed compactor policy is read only by the tool loop's test-only diff --git a/crates/promptforge-core/tests/prompts/execution/fanout-cross-arm-append.md b/crates/promptforge-core/tests/prompts/execution/fanout-cross-arm-append.md new file mode 100644 index 000000000..153c4f67d --- /dev/null +++ b/crates/promptforge-core/tests/prompts/execution/fanout-cross-arm-append.md @@ -0,0 +1,25 @@ +--- +name: fanout_cross_arm_append +description: Two arms append to one path +promptforge: 0 +--- + +# Fanout Cross Arm Append + +## Research + +```lua +local replies = fanout("### Worker", {"alpha", "beta"}) +return table.concat(replies, ",") +``` + +### Worker + +```lua +-- The pcall proves the violation is not catchable from Lua: the +-- claims-model conflict never resumes into the arm, so this handler +-- never runs and the run terminates instead of returning "alpha,beta". +local ok, err = pcall(store.append, "evidence.md", item .. "\n") +store.write("caught-" .. sys.index .. ".txt", tostring(ok)) +return item +``` diff --git a/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md b/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md index 223738b70..bd118a9d2 100644 --- a/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md +++ b/crates/promptforge-core/tests/prompts/execution/fanout-store-writes.md @@ -11,6 +11,10 @@ promptforge: 0 ```lua local replies = fanout("### Worker", list_from_section("### Topics")) local files = store.glob("arm-*.md") +-- The ordered merge: the join delivers arm results in collection order, +-- never finish order, so the parent's merge is deterministic by +-- construction. +store.write("merged.md", table.concat(replies, ",")) return tostring(#files) .. ":" .. table.concat(replies, ",") ``` diff --git a/crates/promptforge-core/tests/suite/fanout.rs b/crates/promptforge-core/tests/suite/fanout.rs index 9de0b5aac..53442d8e9 100644 --- a/crates/promptforge-core/tests/suite/fanout.rs +++ b/crates/promptforge-core/tests/suite/fanout.rs @@ -11,11 +11,14 @@ const FANOUT_BASIC_EXECUTION: &str = "fixture-fanout-basic"; const FANOUT_EPILOG_EXECUTION: &str = "fixture-fanout-epilog"; const FANOUT_STORE_EXECUTION: &str = "fixture-fanout-store"; const FANOUT_FAILURE_EXECUTION: &str = "fixture-fanout-failure"; +const FANOUT_CROSS_ARM_EXECUTION: &str = "fixture-fanout-cross-arm-append"; const FANOUT_BASIC: &str = include_str!("../prompts/execution/fanout-basic.md"); const FANOUT_EPILOG: &str = include_str!("../prompts/execution/fanout-epilog.md"); const FANOUT_STORE_WRITES: &str = include_str!("../prompts/execution/fanout-store-writes.md"); const FANOUT_ARM_FAILURE: &str = include_str!("../prompts/execution/fanout-arm-failure.md"); +const FANOUT_CROSS_ARM_APPEND: &str = + include_str!("../prompts/execution/fanout-cross-arm-append.md"); /// The worker-template section name both fanout arms execute under. The /// observation stream keys arm events by this section, not by `sys.index` @@ -132,6 +135,60 @@ async fn fanout_store_writes_persist_across_arms() { run.store.read("arm-2.md").expect("arm 2 must write"), "beta" ); + // The ordered merge: the join's collection-order results land in one + // parent-written file, deterministic by construction. + assert_eq!( + run.store.read("merged.md").expect("the merge must land"), + "alpha,beta" + ); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_cross_arm_append_terminates_the_run_with_a_determinism_violation() { + // Every store operation is a leaf yield now, so two live arms appending + // one path genuinely race in the blocking pool; the claims model booms + // the loser and the violation fails the whole run at the answer + // boundary. The fixture's pcall proves the violation is uncatchable: + // were it resumed into the arm, the handler would record the catch and + // the run would return "alpha,beta" instead of failing. + let run = run_fixture( + FANOUT_CROSS_ARM_APPEND, + "execution/fanout-cross-arm-append.md", + FANOUT_CROSS_ARM_EXECUTION, + "", + None, + ) + .await; + let error = match run.result { + Ok(value) => panic!("a cross-arm append must terminate the run, got {value:?}"), + Err(error) => error, + }; + assert_eq!( + error.kind(), + RunErrorKind::Determinism, + "a claims conflict classifies as a determinism violation: {error:?}" + ); + let text = error.to_string(); + assert!( + text.contains("evidence.md"), + "the violation names the contested path: {text}" + ); + assert!( + text.contains("conflicts with"), + "the violation names the conflicting claim: {text}" + ); + assert_eq!( + text.matches("ExecId(").count(), + 2, + "the violation names both arms' identities: {text}" + ); + // The losing arm's append never reached the backend: exactly one arm's + // line landed. + let evidence = run.store.read("evidence.md").expect("one arm appended"); + assert!( + evidence == "alpha\n" || evidence == "beta\n", + "exactly one arm's append may land: {evidence:?}" + ); } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/crates/promptforge-core/tests/suite/vfs.rs b/crates/promptforge-core/tests/suite/vfs.rs index 7824dd9ed..5c2d3cee3 100644 --- a/crates/promptforge-core/tests/suite/vfs.rs +++ b/crates/promptforge-core/tests/suite/vfs.rs @@ -6,9 +6,9 @@ use promptforge_core::parser::Prompt; use promptforge_core::store::{Store, StoreError, StoreExt}; -use shared_vfs::VfsRef; +use shared_vfs::{HostBackend, VfsRef}; -use super::support::{RunOptions, parse_execution_fixture, run}; +use super::support::{RunOptions, parse_execution_fixture, run, run_fixture}; use crate::support::Recorder; use std::sync::Arc; @@ -200,3 +200,88 @@ async fn a_missing_declared_output_is_a_contract_error_naming_the_prompts_promis "the error names the promise's description: {error}" ); } + +/// A unique temporary directory that removes itself on drop. The suite has +/// no tempfile dependency; this mirrors shared-vfs's own test helper. +struct TempDir(std::path::PathBuf); + +impl TempDir { + fn new(name: &str) -> TempDir { + let unique = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("the clock is after the epoch") + .as_nanos(); + let dir = std::env::temp_dir().join(format!( + "promptforge-core-vfs-{}-{unique}-{name}", + std::process::id(), + )); + std::fs::create_dir_all(&dir).expect("the temp dir creates"); + TempDir(dir) + } +} + +impl Drop for TempDir { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.0); + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn fanout_interleaving_is_invariant_across_memory_and_host_backends() { + // The consistency rule: every store operation takes the leaf-yield path + // uniformly, with no inline fast path, so a run's observable behavior + // cannot depend on which backend serves the store mount. The same + // fanout fixture (arm-scoped writes, a post-join glob, the ordered + // merge) runs over the stock memory mount and over a host backend + // rooted in a temp dir; the result and the stored contents must be + // identical. + const FANOUT_STORE_WRITES: &str = include_str!("../prompts/execution/fanout-store-writes.md"); + let memory = run_fixture( + FANOUT_STORE_WRITES, + "execution/fanout-store-writes.md", + "vfs-invariance-memory", + "", + None, + ) + .await; + let memory_result = memory + .result + .expect("the memory-backed fanout must execute offline"); + + let temp = TempDir::new("host-backend"); + let host_vfs = VfsRef::builder() + .mount( + promptforge_vfs::STORE_MOUNT, + HostBackend::rooted(&temp.0).expect("the temp dir roots the host backend"), + ) + .build(); + let host = run_fixture( + FANOUT_STORE_WRITES, + "execution/fanout-store-writes.md", + "vfs-invariance-host", + "", + Some(host_vfs), + ) + .await; + let host_result = host + .result + .expect("the host-backed fanout must execute offline"); + + assert_eq!( + memory_result, host_result, + "the run's result must not depend on the backend" + ); + for path in ["arm-1.md", "arm-2.md", "merged.md"] { + assert_eq!( + memory.store.read(path).ok(), + host.store.read(path).ok(), + "stored contents at {path} must not depend on the backend" + ); + } + // The host backend really served the mount: the arm's write landed on + // the host filesystem under the root. + assert!( + temp.0.join("arm-1.md").is_file(), + "the host backend must persist the arm's write under its root" + ); +} diff --git a/crates/promptforge-lua/src/__impl_coro.lua b/crates/promptforge-lua/src/__impl_coro.lua index 8c01204bc..20f62177c 100644 --- a/crates/promptforge-lua/src/__impl_coro.lua +++ b/crates/promptforge-lua/src/__impl_coro.lua @@ -122,6 +122,53 @@ local function user_input(...) return text, available end +-- store.*: every store operation is a leaf yield, answered by the driver +-- against the sync VFS uniformly for all backends - no inline fast path, +-- so interleaving behavior never depends on which backend serves the +-- mount. The host installs these onto the store table of section VMs and +-- the live H1 VM only; an agent VM's store table keeps its direct +-- closures. `end` is a keyword, so the read bounds travel under bracket +-- keys. +local function store_request(store_op, fields) + fields.op = "store" + fields.store_op = store_op + local ok, result = yield(fields) + if not ok then error(result, 0) end + return result +end + +local function store_write(path, contents) + return store_request("write", { path = path, contents = contents }) +end + +local function store_append(path, contents) + return store_request("append", { path = path, contents = contents }) +end + +local function store_read(path, start, finish) + return store_request("read", { path = path, start = start, ["end"] = finish }) +end + +local function store_read_numbered(path, start, finish) + return store_request("read_numbered", { path = path, start = start, ["end"] = finish }) +end + +local function store_str_replace(path, old, new) + return store_request("str_replace", { path = path, old = old, new = new }) +end + +local function store_delete(path) + return store_request("delete", { path = path }) +end + +local function store_glob(pattern) + return store_request("glob", { pattern = pattern }) +end + +local function store_exists(path) + return store_request("exists", { path = path }) +end + -- The section install passes the section's namespace tables; the live H1 -- base install passes nil for both (H1's live models table exists only per -- block, given the shim by the host's per-step wrap) and takes `infer` from @@ -140,4 +187,14 @@ return { infer = infer, loop = models_loop, user_input = user_input, + store = { + write = store_write, + append = store_append, + read = store_read, + read_numbered = store_read_numbered, + str_replace = store_str_replace, + delete = store_delete, + glob = store_glob, + exists = store_exists, + }, } diff --git a/crates/promptforge-lua/src/coro.rs b/crates/promptforge-lua/src/coro.rs index 3303fa1d8..910e64815 100644 --- a/crates/promptforge-lua/src/coro.rs +++ b/crates/promptforge-lua/src/coro.rs @@ -47,6 +47,14 @@ const LOOP_REGISTRY: &str = "promptforge.impl_coro.loop"; /// stays nil because nothing ever reads this stash there. const USER_INPUT_REGISTRY: &str = "promptforge.impl_coro.user_input"; +/// The registry key for the shim's store function table, stashed by the +/// prelude and the live H1 base install so the executor can install the +/// store yield shims onto a VM's `store` table. The registry is host-side +/// only: an agent VM never installs them, so its store table keeps the +/// direct closures - the agent driver is a single-identity loop with no +/// interleaving for the claims model to govern. +const STORE_REGISTRY: &str = "promptforge.impl_coro.store"; + /// The shim program, compiled once and loaded per VM. Compilation of the /// bundled source fails only on a crate bug, so the payload is a shareable /// [`SharedSource`] cause (the crate `Error` is not `Clone`), re-wrapped as @@ -100,6 +108,9 @@ pub(crate) fn install_shim_prelude(lua: &Lua) -> Result<()> { let user_input: Function = shims.raw_get("user_input").map_err(Error::lua)?; lua.set_named_registry_value(USER_INPUT_REGISTRY, user_input) .map_err(Error::lua)?; + let store: Table = shims.raw_get("store").map_err(Error::lua)?; + lua.set_named_registry_value(STORE_REGISTRY, store) + .map_err(Error::lua)?; globals .raw_set("coroutine", Value::Nil) .map_err(Error::lua)?; @@ -196,12 +207,43 @@ pub fn install_live_h1_shim_base(lua: &Lua) -> Result<()> { let infer: Function = shims.raw_get("infer").map_err(Error::lua)?; lua.set_named_registry_value(INFER_REGISTRY, infer) .map_err(Error::lua)?; + let store: Table = shims.raw_get("store").map_err(Error::lua)?; + lua.set_named_registry_value(STORE_REGISTRY, store) + .map_err(Error::lua)?; globals .raw_set("coroutine", Value::Nil) .map_err(Error::lua)?; Ok(()) } +/// Installs the store yield shims onto a VM's `store` table, replacing the +/// direct closures the host API install put there. Every store operation +/// then suspends the block as a leaf yield the driver answers against the +/// sync VFS via the blocking pool - uniformly for all backends, with no +/// inline fast path, so interleaving behavior never depends on which +/// backend serves the mount. +/// +/// The executor's section setup and live H1 setup are the only callers: +/// an agent VM never receives the shims (its driver is a single-identity +/// loop with no interleaving for the claims model to govern), so its +/// store table keeps the direct closures. +/// +/// # Errors +/// Returns [`Error::Lua`] if the shim prelude (or the live H1 base +/// install) never ran on this VM, the `store` table is absent, or the +/// install fails. +pub fn install_store_shims(lua: &Lua) -> Result<()> { + let shims: Table = lua + .named_registry_value(STORE_REGISTRY) + .map_err(Error::lua)?; + let store: Table = lua.globals().raw_get("store").map_err(Error::lua)?; + for pair in shims.pairs::() { + let (name, function) = pair.map_err(Error::lua)?; + store.raw_set(name, function).map_err(Error::lua)?; + } + Ok(()) +} + /// Gives one live H1 block's freshly installed live models table the yield /// shim as its `models.infer`. /// diff --git a/crates/promptforge-lua/src/host.rs b/crates/promptforge-lua/src/host.rs index 93289ce0a..05cb14d9c 100644 --- a/crates/promptforge-lua/src/host.rs +++ b/crates/promptforge-lua/src/host.rs @@ -381,3 +381,43 @@ pub(crate) fn install_store_table( globals.raw_set("store", table).map_err(Error::lua)?; Ok(()) } + +/// Executes one validated store operation against the facade: the single +/// implementation behind both the legacy direct closures and the +/// executor's leaf-yield dispatch, so the two paths cannot drift. The +/// bounded-read argument rules (a negative bound converts to 0, an `end` +/// without a `start` is refused) live in the shared `read_store_bounded` +/// helper above; the read ops route through their named wrappers exactly +/// as the closures do. +/// +/// # Errors +/// Returns the [`StoreError`](promptforge_store::StoreError) the facade +/// produces for the operation: path validation, not-found, anchor, range, +/// write-race, or backend failure, exactly as the legacy closures +/// surfaced it. +pub fn run_store_op( + store: &Store, + op: crate::protocol::StoreOp, +) -> std::result::Result { + use crate::protocol::{StoreOp, StoreOutcome}; + match op { + StoreOp::Write { path, contents } => { + store.write(&path, &contents).map(|()| StoreOutcome::Unit) + } + StoreOp::Append { path, contents } => { + store.append(&path, &contents).map(|()| StoreOutcome::Unit) + } + StoreOp::Read { path, start, end } => { + read_store(store, &path, start, end).map(StoreOutcome::Text) + } + StoreOp::ReadNumbered { path, start, end } => { + read_store_numbered(store, &path, start, end).map(StoreOutcome::Text) + } + StoreOp::StrReplace { path, old, new } => store + .str_replace(&path, &old, &new) + .map(|()| StoreOutcome::Unit), + StoreOp::Delete { path } => store.delete(&path).map(|()| StoreOutcome::Unit), + StoreOp::Glob { pattern } => store.glob(&pattern).map(StoreOutcome::Paths), + StoreOp::Exists { path } => store.exists(&path).map(StoreOutcome::Bool), + } +} diff --git a/crates/promptforge-lua/src/lib.rs b/crates/promptforge-lua/src/lib.rs index b708e707f..44f244351 100644 --- a/crates/promptforge-lua/src/lib.rs +++ b/crates/promptforge-lua/src/lib.rs @@ -124,7 +124,7 @@ pub use compactors::{Compactor, OverflowReason, invoke_selected, is_context_over #[doc(hidden)] pub use coro::{ install_agent_chat_shim, install_live_h1_shim_base, install_section_loop_shim, - install_section_user_input_shim, shim_live_h1_models, + install_section_user_input_shim, install_store_shims, shim_live_h1_models, }; #[doc(hidden)] pub use dispatch::{ScriptReport, ToolDispatch, dispatch_tool}; @@ -134,6 +134,8 @@ pub use handles::{ ToolView, }; #[doc(hidden)] +pub use host::run_store_op; +#[doc(hidden)] pub use live::LiveBindingProducer; #[doc(hidden)] pub use models::ModelRuntime; @@ -143,8 +145,9 @@ pub use projection::project_messages; pub use prose::ProseState; #[doc(hidden)] pub use protocol::{ - Answer, ChatResult, ContentPart, MessageContent, MessageRecord, MessageRole, Request, - ToolCallOutcome, ToolCallRecord, UserInputOutcome, YieldParse, append_message_record, + Answer, ChatResult, ContentPart, MessageContent, MessageRecord, MessageRole, Request, StoreOp, + StoreOutcome, ToolCallOutcome, ToolCallRecord, UserInputOutcome, YieldParse, + append_message_record, }; #[doc(hidden)] pub use runtime_events::{EventsSnapshot, install_runtime_events}; diff --git a/crates/promptforge-lua/src/protocol.rs b/crates/promptforge-lua/src/protocol.rs index ce4e21e5b..d8c032170 100644 --- a/crates/promptforge-lua/src/protocol.rs +++ b/crates/promptforge-lua/src/protocol.rs @@ -2,8 +2,8 @@ //! yield/resume boundary between section Lua and the scheduler driver. //! //! A suspending host call (`models.infer(handle?, prompt)`, `call`, -//! `fanout`, `tools.call`, the section-only `models.loop` and -//! `user_input()`, the agent-only `models.chat`) is a Lua-side shim +//! `fanout`, `tools.call`, the section-only `models.loop`, `user_input()`, +//! and `store.*`, the agent-only `models.chat`) is a Lua-side shim //! that yields a request table; the driver validates the yield into a //! [`Request`], dispatches it, and resumes the coroutine with the //! `(ok, result)` envelope rendered from an [`Answer`]. The two enums are @@ -214,6 +214,17 @@ pub enum Request { /// exhaustive match forces. The request carries no arguments: the /// broker and its host policy own the whole interaction. UserInput, + /// `store.*(...)`: one run-scoped store operation as a leaf yield. + /// Section VMs and the live H1 VM run the store shims; the agent + /// driver carries an unreachable internal-invariant guard for the arm + /// its exhaustive match forces (an agent VM's store table keeps the + /// direct closures). Every operation takes this path uniformly - + /// memory- and host-backed alike, with no inline fast path - so + /// interleaving behavior never depends on the backend. + Store { + /// The validated operation and its author-supplied arguments. + op: StoreOp, + }, /// Reserved. Never dispatched: receiving one is a typed protocol error. Mcp { /// The reserved server name. @@ -263,6 +274,7 @@ impl Request { // No author arguments exist to fail validation: a well-formed // `user_input` yield is always the unit request. "user_input" => YieldParse::Request(Request::UserInput), + "store" => classify(parse_store(table), |error| Answer::Store(Err(error))), "mcp" => match parse_mcp(lua, table) { Ok(request) => YieldParse::Request(request), Err(_) => YieldParse::Malformed(direct_yield_error()), @@ -282,6 +294,90 @@ impl Request { } } +/// One validated store operation: the `store.*` call's name and its +/// author-supplied arguments, checked once here at the protocol boundary. +/// +/// The read bounds stay `i64` exactly as the legacy callback's signature +/// had them: a negative bound converts to 0 at execution, which the +/// facade's range validation rejects with the same error a zero bound +/// earns. +#[derive(Debug)] +pub enum StoreOp { + /// `store.write(path, contents)`. + Write { + /// The author-supplied logical path. + path: String, + /// The author-supplied file contents. + contents: String, + }, + /// `store.append(path, contents)`. + Append { + /// The author-supplied logical path. + path: String, + /// The author-supplied text to append. + contents: String, + }, + /// `store.read(path, start?, end?)`: no `start` reads the whole file; + /// a present `start` slices a 1-based inclusive line range. + Read { + /// The author-supplied logical path. + path: String, + /// The optional 1-based first line. + start: Option, + /// The optional 1-based last line. + end: Option, + }, + /// `store.read_numbered(path, start?, end?)`: the read with absolute + /// line numbers under the same optional bounds. + ReadNumbered { + /// The author-supplied logical path. + path: String, + /// The optional 1-based first line. + start: Option, + /// The optional 1-based last line. + end: Option, + }, + /// `store.str_replace(path, old, new)`. + StrReplace { + /// The author-supplied logical path. + path: String, + /// The anchor text, required to occur exactly once. + old: String, + /// The replacement text. + new: String, + }, + /// `store.delete(path)` (idempotent). + Delete { + /// The author-supplied logical path. + path: String, + }, + /// `store.glob(pattern)`. + Glob { + /// The author-supplied glob pattern. + pattern: String, + }, + /// `store.exists(path)`. + Exists { + /// The author-supplied logical path. + path: String, + }, +} + +/// The outcome of one dispatched store operation: the value the shim +/// returns to its caller. Mutating ops carry `Unit` (the shim returns +/// nil), exactly as the legacy closures returned nil. +#[derive(Debug)] +pub enum StoreOutcome { + /// The operation succeeded with no return value. + Unit, + /// `read`/`read_numbered`: the (possibly bounded) file text. + Text(String), + /// `glob`: the matching paths, sorted. + Paths(Vec), + /// `exists`: the presence flag. + Bool(bool), +} + /// Maps one per-op parse to the boundary outcome: a validated request, an /// author-argument failure as the call's answer, or a malformed yield. fn classify( @@ -392,6 +488,85 @@ fn parse_tool_call(lua: &Lua, table: &mlua::Table) -> std::result::Result std::result::Result, FieldFailure> { + match table.raw_get::(name) { + Ok(Value::Nil) => Ok(None), + Ok(Value::Integer(line)) => Ok(Some(line)), + // The bounds are exact powers of two (-2^63 and 2^63), so the + // range check needs no lossy i64-to-f64 cast. + Ok(Value::Number(line)) + if line.fract() == 0.0 + && (-9_223_372_036_854_775_808.0..9_223_372_036_854_775_808.0).contains(&line) => + { + #[expect( + clippy::cast_possible_truncation, + reason = "the range check above bounds the value to i64" + )] + Ok(Some(line as i64)) + } + Ok(other) => Err(FieldFailure::Call(Error::Lua(format!( + "{name} must be an integer, got {}", + other.type_name() + )))), + Err(_) => Err(FieldFailure::Malformed), + } +} + +/// Parses a `store` request: the operation name and its author-supplied +/// arguments. Every wrong shape is the call's error, resumed as the answer +/// so the shim raises it at the call site - an author `pcall` catches it, +/// exactly as the legacy callback's argument conversion failed there. +fn parse_store(table: &mlua::Table) -> std::result::Result { + let op = call_string(table, "store_op")?; + let op = match op.as_str() { + "write" => StoreOp::Write { + path: call_string(table, "path")?, + contents: call_string(table, "contents")?, + }, + "append" => StoreOp::Append { + path: call_string(table, "path")?, + contents: call_string(table, "contents")?, + }, + "read" => StoreOp::Read { + path: call_string(table, "path")?, + start: call_optional_line(table, "start")?, + end: call_optional_line(table, "end")?, + }, + "read_numbered" => StoreOp::ReadNumbered { + path: call_string(table, "path")?, + start: call_optional_line(table, "start")?, + end: call_optional_line(table, "end")?, + }, + "str_replace" => StoreOp::StrReplace { + path: call_string(table, "path")?, + old: call_string(table, "old")?, + new: call_string(table, "new")?, + }, + "delete" => StoreOp::Delete { + path: call_string(table, "path")?, + }, + "glob" => StoreOp::Glob { + pattern: call_string(table, "pattern")?, + }, + "exists" => StoreOp::Exists { + path: call_string(table, "path")?, + }, + other => { + return Err(FieldFailure::Call(Error::Lua(format!( + "unknown store operation {other:?}" + )))); + } + }; + Ok(Request::Store { op }) +} + /// Parses a reserved `mcp` request. No call surface produces one, so every /// field is shim-internal by construction. fn parse_mcp(lua: &Lua, table: &mlua::Table) -> std::result::Result { @@ -1145,6 +1320,8 @@ pub enum Answer { /// The outcome of a `user_input` request: the resumed text and its /// availability flag. UserInput(std::result::Result), + /// The outcome of a `store` request: the operation's return value. + Store(std::result::Result), } impl Answer { @@ -1158,6 +1335,7 @@ impl Answer { Answer::Chat(result) => Answer::Chat(result.map_err(map)), Answer::Loop(result) => Answer::Loop(result.map_err(map)), Answer::UserInput(result) => Answer::UserInput(result.map_err(map)), + Answer::Store(result) => Answer::Store(result.map_err(map)), } } } @@ -1234,12 +1412,28 @@ impl Answer { None, )) } + // The store op's return value: nil for the mutating ops, the + // text for reads, a sequence table for glob, a boolean for + // exists - the legacy closures' exact return shapes. + Answer::Store(Ok(outcome)) => { + let value = match outcome { + StoreOutcome::Unit => Value::Nil, + StoreOutcome::Text(text) => Value::String(lua.create_string(&text)?), + StoreOutcome::Paths(paths) => Value::Table(lua.create_sequence_from(paths)?), + StoreOutcome::Bool(exists) => Value::Boolean(exists), + }; + Ok(( + MultiValue::from_vec(vec![Value::Boolean(true), value]), + None, + )) + } Answer::Infer(Err(error)) | Answer::Call(Err(error)) | Answer::Fanout(Err(error)) | Answer::ToolCallResult(Err(error)) | Answer::Chat(Err(error)) | Answer::Loop(Err(error)) + | Answer::Store(Err(error)) | Answer::UserInput(Err(error)) => { let message = lua.create_string(error.to_string())?; Ok(( diff --git a/crates/promptforge-store/src/error.rs b/crates/promptforge-store/src/error.rs index 8c1965c09..6605003af 100644 --- a/crates/promptforge-store/src/error.rs +++ b/crates/promptforge-store/src/error.rs @@ -165,6 +165,10 @@ pub enum StoreError { WriteRace { /// The logical path both arms wrote. path: String, + /// The claims model's conflict diagnosis, naming the canonical + /// path, both identities, and both claim kinds; the executor's + /// fatal determinism violation carries it verbatim. + detail: String, }, /// The backend failed for a reason of its own, kept as an opaque source. @@ -244,11 +248,23 @@ impl StoreError { | StoreError::AnchorAmbiguous { path, .. } | StoreError::InvalidPath { path, .. } | StoreError::InvalidRange { path, .. } - | StoreError::WriteRace { path } => Some(path), + | StoreError::WriteRace { path, .. } => Some(path), StoreError::InvalidPattern { .. } | StoreError::Backend { .. } => None, } } + /// Returns the claims model's conflict diagnosis when this is a write + /// race: the canonical path, both identities, and both claim kinds. + /// The executor carries it verbatim into its fatal determinism + /// violation, whose message is the whole diagnosis. + #[must_use] + pub fn conflict_detail(&self) -> Option<&str> { + match self { + StoreError::WriteRace { detail, .. } => Some(detail), + _ => None, + } + } + /// Wraps a backend's own error as an opaque [`StoreError::Backend`] source. /// /// The facade uses this for VFS failures without a store-vocabulary diff --git a/crates/promptforge-store/src/lib.rs b/crates/promptforge-store/src/lib.rs index e3eea767c..032438e51 100644 --- a/crates/promptforge-store/src/lib.rs +++ b/crates/promptforge-store/src/lib.rs @@ -490,8 +490,9 @@ fn map_vfs(err: VfsError, path: &str) -> StoreError { VfsError::NotFound(_) => StoreError::NotFound { path: path.to_owned(), }, - VfsError::Conflict(_) => StoreError::WriteRace { + VfsError::Conflict(message) => StoreError::WriteRace { path: path.to_owned(), + detail: message, }, other => StoreError::backend(other), } diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index 1ee86518f..4298e89d0 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -655,7 +655,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 10: store operations as leaf yields +### Step 10: store operations as leaf yields [completed] - Component: promptforge-core - Add the new Request/Answer variants and one dispatch arm (the proven tools.call pattern) so every Lua store operation becomes a leaf yield answered via spawn_blocking against the sync Vfs: uniform for all backends, no inline fast path. Map the claims-violation VfsError to the fatal determinism RunErrorKind that terminates the run instantly and is not catchable from Lua. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index 21d5a1fb3..01372a9eb 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -122,3 +122,7 @@ - Decision: `run_agent` takes `&VfsRef` - the census missed promptforge-agent; StoreRef's deletion forced it | Falsifier: the four-crates API criterion read strictly. - Decision: the fanout-store-writes fixture's ready-*.md rendezvous was removed - polling a live sibling's writes is exactly the cross-arm read-while-written pattern claims reject; interleaving coverage stays in fanout_arms_interleave_at_io_points_on_one_thread | Falsifier: a requirement that this fixture prove arm concurrency. - Decision: run() probes stat(STORE_MOUNT) and overlays a fresh memory store only when absent | Falsifier: a router whose mount exists but stats error getting a shadowing overlay. +- Step 10: store operations as leaf yields - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p promptforge-core` - pass (429/429 core tests; coding run also green on promptforge-store 44, promptforge-lua 91 filtered, promptforge-agent 26). Review: 1 Critical (store yield shims installed before replay_shared broke load-time store calls in the shared library - main chunk cannot yield), closed by moving install_store_shims after replay_shared; 1 Minor (dead conflict_detail accessor), closed by routing classify_store_failure through it. Verification fix: clippy needless_pass_by_value on classify_store_failure. + - Decision: agent VMs keep direct store closures (single-identity driver, no interleaving) | Falsifier: the agent driver gains fanout or shares its VfsRef with a second live identity. + - Decision: non-conflict store failures resume as `Error::Lua` answers (legacy classification preserved) | Falsifier: a host needs `RunErrorKind::Store` for Lua-triggered store failures. + - Noted behavior change: cross-arm same-path writes now boom even with no other suspension point; the old two_arms_appending_one_path_succeed premise was inverted and the fanout_arms_take_global_ids fixture restructured to arm-scoped paths. From 656b536980eddee9fa35d0ec592f7c5905ca7c9f Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 21:10:00 -0700 Subject: [PATCH 11/26] Add Bashkit FsBackend adapter over the VFS handle Spike a storage adapter that lets the script engine run its filesystem over the shared virtual filesystem handle, evidencing that the trait subsumes the engine's storage contract. Reads, writes, and directory operations forward through the capability-based access layer, so every script operation is attributed to one execution identity and the claims model sees the whole session as a single thread of execution. Operations the virtual filesystem does not implement report the engine's own unsupported error, and honestly absent metadata becomes deterministic defaults rather than fabricated values. - `VfsBackend` holds one access capability for the engine session's lifetime: the constructor either captures a fresh identity from the handle or binds a capability the host already holds. - `crates/promptforge-bashkit/Cargo.toml` takes a spike-only path dependency on the local engine clone with default features off, and the crate stays unpublished. - `to_io` maps each virtual filesystem error onto the engine's io-error channel, folding claims conflicts and policy denials into PermissionDenied while preserving the source chain. - `metadata` emits the 0o644 or 0o755 mode defaults and the epoch for absent timestamps instead of inventing the current time. - `file_type` maps the first four kinds directly and reports the three specials as File with a warning trace. - `an_ls_cat_grep_script_runs_against_the_store_mount` and its memory-backed sibling run ls, cat, and grep end to end through the engine, asserting on exit code and stdout. - `symlink`, `read_link`, and `chmod` return the engine's unsupported error rather than faking success. Design: new constructor-injection @ crates/promptforge-bashkit/src/lib.rs::VfsBackend boundary: pub Design: new pure-function @ crates/promptforge-bashkit/src/lib.rs::vfs_path deps: Path Design: new pure-function @ crates/promptforge-bashkit/src/lib.rs::to_io deps: VfsError Design: new pure-function @ crates/promptforge-bashkit/src/lib.rs::metadata deps: Stat Design: new pure-function @ crates/promptforge-bashkit/src/lib.rs::unsupported deps: str Plan: vibe/2026-09-11-3-vfs-foundation.md --- Cargo.lock | 186 ++++++++++++++- crates/promptforge-bashkit/Cargo.toml | 24 ++ crates/promptforge-bashkit/src/lib.rs | 317 ++++++++++++++++++++++++++ vibe/2026-09-11-3-vfs-foundation.md | 2 +- vibe/vibe-ledger.md | 3 + 5 files changed, 528 insertions(+), 4 deletions(-) create mode 100644 crates/promptforge-bashkit/Cargo.toml create mode 100644 crates/promptforge-bashkit/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 5e6bfa885..182838ae3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -346,7 +346,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.7", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -393,6 +393,56 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64" +version = "0.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" + +[[package]] +name = "bashkit" +version = "0.18.0" +dependencies = [ + "anyhow", + "async-trait", + "base64 0.23.1", + "bigdecimal", + "bitflags 2.13.1", + "bzip2", + "chrono", + "clap", + "fancy-regex 0.19.1", + "flate2", + "futures-util", + "getrandom 0.4.3", + "hmac", + "md-5", + "num-traits", + "os_display", + "regex", + "serde", + "serde_json", + "sha1 0.11.0", + "sha2 0.11.0", + "thiserror 2.0.19", + "tokio", + "unit-prefix", + "url", +] + +[[package]] +name = "bigdecimal" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" +dependencies = [ + "autocfg", + "libm", + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "bit-set" version = "0.8.0" @@ -620,6 +670,15 @@ dependencies = [ "serde", ] +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "libbz2-rs-sys", +] + [[package]] name = "cairo-rs" version = "0.18.5" @@ -705,7 +764,7 @@ dependencies = [ "byteorder", "candle-core", "candle-nn", - "fancy-regex", + "fancy-regex 0.18.0", "num-traits", "rand 0.9.5", "rayon", @@ -879,6 +938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", + "clap_derive", ] [[package]] @@ -891,6 +951,18 @@ dependencies = [ "clap_lex", ] +[[package]] +name = "clap_derive" +version = "4.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 3.0.3", +] + [[package]] name = "clap_lex" version = "1.1.0" @@ -906,6 +978,12 @@ dependencies = [ "cc", ] +[[package]] +name = "cmov" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" + [[package]] name = "colored" version = "3.1.1" @@ -1260,6 +1338,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "ctutils" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +dependencies = [ + "cmov", +] + [[package]] name = "darling" version = "0.20.11" @@ -1477,6 +1564,7 @@ dependencies = [ "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", + "ctutils", ] [[package]] @@ -1778,6 +1866,17 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "fancy-regex" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e0387578e845beb7a1acff126228499f26cb18edf12919cc513bb863266464" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -2815,6 +2914,15 @@ dependencies = [ "xet-runtime", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "hound" version = "3.5.1" @@ -3494,6 +3602,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" version = "0.2.189" @@ -3707,6 +3821,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if 1.0.4", + "digest 0.11.3", +] + [[package]] name = "memchr" version = "2.8.3" @@ -3977,6 +4101,16 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-bigint" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -3993,6 +4127,15 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-integer" +version = "0.1.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -4390,6 +4533,15 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "os_display" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad5fd71b79026fb918650dde6d125000a233764f1c2f1659a1c71118e33ea08f" +dependencies = [ + "unicode-width", +] + [[package]] name = "os_str_bytes" version = "6.6.1" @@ -4824,6 +4976,17 @@ dependencies = [ "tokio", ] +[[package]] +name = "promptforge-bashkit" +version = "0.3.0" +dependencies = [ + "bashkit", + "promptforge-vfs", + "shared-vfs", + "tokio", + "tracing", +] + [[package]] name = "promptforge-core" version = "0.3.0" @@ -6036,6 +6199,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if 1.0.4", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -7473,7 +7647,7 @@ dependencies = [ "httparse", "log", "rand 0.9.5", - "sha1", + "sha1 0.10.7", "thiserror 2.0.19", ] @@ -7598,6 +7772,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unit-prefix" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" + [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/crates/promptforge-bashkit/Cargo.toml b/crates/promptforge-bashkit/Cargo.toml new file mode 100644 index 000000000..8e04d17c3 --- /dev/null +++ b/crates/promptforge-bashkit/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "promptforge-bashkit" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +publish = false + +description = "Spike: the Bashkit engine's FsBackend implemented over the shared VFS handle, evidencing that the VFS trait subsumes Bashkit" + +[dependencies] +shared-vfs.workspace = true +tracing.workspace = true +# Spike-only path dependency on the local Bashkit clone (a sibling of this +# repository). Default features stay off: the spike needs the embeddable +# interpreter only, not the LLM tool wrapper or the timezone database. +bashkit = { path = "../../../bashkit/crates/bashkit", default-features = false } + +[dev-dependencies] +promptforge-vfs.workspace = true +tokio.workspace = true + +[lints] +workspace = true diff --git a/crates/promptforge-bashkit/src/lib.rs b/crates/promptforge-bashkit/src/lib.rs new file mode 100644 index 000000000..38bbd636c --- /dev/null +++ b/crates/promptforge-bashkit/src/lib.rs @@ -0,0 +1,317 @@ +//! Spike: the Bashkit engine's `FsBackend` trait implemented over the +//! shared VFS handle. +//! +//! The deliverable is evidence that `Vfs` subsumes Bashkit's storage +//! contract: whole-file reads serve from `read`, `symlink`/`chmod` return +//! the engine's unsupported error, the first four file types map directly +//! and the three specials map to `File` with a trace, and an absent `Stat` +//! mode emits the 0o644/0o755 defaults. The adapter captures the current +//! [`ExecId`] at exec start: one identity per engine session, so the +//! claims model attributes every script operation to that session. + +use std::io::{Error as IoError, ErrorKind}; +use std::path::{Path, PathBuf}; +use std::time::SystemTime; + +use bashkit::{DirEntry, Error, FileType as BashFileType, FsBackend, Metadata, Result}; +use shared_vfs::{Access, ExecId, FileType as VfsFileType, Stat, VfsError, VfsRef}; + +/// A Bashkit storage backend serving from a VFS handle. +/// +/// Constructing one acquires an [`Access`] capability: the adapter holds +/// one [`ExecId`] for the engine session's lifetime, so every script +/// operation is attributed to that identity and the claims model sees +/// the session as one thread of execution. The engine's `PosixFs` +/// wrapper enforces POSIX semantics above this raw storage layer. +#[derive(Debug)] +pub struct VfsBackend { + access: Access, +} + +impl VfsBackend { + /// Captures a fresh identity from `vfs`: call at exec start. + #[must_use] + pub fn new(vfs: &VfsRef) -> VfsBackend { + VfsBackend { + access: vfs.acquire(), + } + } + + /// Binds the backend to an existing capability: a host that already + /// holds the run's [`Access`] keeps the script under that identity. + #[must_use] + pub fn from_access(access: Access) -> VfsBackend { + VfsBackend { access } + } + + /// The identity this backend's operations are attributed to. + #[must_use] + pub fn id(&self) -> ExecId { + self.access.id() + } +} + +/// The engine hands `Path` values; the virtual namespace is POSIX-shaped +/// text. Lossy conversion with separator normalization is sufficient: +/// the engine never produces host-native paths here. +fn vfs_path(path: &Path) -> String { + path.to_string_lossy().replace('\\', "/") +} + +/// Maps the VFS error onto the engine's io-error channel, preserving the +/// kind so builtins report the right failure (`PermissionDenied` for a +/// claims conflict or policy denial, `Unsupported` for unimplemented +/// operations, and so on). +fn to_io(error: VfsError) -> Error { + let kind = match &error { + VfsError::NotFound(_) => ErrorKind::NotFound, + VfsError::PermissionDenied(_) | VfsError::Conflict(_) => ErrorKind::PermissionDenied, + VfsError::AlreadyExists(_) => ErrorKind::AlreadyExists, + VfsError::InvalidPath(_) => ErrorKind::InvalidInput, + VfsError::NotADirectory(_) => ErrorKind::NotADirectory, + VfsError::IsADirectory(_) => ErrorKind::IsADirectory, + VfsError::DirectoryNotEmpty(_) => ErrorKind::DirectoryNotEmpty, + VfsError::Unsupported(_) => ErrorKind::Unsupported, + _ => ErrorKind::Other, + }; + IoError::new(kind, error).into() +} + +/// The first four kinds map directly; the three specials map to `File` +/// with a trace (unreachable in practice: neither our v1 backends nor +/// the engine's ever produce them). +fn file_type(kind: VfsFileType) -> BashFileType { + match kind { + VfsFileType::File => BashFileType::File, + VfsFileType::Directory => BashFileType::Directory, + VfsFileType::Symlink => BashFileType::Symlink, + VfsFileType::Fifo => BashFileType::Fifo, + special => { + tracing::warn!( + ?special, + "VFS special file type reported to the engine as File" + ); + BashFileType::File + } + } +} + +/// The VFS says `None` rather than fabricating; the engine's `Metadata` +/// has no options, so an absent mode emits the 0o644/0o755 defaults and +/// absent timestamps become the epoch - deterministic, never an invented +/// `now()`. +fn metadata(stat: &Stat) -> Metadata { + let mode = stat.mode.unwrap_or(match stat.file_type { + VfsFileType::Directory => 0o755, + _ => 0o644, + }); + Metadata { + file_type: file_type(stat.file_type), + size: stat.size, + mode, + modified: stat.modified.unwrap_or(SystemTime::UNIX_EPOCH), + created: stat.created.unwrap_or(SystemTime::UNIX_EPOCH), + } +} + +/// The engine's unsupported error, matching its own convention of an +/// io error with `ErrorKind::Unsupported`. +fn unsupported(op: &str) -> Error { + IoError::new( + ErrorKind::Unsupported, + format!("{op} is not supported by the VFS adapter"), + ) + .into() +} + +#[bashkit::async_trait] +impl FsBackend for VfsBackend { + async fn read(&self, path: &Path) -> Result> { + self.access.read(&vfs_path(path)).map_err(to_io) + } + + async fn write(&self, path: &Path, content: &[u8]) -> Result<()> { + self.access.write(&vfs_path(path), content).map_err(to_io) + } + + async fn append(&self, path: &Path, content: &[u8]) -> Result<()> { + self.access.append(&vfs_path(path), content).map_err(to_io) + } + + async fn mkdir(&self, path: &Path, recursive: bool) -> Result<()> { + self.access.mkdir(&vfs_path(path), recursive).map_err(to_io) + } + + async fn remove(&self, path: &Path, recursive: bool) -> Result<()> { + self.access + .remove(&vfs_path(path), recursive) + .map_err(to_io) + } + + async fn stat(&self, path: &Path) -> Result { + self.access + .stat(&vfs_path(path)) + .map(|stat| metadata(&stat)) + .map_err(to_io) + } + + async fn read_dir(&self, path: &Path) -> Result> { + let entries = self.access.list(&vfs_path(path)).map_err(to_io)?; + Ok(entries + .into_iter() + .map(|entry| DirEntry { + name: entry.name, + metadata: metadata(&entry.stat), + }) + .collect()) + } + + async fn exists(&self, path: &Path) -> Result { + self.access.exists(&vfs_path(path)).map_err(to_io) + } + + async fn rename(&self, from: &Path, to: &Path) -> Result<()> { + self.access + .rename(&vfs_path(from), &vfs_path(to)) + .map_err(to_io) + } + + async fn copy(&self, from: &Path, to: &Path) -> Result<()> { + self.access + .copy(&vfs_path(from), &vfs_path(to)) + .map_err(to_io) + } + + async fn symlink(&self, _target: &Path, _link: &Path) -> Result<()> { + Err(unsupported("symlink")) + } + + async fn read_link(&self, _path: &Path) -> Result { + Err(unsupported("read_link")) + } + + async fn chmod(&self, _path: &Path, _mode: u32) -> Result<()> { + Err(unsupported("chmod")) + } +} + +#[cfg(test)] +mod tests { + use std::io::ErrorKind; + use std::path::Path; + use std::sync::Arc; + + use bashkit::{Bash, Error, FsBackend, PosixFs}; + use shared_vfs::{FileType as VfsFileType, MemoryBackend, VfsRef}; + + use super::{VfsBackend, file_type}; + use bashkit::FileType as BashFileType; + + type TestResult = Result<(), Box>; + + /// An engine whose entire filesystem is the VFS handle, with POSIX + /// semantics enforced by the engine's own wrapper. + fn engine(vfs: &VfsRef) -> Bash { + let backend = VfsBackend::new(vfs); + let fs = Arc::new(PosixFs::new(backend)); + Bash::builder().fs(fs).build() + } + + #[tokio::test] + async fn an_ls_cat_grep_script_runs_against_a_mounted_memory_backend() -> TestResult { + let vfs = VfsRef::builder().mount("/", MemoryBackend::new()).build(); + let mut bash = engine(&vfs); + let result = bash + .exec( + "mkdir -p /tmp/docs && echo hello > /tmp/docs/a.txt \ + && ls /tmp/docs && cat /tmp/docs/a.txt \ + && grep hello /tmp/docs/a.txt", + ) + .await?; + assert_eq!(result.exit_code, 0, "stderr: {}", result.stderr); + let stdout = result.stdout.text_lossy().into_owned(); + assert!(stdout.contains("a.txt"), "ls lists the file: {stdout}"); + assert!( + stdout.contains("hello"), + "cat and grep serve reads: {stdout}" + ); + Ok(()) + } + + #[tokio::test] + async fn an_ls_cat_grep_script_runs_against_the_store_mount() -> TestResult { + let vfs = promptforge_vfs::empty(); + vfs.acquire() + .write("/_promptforge/store/paper.md", b"# Draft\nhello world\n")?; + let mut bash = engine(&vfs); + let result = bash + .exec( + "ls /_promptforge/store && cat /_promptforge/store/paper.md \ + && grep hello /_promptforge/store/paper.md", + ) + .await?; + assert_eq!(result.exit_code, 0, "stderr: {}", result.stderr); + let stdout = result.stdout.text_lossy().into_owned(); + assert!(stdout.contains("paper.md"), "ls lists the store: {stdout}"); + assert!( + stdout.contains("hello world"), + "cat and grep read the store: {stdout}" + ); + Ok(()) + } + + #[tokio::test] + async fn symlink_and_chmod_return_the_engine_unsupported_error() -> TestResult { + let vfs = VfsRef::new(MemoryBackend::new()); + let backend = VfsBackend::new(&vfs); + match backend.symlink(Path::new("/a"), Path::new("/b")).await { + Err(Error::Io(io)) => assert_eq!(io.kind(), ErrorKind::Unsupported), + other => panic!("expected an unsupported io error, got {other:?}"), + } + match backend.read_link(Path::new("/a")).await { + Err(Error::Io(io)) => assert_eq!(io.kind(), ErrorKind::Unsupported), + other => panic!("expected an unsupported io error, got {other:?}"), + } + match backend.chmod(Path::new("/a"), 0o600).await { + Err(Error::Io(io)) => assert_eq!(io.kind(), ErrorKind::Unsupported), + other => panic!("expected an unsupported io error, got {other:?}"), + } + Ok(()) + } + + #[tokio::test] + async fn an_absent_stat_mode_emits_the_posix_defaults() -> TestResult { + let vfs = VfsRef::new(MemoryBackend::new()); + let backend = VfsBackend::new(&vfs); + backend.write(Path::new("/f.txt"), b"x").await?; + backend.mkdir(Path::new("/d"), false).await?; + // The memory backend honestly reports mode None; the adapter + // emits the engine's expected defaults instead. + assert_eq!(backend.stat(Path::new("/f.txt")).await?.mode, 0o644); + assert_eq!(backend.stat(Path::new("/d")).await?.mode, 0o755); + Ok(()) + } + + #[test] + fn special_file_types_map_to_file_and_the_first_four_map_directly() { + assert_eq!(file_type(VfsFileType::File), BashFileType::File); + assert_eq!(file_type(VfsFileType::Directory), BashFileType::Directory); + assert_eq!(file_type(VfsFileType::Symlink), BashFileType::Symlink); + assert_eq!(file_type(VfsFileType::Fifo), BashFileType::Fifo); + assert_eq!(file_type(VfsFileType::Socket), BashFileType::File); + assert_eq!(file_type(VfsFileType::CharDevice), BashFileType::File); + assert_eq!(file_type(VfsFileType::BlockDevice), BashFileType::File); + } + + #[test] + fn each_adapter_captures_a_fresh_exec_identity_at_exec_start() { + let vfs = VfsRef::new(MemoryBackend::new()); + let first = VfsBackend::new(&vfs); + let second = VfsBackend::new(&vfs); + assert_ne!(first.id(), second.id()); + // A host that already holds a capability binds it explicitly. + let access = vfs.acquire(); + let bound = VfsBackend::from_access(access); + assert_ne!(bound.id(), first.id()); + } +} diff --git a/vibe/2026-09-11-3-vfs-foundation.md b/vibe/2026-09-11-3-vfs-foundation.md index 4298e89d0..b4e858544 100644 --- a/vibe/2026-09-11-3-vfs-foundation.md +++ b/vibe/2026-09-11-3-vfs-foundation.md @@ -665,7 +665,7 @@ Parity is the gate: the existing store suite must pass against the rewritten fac -### Step 11: Bashkit adapter spike +### Step 11: Bashkit adapter spike [completed] - Component: bashkit adapter spike - Implement `bashkit::FsBackend` over VfsRef as a path dependency against the local `bashkit/` clone: whole-file reads served from read, symlink/chmod return the engine's unsupported error, FileType maps the first four kinds directly and the three specials to File with a trace, Stat mode None emits the 0o644/0o755 defaults, and the adapter captures the current ExecId at exec start. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md index 01372a9eb..cf8b897c0 100644 --- a/vibe/vibe-ledger.md +++ b/vibe/vibe-ledger.md @@ -126,3 +126,6 @@ - Decision: agent VMs keep direct store closures (single-identity driver, no interleaving) | Falsifier: the agent driver gains fanout or shares its VfsRef with a second live identity. - Decision: non-conflict store failures resume as `Error::Lua` answers (legacy classification preserved) | Falsifier: a host needs `RunErrorKind::Store` for Lua-triggered store failures. - Noted behavior change: cross-arm same-path writes now boom even with no other suspension point; the old two_arms_appending_one_path_succeed premise was inverted and the fanout_arms_take_global_ids fixture restructured to arm-scoped paths. +- Step 11: Bashkit adapter spike - FULL verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings` plus workshop crates, `mdbook build guide`, `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server`, `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --all-features`, doctests - pass. Review: clean. Spike result: the Vfs trait subsumes Bashkit's FsBackend; ls/cat/grep scripts pass against memory and store mounts; no mapping failure. Verification fixes: fmt normalization; `to_io` consumes the VfsError into IoError::new preserving the source chain. + - Decision: absent `Stat` timestamps map to `SystemTime::UNIX_EPOCH`, not `now()` | Falsifier: a script's time-based behavior (e.g. `ls -t` ordering) needs real mtimes, requiring the memory backend to track them rather than the adapter fabricating them. + - Decision: claims conflicts and policy denials both map to `ErrorKind::PermissionDenied` | Falsifier: a builtin must distinguish "locked by another identity" from "denied by policy" to recover correctly, requiring a richer `to_io` mapping. From 168035244739cb3d0e23f16d8aa596eb679a68bc Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Fri, 11 Sep 2026 21:32:38 -0700 Subject: [PATCH 12/26] Close plan: vfs foundation Plan: vibe/2026-09-11-3-vfs-foundation.md --- vibe/ACTIVE | 1 - 1 file changed, 1 deletion(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index ab3148d63..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-09-11-3-vfs-foundation.md From 27354d9e91a6f3bbbb0aac6a7e243afca2dc788b Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 02:13:36 -0700 Subject: [PATCH 13/26] Add the test-namespace and VFS debt removal plan The plan removes test-only re-exports from non-test modules across five crates, retires the committed run ledgers, removes the bashkit spike crate, de-interns VfsPath, and makes handle acquisition fallible. Plan: vibe/2026-09-12-1-test-namespace-vfs-debt.md --- vibe/2026-09-12-1-test-namespace-vfs-debt.md | 195 +++++++++++++++++++ vibe/ACTIVE | 1 + 2 files changed, 196 insertions(+) create mode 100644 vibe/2026-09-12-1-test-namespace-vfs-debt.md create mode 100644 vibe/ACTIVE diff --git a/vibe/2026-09-12-1-test-namespace-vfs-debt.md b/vibe/2026-09-12-1-test-namespace-vfs-debt.md new file mode 100644 index 000000000..7c381ce8e --- /dev/null +++ b/vibe/2026-09-12-1-test-namespace-vfs-debt.md @@ -0,0 +1,195 @@ +# Test-Namespace Cleanup and VFS Debt Removal + + + +## Product Requirements + +Two workstreams land together in the promptforge repository, with two supporting file patches beside it. First, test-only imports and re-exports are removed from non-test modules across the workspace, and the vibe coding tool is patched so runs stop committing their audit ledger into the repository. Second, four debts introduced by the VFS foundation run (the twelve commits `upstream/master..16803524`) are removed: a workspace member whose out-of-repo path dependency breaks every fresh clone and CI run, a process-global path interner that leaks every distinct path for the process lifetime and is reachable from model-controlled Lua, a handle boundary that panics where its trait documents a recoverable error, and a self-policing manifest test that a standard TOML idiom bypasses. + +- Problem and users: test plumbing (`#[cfg(test)] use` and `pub use` lines) sits in non-test modules in ten files across five crates, where it exists only to feed descendant test globs; the vibe tool's Mark step stages its ledger into every step commit, so drained ledgers and review queues accumulate as committed repo files; the VFS foundation run shipped four debts found by a debt-collection pass over `upstream/master..HEAD`. Users are the promptforge maintainers and every CI runner. +- Goals: every non-test module carries production items only; test modules import what they need directly; the vibe ledger is scratch, never committed; the four debts are removed with the operator's chosen remedies; `vibe/archdoc.md` describes the real component graph. +- Non-goals: no content-level string dedup machinery; no change to the legitimate pattern of `#[cfg(test)]` methods, functions, and imports whose consumers live in the same module; no change to the deliberate facade re-exports from implementation crates (`promptforge-lua`, `promptforge-model-client`, `promptforge-core-support`, `promptforge-parser`); no remediation of the twelve rejected debt candidates; no commit sequencing (left to the executing tool). +- Success criteria: a workspace grep finds no `#[cfg(test)]` import or re-export in a non-test module whose only consumers are test modules; a vibe run's step commits contain no ledger file; `cargo metadata --locked` and `cargo build` succeed with no sibling checkout beside the repository; a loop canonicalizing distinct paths does not grow the process heap monotonically; a backend whose `acquire` fails produces a run error, not a panic; the shared-vfs manifest test fails when a `[dependencies.foo]` sub-table is injected. +- Constraints: the workspace deletion rule moves removed files to `cabinet/_trash/` and never hard-deletes; the promptforge-core crate policy (`promptforge/crates/promptforge-core/AGENTS.md`) permits verbatim compatibility re-exports and forbids new compatibility vocabulary, which the removals respect because none of the removed lines is a compatibility path; `vibe/archdoc-next.md` does not exist at the disposition ref and is not created. +- Open questions: None + +## Functional Specification + +The work is code motion and contract correction, not new behavior. The one externally observable behavior change is failure behavior at the VFS handle boundary: a backend session-open failure stops being a process panic and becomes a recoverable error that fails the run. The one security-relevant change is that model-controlled path volume can no longer grow host memory without bound. + +- Actors and workflows: the executing agent applies unordered work items; maintainers review per-item diffs; CI validates the whole. +- Inputs and outputs: inputs are the ten source files, two rulebook/tool files, three committed ledger files, and the VFS crates; outputs are the same trees with the corrections applied, plus one promptforge commit removing the three ledger files. +- States and validation: `VfsPath` changes shape (Clone, not Copy); every other change is import motion, file removal, or documentation. +- Errors and recovery: `VfsRef::acquire` and `Access::spawn` return `Result<_, VfsError>`; the executor maps acquisition failure to `RunErrorKind::Store`; removed files remain recoverable from `cabinet/_trash/` and git history. +- Security and privacy behavior: the interner leak was reachable from model-controlled Lua (`store.write` with fresh names); after the fix, path strings are freed when their last owner drops. +- Acceptance criteria: the success criteria above, plus a green focused test scope per work item and a green workspace clippy. + + + + +## Technical Design + +The only cross-module design is in `shared-vfs`: `VfsPath` becomes an `Arc`-backed value with no global table, and the handle's acquisition boundary becomes fallible to match the trait's documented contract. Everything else is local: import motion into test modules, one crate removal, one visibility correction, and documentation patches. + +- Architecture: `VfsPath` holds `Arc`; `canonicalize` allocates one `Arc` per call; claims tables hold clones; the string frees when its last owner drops. Sharing is per value lineage (clones share one allocation), not per string content (no dedup table). No process-global state, no lock, no eviction machinery; the u32-exhaustion panic disappears with the interner. +- Modules and interfaces: `promptforge/crates/shared-vfs/src/path.rs` loses the `Interner`, the `OnceLock>` global, and gains the `Arc` field; `promptforge/crates/shared-vfs/src/handle.rs` makes `VfsRef::acquire`, `acquire_with`, and `Access::spawn` return `Result<_, VfsError>`; `promptforge/crates/shared-vfs/src/router.rs` already propagates backend acquisition failure and needs no contract change; `promptforge/crates/shared-vfs/src/lib.rs` fixes the manifest test's section matching. +- File and public API changes: `VfsPath` loses `Copy`; `VfsPath::as_str` returns `&str` borrowed from self instead of `&'static str`; trait signatures already take `&VfsPath`, so backends are untouched; `VfsRef::acquire` and `Access::spawn` gain `Result`; the `promptforge-bashkit` crate leaves the workspace; `promptforge/crates/gateway-local/src/artifacts.rs` changes `mod confine;` to `pub(crate) mod confine;` so `cache.rs` tests reach `source_marker_path` by its real path. +- Data, persistence, failure, security, and privacy constraints: no persisted or wire format changes; the panic-to-error change alters failure behavior only in a path no current backend can reach (both shipped backends have infallible acquisition); the interner removal alters memory behavior only upward (bounded by live owners instead of by history). + + + + +## Testing Plan + +Each work item carries a focused check; the shared-vfs changes add new regression tests; exit is the union of the per-crate suites and workspace lints. No test behavior changes except the two new regression tests and the updated interner property test. + +- Unit: new regression test in shared-vfs - a stub backend whose `acquire` returns `Err` fails acquisition with a `VfsError` instead of panicking; the manifest test gains a `[dependencies.foo]` sub-table row; the `identical_paths_intern_to_one_entry` test's pointer-equality assertion becomes content equality; a loop canonicalizing distinct paths is asserted not to grow the heap monotonically. +- Integration and end-to-end: the claims conflict tests (`two_writes_by_two_identities_conflict`, the copy/rename claim matrix) and the fanout suite in promptforge-core prove the de-interned path still keys the claims model; the executor suite proves acquisition failure surfaces as `RunErrorKind::Store`. +- Regression, security, and performance: the heap-growth check covers the model-reachable leak; no performance criterion - claim lookups move from integer compare to short-string hash, unmeasured because the rate is one per VFS operation. +- Exit criteria: `cargo check`, `cargo clippy --all-targets`, and `cargo test` green for `promptforge-core`, `promptforge-lua`, `gateway-config`, `gateway-local`, `workshop-server`, `shared-vfs`, `promptforge-vfs`, `promptforge-store`, and `promptforge-agent`; `cargo metadata --locked` and `cargo build` green at workspace root; a grep of `tools-public/coding/vibe-coder.md` confirms no instruction stages or commits a ledger file. + + + + +## Decision Record + +- Decisions: + - Coarse steps over fine steps: the re-export removals are one behavior slice verified by one test scope, not one commit per file; the whole run targets about five commits. The user's words: "15 steps is way too many individual commits ... more steps than the previous plan which added a whole feature". + - Remove `crates/promptforge-bashkit/` rather than relocating or excluding it: the spike's deliverable was evidence already recorded in the run ledger. The user's words: "remove the crate (c)". + - De-intern `VfsPath` to an `Arc`-backed value with no global table (option A): matches the intended ownership semantics with the least machinery; the integer-compare optimization it discards was never measured. The user's words: "why do we intern strings? that's not what I wanted. I suggested shared strings, but an owner would have to exist. once the last claim which shares a string goes away then we dont need the shared string anymore" and "I lean A". + - No content-level dedup: unmeasured need, and dedup can be added later as a purely internal change behind the same `Arc` shape. The user asked "should we implement the dedup?" and did not object to the negative recommendation. + - Keep `Arc` rather than `Box`: equal size, cheaper clones at per-operation claim sites, `Send + Sync` (claims ride with `Access` into `spawn_blocking`), and exactly the requested shared-with-owner semantics. The user asked "do we even need the Arc then" and did not object to the recommendation to keep it. + - Make `VfsRef::acquire` and `Access::spawn` fallible (option a): about six production call sites after the bashkit removal; the trait already documents the error and the router already propagates it. The user's words: "what do you think we should do? there's almost no callers", then "3. A". + - Fix the stale `store` component line in `promptforge/vibe/archdoc.md` and add the missing VFS layer line. The user's words: "fix archdoc.md". + - Patch `tools-public/coding/vibe-coder.md` so the ledger is scratch and never staged: the 2026-09-08 rewrite regressed the original scratch-ledger convention. The user's words: "can we fix vibe-coder.md to not generate these things in the repo?" + - Remove the three committed ledger files from promptforge. The user selected "Patch the tool and remove the three committed files". + - Patch `tools-public/rulebooks/rust-rulebook.md` section 11 with the test-import rule, detection entry, and correction pair: the rulebook sanctioned the test glob without constraining who may feed it. The user's words: "does rust-rulebook.md need a patch to prevent this?" +- Rejected alternatives: + - Refcounted global interner with eviction (DEBT-VFS-02 option B): keeps process-global state, a lock on every `canonicalize`, and eviction bookkeeping. Revisit only if profiling shows `canonicalize` allocations hot; option A does not foreclose it. + - Per-owner interner (DEBT-VFS-02 option C): overlays mount one handle as another's backend, so paths from two tables mix in one claims flow; cross-table identity is a real correctness risk. No revisit condition identified. + - Narrow the `Vfs::acquire` contract to must-not-fail and keep the panic (DEBT-VFS-03 option b): cements the contradiction and forces the anticipated SQLite backend into infallible-acquire contortions. Revisit only if fallible acquisition proves unaffordable at a future boundary. + - Relocate the bashkit spike to `spikes/` or exclude it in the root manifest (DEBT-VFS-01 options a and b): both keep dead weight whose evidence is already recorded. Revisit if the adapter regains a consumer. +- Assumptions, risks, and notes: + - The CI failure from the bashkit path dependency is inferred from cargo's probed resolution behavior, not an observed run; the mechanism is certain on any sibling-less machine. + - Debt evidence came from a two-pass collection (analysis, then independent challenge) over the twelve target commits; the challenger upheld all four accepted findings and all twelve rejections. + - Rejected debt candidates, recorded so they are not re-litigated: claim granularity for glob/list/grep matches the designed per-path contract; overlays lose `read_range` push-down (performance only); `Ask` collapses to `PermissionDenied` (recorded v1 decision); `write_owned` deferred with no caller; host-backend stage-1 containment limits explicitly scoped; nested-handle double claim registration never self-conflicts; plan-mode copy refusal is conservative and visible; a cancelled run's in-flight store op completes (bounded, documented); anchor-replace duplication shows no drift; agent VMs keep inline store closures (single identity); process-global tables beyond the interner show no independent contradiction. + - `promptforge/vibe/archdoc-next.md` does not exist at the disposition ref; no queue records needed resolution. + + + + +## Project Survey + +- Status: complete +- Build command: `cargo build` (builds only the gateway, the default workspace member; `cargo build -p workshop` for the desktop app, or `cargo workshop` for the one-command staged Workshop build) +- Focused test command pattern: `cargo test -p ` or `cargo nextest run -p ` (CI uses e.g. `cargo test -p gateway-stt --test it architecture`) +- Component test command pattern: `cargo nextest run --locked -p ` (add `--all-features` where the crate gates features; gateway race tests use `cargo test --locked -p gateway --no-default-features --features test-fixtures --test it `) +- Full-suite test command: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --all-features`, then doctests via `cargo test --workspace --exclude workshop --exclude workshop-server --all-features --doc`; workshop crates separately: `cargo nextest run --locked -p workshop -p workshop-server` plus `cargo test --doc -p workshop -p workshop-server` +- Linter command: `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings` (workshop crates: `cargo clippy -p workshop -p workshop-server --all-targets -- -D warnings`); supply-chain: `cargo deny check` and `cargo audit` +- Formatter check command: `cargo fmt --all --check` +- Docs command: `RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server`; user guide: `mdbook build guide` +- Test placement and naming conventions: unit tests live inline in source files under `#[cfg(test)] mod tests`; integration tests are a single `it` target rooted at `crates//tests/it/main.rs` with one module per area (e.g. `tests/it/boot.rs`, nested `tests/it/realtime_stt/*.rs`), fixtures under `tests/fixtures/` and shared helpers under `tests/common/`; a few crates use standalone `tests/.rs` targets instead. Test functions are long snake_case sentences (e.g. `a_direct_launch_recovers_the_lease_from_a_terminated_owner`). Ordinary `cargo test` must stay fully offline (no downloads, network, external processes, models, or credentials). UI packages under `crates/*/ui` test with `npm test`; Node helper scripts in `tools/` have sibling `*.test.mjs` files. Benchmarks use criterion (dev-only). +- Directory map: `crates/` holds all 37 Rust workspace members plus `shared-ui` (a TypeScript+CSS package excluded from the Cargo glob); `guide/` is the mdbook user guide (four doc sets: Workshop, gateway, prompt language, agent programs); `design/` holds design notes; `tools/` holds Node.js helper scripts (gateway sidecar staging, TTS live checks); `prompts/` holds prompt files; `vibe/` holds session plans, the ledger, and `archdoc.md`; `images/` holds docs art; `local/` holds local config; `.github/workflows/` holds CI (ci.yml, release, nightly, guide); `.config/nextest.toml` configures nextest; `clippy.toml`, `rustfmt.toml`, `deny.toml`, `dist-workspace.toml`, `rust-toolchain.toml` pin tooling at the root. +- Component boundaries (per `vibe/archdoc.md` and `AGENTS.md`): the executor product (`promptforge`, `promptforge-core`, `promptforge-core-support`, `promptforge-parser`, `promptforge-lua`, `promptforge-agent`, `promptforge-store`, `promptforge-vfs`, `promptforge-tools`, `promptforge-bashkit`, `promptforge-webfetch`, `promptforge-web-search`, `promptforge-model-client`, `promptforge-tool-picker`) parses and runs prompt pipelines and Lua agent programs; the gateway product (`gateway`, `gateway-config`, `gateway-config-ui`, `gateway-local`, `gateway-logging`, `gateway-routing`, `gateway-stt`, `gateway-stt-engine`, `gateway-stt-backend-whisper`, `gateway-whisper-ffi`, `gateway-web-search`) owns model routing, provider credentials, and local inference; the workshop product (`workshop`, `workshop-server`) is the Tauri desktop shell and in-process server; the shared substrate (`shared-vfs`, `shared-loopback`, `shared-progress`, `shared-protocol`, `shared-sidecar`, `shared-ui`) depends on no product crate. Four dependency rules bind: gateway crates cannot depend on workshop crates; promptforge crates cannot depend on gateway or workshop crates; gateway crates cannot depend on promptforge crates; workshop crates cannot depend on gateway crates. `product-integration-tests` exercises cross-product behavior; `build-*` crates are build tooling. +- Conventions summary: Rust edition 2024 with workspace lint tables in the root `Cargo.toml` (`unsafe_code` forbidden, clippy `all` denied and `pedantic` warned, `unwrap_used`/`expect_used` denied, rustdoc link lints denied); lints live in manifest tables, not attributes. Behavior changes ship with tests in the same change; structural enforcement tests require explicit user approval. No clap - binaries hand-roll argv parsing. `Result` for expected failures; library and serve paths never exit the process or install process-global state. Comments explain non-obvious constraints and cite upstream issue URLs for workarounds. Long-running work reports through `shared-progress`. Unsafe stays in its owned boundary (`gateway-whisper-ffi`) with documented invariants. Cargo features gate real constraints (toolchain, native builds), not product shape. Builds must not dirty the git tree (CI enforces a clean-tree check). UI bundles build into `OUT_DIR` via crate build scripts driven by `npm ci`. + + + + +## Execution Instructions + +Components in dependency order, re-decomposed after the operator rejected the fine-grained layout ("way too many individual commits ... more steps than the previous plan which added a whole feature"): + +- `tooling` (steps 1-2): the rust-rulebook and vibe-coder patches. Both landed out of band in the tools-public repository as `3705831` before the run started, so both steps are complete at seed time and the patched rules govern this run. +- `cleanup` (steps 3-4): the workspace-wide test-namespace cleanup as one behavior slice verified by one test scope, then the removal of the three committed ledger files. Placed before the VFS work so later commits stay free of ledger staging. +- `vfs-debt-removal` (steps 5-7): the bashkit removal, then the shared-vfs contract corrections as one slice (de-interning, the manifest test bypass, and the archdoc correction whose wording depends on de-interning), then fallible acquisition, whose caller set assumes the bashkit adapter is gone. + +Each step is one commit containing its code and tests. + + + +### Step 1: add the test-import rule to rust-rulebook.md [completed] + +- Component: tooling +- In `tools-public/rulebooks/rust-rulebook.md` section 11: add the rule that test imports live inside the test module (with the same-module `#[cfg(test)]` code carve-out), the detection entry for `#[cfg(test)]` imports or re-exports whose only consumers are test modules, and the correction pair showing the move into `mod tests`. +- Landed out of band: applied directly and committed in the tools-public repository as `3705831` before the run started. + + + + + +### Step 2: make the vibe ledger scratch in vibe-coder.md [completed] + +- Component: tooling +- In `tools-public/coding/vibe-coder.md`: allocate one scratch `vibe-ledger.md` per run in the Per-Step Cycle preamble (keyed by the active plan's vibe name, never staged); make Resume Recovery read the scratch ledger only when it survives, with plan marks and git log authoritative otherwise; make the Mark step append to the scratch ledger and stage only the plan marker; align the human-facing preamble ("the audit ledger is scratch, never committed"). +- Landed out of band: applied directly and committed in the tools-public repository as `3705831` before the run started. + + + + + +### Step 3: remove test-only re-exports workspace-wide and relocate test imports + +- Component: cleanup +- One behavior slice across five crates: delete every `#[cfg(test)]` import or re-export whose only consumers are test modules, and let each consumer import the name directly from its real home. The `execute.rs` and `execute/tests/mod.rs` portion is already applied in the worktree (coded before the re-decomposition); this step completes the remaining files and lands the whole slice as one commit. +- In `promptforge/crates/promptforge-core/src/execute.rs`: delete the `#[cfg(test)]` re-export block and the stray `ModelSet` re-export, change `pub(crate) use context::RunContext;` to a private `use`, and change `pub(crate) mod scheduler;` to `mod scheduler;` (already applied). In `promptforge/crates/promptforge-core/src/execute/tests/mod.rs`: the relocated imports, merged into existing use lines (already applied). The thirteen child test files keep their existing globs. +- In `promptforge/crates/promptforge-core/src/lua.rs`: delete the `#[cfg(test)]` re-exports of `Compactor`, `Conflict`, and `ToolRuntime`; point the consumers (`execute/tool_loop.rs`'s test-only wrapper signature, `execute/tests/tool_loop.rs`, `execute/tests/tool_scoping.rs`) at `promptforge_lua` directly. +- In `promptforge/crates/promptforge-core/src/client.rs`: delete the `#[cfg(test)]` `ToolSchemaError` re-export; the test module of `promptforge/crates/promptforge-core/src/error.rs` imports `promptforge_model_client::client::ToolSchemaError` directly. +- In `promptforge/crates/promptforge-core/src/model.rs`: delete the `#[cfg(test)]` `ModelInvocation` re-export; the five consumer test files (`execute/tests/scheduler.rs`, `execute/tests/input.rs`, `execute/tests/models_loop.rs`, `model/tests/mod.rs`, `lua/coro_tests.rs`) import `promptforge_model_client::model::ModelInvocation` directly. +- In `promptforge/crates/promptforge-core/src/cancel.rs`: delete the `#[cfg(test)]` `scope` re-export; the three consumer test files import `promptforge_core_support::cancel::scope` directly and call `scope(` instead of `cancel::scope(` at six sites. +- In `promptforge/crates/promptforge-lua/src/lib.rs`: delete the `#[cfg(test)] pub(crate) use vm::{LuaOutcome, run_chunk};`; `promptforge/crates/promptforge-lua/src/tests.rs` extends its existing `use crate::vm::LocalTools;` to include both names. +- In `promptforge/crates/gateway-config/src/config.rs`: delete the `#[cfg(test)]` `interpolate` re-export (the non-test `interpolate_value` re-export stays); `config/tests.rs` adds `use super::interpolate::interpolate;`. +- In `promptforge/crates/gateway-local/src/artifacts.rs`: delete the `#[cfg(test)] use archive::extract_archive;` and the `#[cfg(test)] pub(crate) use confine::source_marker_path;`; change `mod confine;` to `pub(crate) mod confine;`; `artifacts/tests.rs` extends its existing `super::archive` import with `extract_archive` and adds `use super::confine::source_marker_path;`; `cache.rs` changes its test import to `crate::artifacts::confine::source_marker_path`. +- In `promptforge/crates/workshop-server/src/heartbeat.rs`: move the `#[cfg(test)] use crate::gateway::GatewayClient;` inside the inline `mod tests`. +- Verification: `cargo check`, `cargo clippy --all-targets`, and `cargo test` green for `promptforge-core`, `promptforge-lua`, `gateway-config`, `gateway-local`, and `workshop-server`; a grep finds no `#[cfg(test)]` import or re-export in a non-test module whose only consumers are test modules. + + + + + +### Step 4: remove the three committed ledger files + +- Component: cleanup +- In the promptforge repository: move root `vibe-ledger.md`, root `vibe-review.md`, and `vibe/vibe-ledger.md` to `cabinet/_trash/`, stating the recovery sentence for each, and commit the removal. +- Verification: `git status` clean; the commit touches only those three files. + + + + + +### Step 5: remove the promptforge-bashkit crate + +- Component: vfs-debt-removal +- Remove `promptforge/crates/promptforge-bashkit/` by moving it to `cabinet/_trash/` (stating the recovery sentence), regenerate `Cargo.lock`, and sweep remaining bashkit references in CI, guide, and READMEs. +- Verification: `cargo metadata --locked` and `cargo build` succeed with no sibling checkout beside the repository; no bashkit references remain. + + + + + +### Step 6: de-intern VfsPath, close the manifest test bypass, and correct archdoc + +- Component: vfs-debt-removal +- In `promptforge/crates/shared-vfs/src/path.rs`: remove the `Interner` and the `OnceLock>` global; give `VfsPath` an `Arc` field so `canonicalize` allocates one `Arc` per call and the string frees when its last owner drops. `VfsPath` loses `Copy`; `VfsPath::as_str` returns `&str` borrowed from self instead of `&'static str`. Update claim sites to hold clones, change the `identical_paths_intern_to_one_entry` property test's pointer-equality assertion to content equality, and add the regression check that a loop canonicalizing distinct paths does not grow the heap monotonically. +- In `promptforge/crates/shared-vfs/src/lib.rs`: extend the section matching in `the_manifest_declares_no_dependencies` so any header equal to or starting with `dependencies.`, `dev-dependencies.`, or `build-dependencies.` is a dependency table, with a regression row. +- In `promptforge/vibe/archdoc.md`: correct the `store` component line to name the facade and its dependency, and add a VFS layer line covering `shared-vfs` (canonical paths, claims, routing, memory and host backends) and `promptforge-vfs` (policy gate), both depending on none. +- Verification: the heap-growth check, the claims conflict tests (`two_writes_by_two_identities_conflict`, the copy/rename claim matrix), and the fanout suite pass; the manifest test fails with an injected `[dependencies.foo]` sub-table and passes without it; the archdoc component list matches the workspace's actual dependency directions. + + + + + +### Step 7: make handle acquisition fallible + +- Component: vfs-debt-removal +- Depends on step 5: the described caller set assumes the bashkit adapter (three acquisition call sites) is gone. +- In `promptforge/crates/shared-vfs/src/handle.rs`: make `VfsRef::acquire`, `acquire_with`, and `Access::spawn` return `Result<_, VfsError>`. +- Map acquisition failure to `RunErrorKind::Store` at the executor boundary and to the agent's error in `promptforge-agent`; adjust tests and doc examples mechanically. +- Add the regression test: a stub backend whose `acquire` returns `Err` fails acquisition with a `VfsError` instead of panicking. +- Verification: the failing-backend regression test and the executor and agent suites pass. + + + +- Deferred and out of scope: `write_owned` (deferred pending profiling, no caller); content-level dedup (rejected, revisit on measurement); the twelve rejected debt candidates; `#[cfg(test)]` helpers whose consumers live in the same module (legitimate, stay); the facade re-exports (deliberate, stay); historical run plans under `promptforge/vibe/` that reference the removed ledger files (dated records, stay). +- Exit criteria: per-step verification above, then the Testing Plan exit criteria - `cargo check`, `cargo clippy --all-targets`, and `cargo test` green for `promptforge-core`, `promptforge-lua`, `gateway-config`, `gateway-local`, `workshop-server`, `shared-vfs`, `promptforge-vfs`, `promptforge-store`, and `promptforge-agent`; `cargo metadata --locked` and `cargo build` green at workspace root; a grep of `tools-public/coding/vibe-coder.md` confirms no instruction stages or commits a ledger file. + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..9a9660250 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-09-12-1-test-namespace-vfs-debt.md \ No newline at end of file From 4fc4825067f34e9227b213b6f1580567fadaa597 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 02:26:09 -0700 Subject: [PATCH 14/26] Remove test-only re-exports and relocate test imports Test modules across five crates no longer reach helper names through cfg-gated re-exports on their parent modules; each consumer now imports the name from the module that defines it. The executor drops the compat namespace it kept alive for its test glob, and its scheduler module and run context return to module-private visibility. One artifact submodule becomes crate-visible so cache tests reach a marker-path helper by its real path. No production behavior changes. - `crates/promptforge-core/src/execute.rs` deletes the test-only re-export block and the stray `ModelSet` re-export, narrows `pub(crate) mod scheduler` to `mod scheduler`, and makes `RunContext` a private import; the executor test module now imports those names from their defining submodules. - `crates/gateway-local/src/artifacts.rs` widens `mod confine` to `pub(crate) mod confine` so `cache.rs` tests import `source_marker_path` by its real path instead of through a test-only re-export. - `run_prose_inference` and the test-only `model_client` helper now name `promptforge_lua::Compactor` and `promptforge_model_client::client::GatewayClient` directly in their signatures, replacing aliases that existed only under cfg(test). - `crates/promptforge-core/src/lua.rs`, `crates/promptforge-core/src/cancel.rs`, `crates/promptforge-core/src/client.rs`, `crates/promptforge-core/src/model.rs`, `crates/promptforge-lua/src/lib.rs`, and `crates/gateway-config/src/config.rs` each lose their cfg(test) re-exports; the six cancellation call sites invoke `scope(` from `promptforge_core_support::cancel` directly. - No `#[cfg(test)]` import or re-export remains in any non-test module touched here, and no call site changes observable behavior. Design: removes shim @ crates/promptforge-core/src/execute.rs Plan: vibe/2026-09-12-1-test-namespace-vfs-debt.md --- crates/gateway-config/src/config.rs | 2 - crates/gateway-config/src/config/tests.rs | 1 + crates/gateway-local/src/artifacts.rs | 8 +--- crates/gateway-local/src/artifacts/tests.rs | 3 +- crates/gateway-local/src/cache.rs | 2 +- crates/promptforge-core/src/cancel.rs | 2 - crates/promptforge-core/src/client.rs | 3 -- crates/promptforge-core/src/error.rs | 4 +- crates/promptforge-core/src/execute.rs | 40 +------------------ .../src/execute/tests/input.rs | 8 ++-- .../promptforge-core/src/execute/tests/mod.rs | 23 +++++++---- .../src/execute/tests/models_loop.rs | 3 +- .../src/execute/tests/scheduler.rs | 23 ++++++----- .../src/execute/tests/tool_loop.rs | 3 +- .../src/execute/tests/tool_scoping.rs | 18 ++++----- .../promptforge-core/src/execute/tool_loop.rs | 2 +- crates/promptforge-core/src/lua.rs | 9 ----- crates/promptforge-core/src/lua/coro_tests.rs | 6 ++- crates/promptforge-core/src/model.rs | 2 - .../promptforge-core/src/model/tests/mod.rs | 1 + crates/promptforge-lua/src/lib.rs | 2 - crates/promptforge-lua/src/tests.rs | 2 +- crates/promptforge-lua/src/vm.rs | 4 +- crates/workshop-server/src/heartbeat.rs | 3 +- crates/workshop-server/src/session_agents.rs | 7 ++-- vibe/2026-09-12-1-test-namespace-vfs-debt.md | 2 +- 26 files changed, 72 insertions(+), 111 deletions(-) diff --git a/crates/gateway-config/src/config.rs b/crates/gateway-config/src/config.rs index 947192f90..5890ec9f4 100644 --- a/crates/gateway-config/src/config.rs +++ b/crates/gateway-config/src/config.rs @@ -19,8 +19,6 @@ pub use companion::{ SpeculativeConfig, }; pub(crate) use imp::reject_profiles_directory; -#[cfg(test)] -pub(crate) use interpolate::interpolate; pub(crate) use interpolate::interpolate_value; use stt::RawSttPipelineConfig; pub use stt::{ diff --git a/crates/gateway-config/src/config/tests.rs b/crates/gateway-config/src/config/tests.rs index 48e27625e..e280fec30 100644 --- a/crates/gateway-config/src/config/tests.rs +++ b/crates/gateway-config/src/config/tests.rs @@ -1,3 +1,4 @@ +use super::interpolate::interpolate; use super::*; const SAMPLE: &str = r#" diff --git a/crates/gateway-local/src/artifacts.rs b/crates/gateway-local/src/artifacts.rs index a8e55655e..bb5fcadad 100644 --- a/crates/gateway-local/src/artifacts.rs +++ b/crates/gateway-local/src/artifacts.rs @@ -14,7 +14,7 @@ mod archive; mod assets; -mod confine; +pub(crate) mod confine; mod digest; mod download; mod progress; @@ -33,8 +33,6 @@ use tokio_util::sync::CancellationToken; use crate::error::LocalError; -#[cfg(test)] -use archive::extract_archive; use archive::extract_archive_with_progress; use archive::find_executable; use archive::require_executable; @@ -56,10 +54,6 @@ pub(crate) use confine::{ enforce_private_cache_root, ensure_cache_directory, part_path, remove_cache_entry, rename_confined, safe_relative_path, validate_cache_path, write_synced, }; -// Test builds only: the resume tests in this module and cache.rs build the -// marker path; the download path itself imports it from confine directly. -#[cfg(test)] -pub(crate) use confine::source_marker_path; pub(crate) use digest::hex_digest; pub use digest::parse_expected_digest; pub(crate) use download::{download_with_progress, hub_bearer_token_from_env}; diff --git a/crates/gateway-local/src/artifacts/tests.rs b/crates/gateway-local/src/artifacts/tests.rs index 86b69f882..e476e824a 100644 --- a/crates/gateway-local/src/artifacts/tests.rs +++ b/crates/gateway-local/src/artifacts/tests.rs @@ -10,8 +10,9 @@ use tempfile::TempDir; use shared_progress::{EventState, ProgressHub}; use tokio_util::sync::CancellationToken; -use super::archive::{extract_archive_with_progress, safe_archive_path}; +use super::archive::{extract_archive, extract_archive_with_progress, safe_archive_path}; use super::assets::ArchiveRef; +use super::confine::source_marker_path; use super::digest::file_digest; use super::download::{hub_bearer_token, is_huggingface_https}; use super::progress::{DownloadProgress, TreeProgress}; diff --git a/crates/gateway-local/src/cache.rs b/crates/gateway-local/src/cache.rs index 9dbeb4260..1d7c0dc90 100644 --- a/crates/gateway-local/src/cache.rs +++ b/crates/gateway-local/src/cache.rs @@ -615,7 +615,7 @@ mod tests { use tempfile::TempDir; use super::*; - use crate::artifacts::source_marker_path; + use crate::artifacts::confine::source_marker_path; use crate::testsupport::{FakeServer, hex_sha256}; /// Test double recording the progress callbacks a download drives. diff --git a/crates/promptforge-core/src/cancel.rs b/crates/promptforge-core/src/cancel.rs index 2225e55c1..8c810e8f4 100644 --- a/crates/promptforge-core/src/cancel.rs +++ b/crates/promptforge-core/src/cancel.rs @@ -4,8 +4,6 @@ //! re-exported here unchanged, so existing `promptforge_core::cancel::*` paths //! keep working. -#[cfg(test)] -pub(crate) use promptforge_core_support::cancel::scope; pub(crate) use promptforge_core_support::cancel::{ CancelHandle, current, is_cancelled, maybe_scope, wait_cancelled, }; diff --git a/crates/promptforge-core/src/client.rs b/crates/promptforge-core/src/client.rs index d23e7020b..bd470094b 100644 --- a/crates/promptforge-core/src/client.rs +++ b/crates/promptforge-core/src/client.rs @@ -18,6 +18,3 @@ pub use promptforge_model_client::client::{ Completion, CompletionResult, GatewayClient, GatewayEndpoint, Message, SecretError, SecretString, StreamDelta, ToolArguments, ToolCall, ToolSchema, }; - -#[cfg(test)] -pub(crate) use promptforge_model_client::client::ToolSchemaError; diff --git a/crates/promptforge-core/src/error.rs b/crates/promptforge-core/src/error.rs index 7bba44848..f9f67e588 100644 --- a/crates/promptforge-core/src/error.rs +++ b/crates/promptforge-core/src/error.rs @@ -812,7 +812,9 @@ mod tests { // F5: the binding and tool-scope failures keep the originating typed // error as a private `source()` instead of flattening it to a string, // and the chain survives through the public `RunError` wrapper. - let schema_error = crate::client::ToolSchemaError::NonObjectSchema { + use promptforge_model_client::client::ToolSchemaError; + + let schema_error = ToolSchemaError::NonObjectSchema { name: "echo".to_owned(), }; let bind = Error::BindSchema { diff --git a/crates/promptforge-core/src/execute.rs b/crates/promptforge-core/src/execute.rs index 4a33a7c55..fe4d71bca 100644 --- a/crates/promptforge-core/src/execute.rs +++ b/crates/promptforge-core/src/execute.rs @@ -77,7 +77,7 @@ mod engine; mod error; mod gateway; pub(crate) mod protocol; -pub(crate) mod scheduler; +mod scheduler; mod scope; mod section_context; pub(crate) mod section_vm; @@ -90,51 +90,15 @@ pub use config::{RunConfig, RunLimits}; pub use error::{RunError, RunErrorKind}; pub use gateway::ResolutionContext; -// Crate-internal items reused through the historical `crate::execute::` path. -// Re-exported so the split stays surface-neutral for the public API while -// keeping one import path for internal collaborators. -pub(crate) use context::RunContext; - +use context::RunContext; use scheduler::Scheduler; -// Everything the executor's own tests reach through `use super::super::*` -// (and that `tests/mod.rs` does not itself import): executor-internal items, -// crate types, and the two external conveniences (`json`, `BTreeMap`). -// Test-only, so the non-test lib carries no unused re-export while the -// historic executor namespace stays intact for the test glob. -#[cfg(test)] -pub(crate) use crate::Result; -#[cfg(test)] -pub(crate) use crate::client::ToolSchema; -#[cfg(test)] -pub(crate) use crate::lua::{SectionVm, ToolCallCounts}; -#[cfg(test)] -pub(crate) use crate::observe::Observer; -#[cfg(test)] -pub(crate) use gateway::GatewaySource; -#[cfg(test)] -pub(crate) use gateway::env_client_with_limits; -#[cfg(test)] -pub(crate) use scope::{DispatchTarget, prepare_effective_scope, prepare_scoped_tools}; -#[cfg(test)] -pub(crate) use serde_json::json; -#[cfg(test)] -pub(crate) use std::collections::BTreeMap; -#[cfg(test)] -pub(crate) use support::{advance_turn, now_rfc3339_checked}; -#[cfg(test)] -pub(crate) use tool_loop::{LocalDispatch, run_prose_inference}; - use crate::Error; use crate::cancel; use crate::observe::detail; use crate::parser::{ParseErrorKind, Prompt}; use crate::store::VfsRef; -// Re-exported for the executor test glob. -#[cfg(test)] -pub(crate) use crate::model::ModelSet; - /// Executes a parsed prompt and returns its final text. /// /// H1 Lua and prose blocks run once in source order with full host access; diff --git a/crates/promptforge-core/src/execute/tests/input.rs b/crates/promptforge-core/src/execute/tests/input.rs index e74f455f7..bc426a52e 100644 --- a/crates/promptforge-core/src/execute/tests/input.rs +++ b/crates/promptforge-core/src/execute/tests/input.rs @@ -8,7 +8,8 @@ use super::*; use crate::execute::scheduler::Scheduler; use crate::input::{INPUT_UNAVAILABLE_FALLBACK, InputBroker, InputError, InputOutcome, InputTool}; use crate::lua::{ToolBinding, ToolSet}; -use crate::model::{ModelBinding, ModelId, ModelInvocation}; +use crate::model::{ModelBinding, ModelId}; +use promptforge_model_client::model::ModelInvocation; /// The model set an input test's run carries: `writer` (the prompt-wide /// default, model `test-model`), so `models.loop` resolves a binding. @@ -302,7 +303,8 @@ async fn an_uncaught_broker_failure_fails_the_run_typed() { #[tokio::test(flavor = "current_thread", start_paused = true)] async fn cancellation_interrupts_a_pending_input_wait() { - use crate::cancel::{self, CancelHandle}; + use crate::cancel::CancelHandle; + use promptforge_core_support::cancel::scope; use std::time::{Duration, Instant}; let md = input_prompt("user_input()\nreturn 'unreachable'"); @@ -317,7 +319,7 @@ async fn cancellation_interrupts_a_pending_input_wait() { }); let start = Instant::now(); let mut scheduler = Scheduler::new(&ctx, None); - let result = cancel::scope(handle, scheduler.drive()).await; + let result = scope(handle, scheduler.drive()).await; assert!( start.elapsed() < Duration::from_secs(5), "cancel during a pending input wait must return promptly, took {:?}", diff --git a/crates/promptforge-core/src/execute/tests/mod.rs b/crates/promptforge-core/src/execute/tests/mod.rs index 190021129..399057b9b 100644 --- a/crates/promptforge-core/src/execute/tests/mod.rs +++ b/crates/promptforge-core/src/execute/tests/mod.rs @@ -1,5 +1,6 @@ //! Unit tests for section execution, tool scoping, and the tool-call loop. +use std::collections::BTreeMap; use std::net::SocketAddr; use std::num::NonZeroU32; use std::sync::atomic::{AtomicU32, AtomicUsize, Ordering}; @@ -13,14 +14,21 @@ use axum::routing::post; use promptforge_tool_picker::{ Catalog, Config as PickerConfig, ToolDescriptor, ToolId as PickerToolId, ToolPicker, }; -use serde_json::Value; +use serde_json::{Value, json}; +use super::gateway::{GatewaySource, env_client_with_limits}; +use super::scope::{DispatchTarget, prepare_effective_scope, prepare_scoped_tools}; +use super::support::{advance_turn, now_rfc3339_checked}; +use super::tool_loop::{LocalDispatch, run_prose_inference}; use super::*; -use crate::client::{GatewayClient, GatewayEndpoint, SecretString}; +use crate::Result; +use crate::client::{GatewayClient, GatewayEndpoint, SecretString, ToolSchema}; use crate::debug::DebugCapture; -use crate::lua::{LuaProgram, current_tool_bindings}; -use crate::model::{CompletionOptions, ModelCatalog, ModelDescriptor, ModelId, ThinkingMode}; -use crate::observe::{NullObserver, Observation, detail}; +use crate::lua::{LuaProgram, SectionVm, ToolCallCounts, current_tool_bindings}; +use crate::model::{ + CompletionOptions, ModelCatalog, ModelDescriptor, ModelId, ModelSet, ThinkingMode, +}; +use crate::observe::{NullObserver, Observation, Observer, detail}; use crate::store::{Access, StoreError, StoreExt, VfsRef}; use crate::tools::{Tool, ToolCatalog, ToolError, ToolErrorKind, ToolId, ToolOutput}; use crate::untrusted::GuardNonce; @@ -1329,7 +1337,8 @@ impl Tool for SlowTool { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn cancel_during_in_flight_tool_call_returns_promptly() { - use crate::cancel::{self, CancelHandle}; + use crate::cancel::CancelHandle; + use promptforge_core_support::cancel::scope; use std::time::{Duration, Instant}; let gateway = ScriptedGateway::start(echo_then_text_script()).await; @@ -1350,7 +1359,7 @@ async fn cancel_during_in_flight_tool_call_returns_promptly() { }); let start = Instant::now(); - let result = cancel::scope( + let result = scope( handle, run_tool_loop( &client, diff --git a/crates/promptforge-core/src/execute/tests/models_loop.rs b/crates/promptforge-core/src/execute/tests/models_loop.rs index 00be0b043..0090c7376 100644 --- a/crates/promptforge-core/src/execute/tests/models_loop.rs +++ b/crates/promptforge-core/src/execute/tests/models_loop.rs @@ -9,7 +9,8 @@ use super::*; use crate::execute::scheduler::Scheduler; use crate::lua::{OverflowReason, ToolSet}; -use crate::model::{ModelBinding, ModelId, ModelInvocation}; +use crate::model::{ModelBinding, ModelId}; +use promptforge_model_client::model::ModelInvocation; /// The model set a loop test's run carries: `writer` (the prompt-wide /// default, model `test-model`) and `other` (model `other-model`), so an diff --git a/crates/promptforge-core/src/execute/tests/scheduler.rs b/crates/promptforge-core/src/execute/tests/scheduler.rs index b6f243804..7dc4e5664 100644 --- a/crates/promptforge-core/src/execute/tests/scheduler.rs +++ b/crates/promptforge-core/src/execute/tests/scheduler.rs @@ -18,7 +18,8 @@ use std::num::NonZeroUsize; use super::*; use crate::execute::protocol::Answer; use crate::execute::scheduler::Scheduler; -use crate::model::{ModelBinding, ModelId, ModelInvocation}; +use crate::model::{ModelBinding, ModelId}; +use promptforge_model_client::model::ModelInvocation; /// The model set the live H1 pass would leave behind: one `writer` binding /// as the prompt-wide default. The scheduler's tests bypass H1, so they @@ -112,7 +113,8 @@ async fn nested_call_and_inference_run_end_to_end_on_a_current_thread_runtime() #[tokio::test(flavor = "current_thread")] async fn cancellation_while_suspended_on_infer_interrupts_the_run() { - use crate::cancel::{self, CancelHandle}; + use crate::cancel::CancelHandle; + use promptforge_core_support::cancel::scope; let gateway = ScriptedGateway::start(vec![resp_delayed_text( "too late", @@ -138,7 +140,7 @@ async fn cancellation_while_suspended_on_infer_interrupts_the_run() { canceller.cancel(); }); - let result = cancel::scope(cancel, async { + let result = scope(cancel, async { Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))) .drive() .await @@ -2095,7 +2097,8 @@ async fn pre_cancelled_fanout_returns_interrupted() { // Mirror of the legacy `pre_cancelled_fanout_returns_interrupted`: a // fanout entered under an already-cancelled handle fails the run with // Error::Interrupted instead of running the arms. - use crate::cancel::{self, CancelHandle}; + use crate::cancel::CancelHandle; + use promptforge_core_support::cancel::scope; let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # Fanout\n\n\ @@ -2110,7 +2113,7 @@ async fn pre_cancelled_fanout_returns_interrupted() { let ctx = scheduler_context(&prompt); let cancel = CancelHandle::new(); cancel.cancel(); - let result = cancel::scope(cancel, async { Scheduler::new(&ctx, None).drive().await }).await; + let result = scope(cancel, async { Scheduler::new(&ctx, None).drive().await }).await; assert!( matches!(result, Err(Error::Interrupted)), "a pre-cancelled fanout must interrupt the run, got {result:?}" @@ -2811,7 +2814,8 @@ async fn cancellation_while_suspended_in_a_fanout_arm_interrupts_the_run() { // exactly-once terminal contract holds on the cancellation path. The // 30-second answers and the timeout guard prove the aborted I/O is // never awaited. - use crate::cancel::{self, CancelHandle}; + use crate::cancel::CancelHandle; + use promptforge_core_support::cancel::scope; let gateway = ScriptedGateway::start(vec![resp_delayed_text( "too late", @@ -2845,7 +2849,7 @@ async fn cancellation_while_suspended_in_a_fanout_arm_interrupts_the_run() { let result = tokio::time::timeout( std::time::Duration::from_secs(10), - cancel::scope(cancel, async { + scope(cancel, async { Scheduler::new(&ctx, Some(gateway_client(gateway.addr()))) .drive() .await @@ -3166,7 +3170,8 @@ impl Tool for SignallingSlowTool { #[tokio::test(flavor = "current_thread", start_paused = true)] async fn cancellation_interrupts_a_slow_script_tools_call() { - use crate::cancel::{self, CancelHandle}; + use crate::cancel::CancelHandle; + use promptforge_core_support::cancel::scope; let md = "---\nname: t\ndescription: d\npromptforge: 0\n---\n\n\ # ToolCall\n\n\ @@ -3199,7 +3204,7 @@ async fn cancellation_interrupts_a_slow_script_tools_call() { }); let start = std::time::Instant::now(); - let result = cancel::scope(cancel, async { Scheduler::new(&ctx, None).drive().await }).await; + let result = scope(cancel, async { Scheduler::new(&ctx, None).drive().await }).await; assert!( matches!(result, Err(Error::Interrupted)), diff --git a/crates/promptforge-core/src/execute/tests/tool_loop.rs b/crates/promptforge-core/src/execute/tests/tool_loop.rs index 78e6c08cd..c6bccc947 100644 --- a/crates/promptforge-core/src/execute/tests/tool_loop.rs +++ b/crates/promptforge-core/src/execute/tests/tool_loop.rs @@ -1,6 +1,7 @@ use super::super::*; use super::*; -use crate::lua::{Compactor, OverflowReason}; +use crate::lua::OverflowReason; +use promptforge_lua::Compactor; /// Runs the standard echo fixture with the requested loop cap. async fn run_echo_loop(addr: SocketAddr, max_iterations: usize) -> Result { diff --git a/crates/promptforge-core/src/execute/tests/tool_scoping.rs b/crates/promptforge-core/src/execute/tests/tool_scoping.rs index b616f3a75..696a8993a 100644 --- a/crates/promptforge-core/src/execute/tests/tool_scoping.rs +++ b/crates/promptforge-core/src/execute/tests/tool_scoping.rs @@ -20,7 +20,7 @@ fn declared_tools_are_not_injected_without_always_or_add() { )], Vec::new(), ); - let runtime = Mutex::new(crate::lua::ToolRuntime { + let runtime = Mutex::new(promptforge_lua::ToolRuntime { added: Vec::new(), description_overrides: BTreeMap::new(), }); @@ -48,7 +48,7 @@ async fn always_advertises_concrete_schema_under_local_alias_and_dispatches_by_i )], vec!["local_alias".to_owned()], ); - let runtime = Mutex::new(crate::lua::ToolRuntime { + let runtime = Mutex::new(promptforge_lua::ToolRuntime { added: Vec::new(), description_overrides: BTreeMap::new(), }); @@ -195,13 +195,13 @@ fn near_duplicate_tools_are_valid_when_isolated_in_separate_scopes() { )); let mut first_binding = crate::lua::ToolBinding::for_test("first_local", "first", Arc::clone(&first)); - first_binding.conflicts.push(crate::lua::Conflict { + first_binding.conflicts.push(promptforge_lua::Conflict { alias: "second_local".to_owned(), similarity: 0.98, }); let mut second_binding = crate::lua::ToolBinding::for_test("second_local", "second", Arc::clone(&second)); - second_binding.conflicts.push(crate::lua::Conflict { + second_binding.conflicts.push(promptforge_lua::Conflict { alias: "first_local".to_owned(), similarity: 0.98, }); @@ -233,13 +233,13 @@ fn near_duplicate_always_scope_fails_at_the_scope_rebuild() { )); let mut first_binding = crate::lua::ToolBinding::for_test("first_local", "first", Arc::clone(&first)); - first_binding.conflicts.push(crate::lua::Conflict { + first_binding.conflicts.push(promptforge_lua::Conflict { alias: "second_local".to_owned(), similarity: 0.98, }); let mut second_binding = crate::lua::ToolBinding::for_test("second_local", "second", Arc::clone(&second)); - second_binding.conflicts.push(crate::lua::Conflict { + second_binding.conflicts.push(promptforge_lua::Conflict { alias: "first_local".to_owned(), similarity: 0.98, }); @@ -247,7 +247,7 @@ fn near_duplicate_always_scope_fails_at_the_scope_rebuild() { vec![first_binding, second_binding], vec!["first_local".to_owned(), "second_local".to_owned()], ); - let runtime = Mutex::new(crate::lua::ToolRuntime { + let runtime = Mutex::new(promptforge_lua::ToolRuntime { added: Vec::new(), description_overrides: BTreeMap::new(), }); @@ -279,13 +279,13 @@ fn near_duplicate_effective_scope_fails_before_the_model_without_payload_reports // half's alias and the picker's score. let mut first_binding = crate::lua::ToolBinding::for_test("first_local", "first", Arc::clone(&first)); - first_binding.conflicts.push(crate::lua::Conflict { + first_binding.conflicts.push(promptforge_lua::Conflict { alias: "second_local".to_owned(), similarity: 0.98, }); let mut second_binding = crate::lua::ToolBinding::for_test("second_local", "second", Arc::clone(&second)); - second_binding.conflicts.push(crate::lua::Conflict { + second_binding.conflicts.push(promptforge_lua::Conflict { alias: "first_local".to_owned(), similarity: 0.98, }); diff --git a/crates/promptforge-core/src/execute/tool_loop.rs b/crates/promptforge-core/src/execute/tool_loop.rs index 3888f5c0c..aeaf3614e 100644 --- a/crates/promptforge-core/src/execute/tool_loop.rs +++ b/crates/promptforge-core/src/execute/tool_loop.rs @@ -466,7 +466,7 @@ pub(crate) async fn run_prose_inference( prose: String, max_tool_iterations: usize, context: NonZeroU32, - compactor: Option, + compactor: Option, execution: &str, observer: &dyn Observer, section: &str, diff --git a/crates/promptforge-core/src/lua.rs b/crates/promptforge-core/src/lua.rs index 325d3a71c..de541b67e 100644 --- a/crates/promptforge-core/src/lua.rs +++ b/crates/promptforge-core/src/lua.rs @@ -21,16 +21,7 @@ pub(crate) use promptforge_lua::{ project_messages, resolve_model_binding, run_store_op, shim_live_h1_models, }; -// The typed compactor policy is read only by the tool loop's test-only -// prose wrapper; the production loop invokes compactor callbacks through -// `invoke_selected`. -#[cfg(test)] -pub(crate) use promptforge_lua::Compactor; - pub(crate) use promptforge_lua::ToolOutputKind; -#[cfg(test)] -pub(crate) use promptforge_lua::{Conflict, ToolRuntime}; - #[cfg(test)] mod coro_tests; diff --git a/crates/promptforge-core/src/lua/coro_tests.rs b/crates/promptforge-core/src/lua/coro_tests.rs index 71af09cb9..0f794c717 100644 --- a/crates/promptforge-core/src/lua/coro_tests.rs +++ b/crates/promptforge-core/src/lua/coro_tests.rs @@ -13,14 +13,16 @@ use serde_json::json; use promptforge_lua::Error; -use crate::cancel::{CancelHandle, scope}; +use crate::cancel::CancelHandle; use crate::execute::protocol::Request; use crate::execute::section_vm::{SectionVmSetup, VmSeed, setup_section_vm}; use crate::lua::{CoroStep, LuaBlockResult, LuaProgram, SectionVm, ToolBinding, ToolSet}; -use crate::model::{ModelBinding, ModelId, ModelInvocation, ModelSet}; +use crate::model::{ModelBinding, ModelId, ModelSet}; use crate::observe::{NullObserver, Observer}; use crate::tools::{Tool, ToolError, ToolId, ToolOutput}; use crate::untrusted::GuardNonce; +use promptforge_core_support::cancel::scope; +use promptforge_model_client::model::ModelInvocation; fn test_models() -> ModelSet { ModelSet { diff --git a/crates/promptforge-core/src/model.rs b/crates/promptforge-core/src/model.rs index 7ae6252da..16c67b570 100644 --- a/crates/promptforge-core/src/model.rs +++ b/crates/promptforge-core/src/model.rs @@ -13,8 +13,6 @@ //! re-exported here unchanged, so existing `promptforge_core::model::*` paths //! keep working. -#[cfg(test)] -pub(crate) use promptforge_model_client::model::ModelInvocation; pub use promptforge_model_client::model::{ CompletionError, CompletionErrorKind, CompletionOptions, ModelCatalog, ModelCatalogError, ModelDescriptor, ModelId, ModelIdError, TemperatureError, ThinkingMode, fetch_model_catalog, diff --git a/crates/promptforge-core/src/model/tests/mod.rs b/crates/promptforge-core/src/model/tests/mod.rs index 35f3cdcfc..06e8a3652 100644 --- a/crates/promptforge-core/src/model/tests/mod.rs +++ b/crates/promptforge-core/src/model/tests/mod.rs @@ -13,6 +13,7 @@ use crate::tools::ToolCatalog; use crate::untrusted::GuardNonce; use crate::{Error, Result}; use promptforge_model_client::Error as GatewayClientError; +use promptforge_model_client::model::ModelInvocation; use serde_json::json; const EXECUTION: &str = "model-bind-test"; diff --git a/crates/promptforge-lua/src/lib.rs b/crates/promptforge-lua/src/lib.rs index 44f244351..cd4cba162 100644 --- a/crates/promptforge-lua/src/lib.rs +++ b/crates/promptforge-lua/src/lib.rs @@ -102,8 +102,6 @@ mod tools; pub(crate) use tools::{LuaToolHandle, install_h2_tools, install_tool_call_counts}; mod vm; pub(crate) use vm::pack_sequence; -#[cfg(test)] -pub(crate) use vm::{LuaOutcome, run_chunk}; mod handles; mod live; mod messages; diff --git a/crates/promptforge-lua/src/tests.rs b/crates/promptforge-lua/src/tests.rs index 507dcb00b..dce7bf208 100644 --- a/crates/promptforge-lua/src/tests.rs +++ b/crates/promptforge-lua/src/tests.rs @@ -2,7 +2,7 @@ use std::sync::{Arc, Mutex}; use super::*; use crate::program::map_chunk_line_to_absolute; -use crate::vm::LocalTools; +use crate::vm::{LocalTools, LuaOutcome, run_chunk}; use promptforge_core_support::observe::{NullObserver, Observation}; use promptforge_store::Store; use promptforge_tools::{Tool, ToolError, ToolOutput}; diff --git a/crates/promptforge-lua/src/vm.rs b/crates/promptforge-lua/src/vm.rs index 7dc9a3ade..cd76b411d 100644 --- a/crates/promptforge-lua/src/vm.rs +++ b/crates/promptforge-lua/src/vm.rs @@ -1,5 +1,3 @@ -#[cfg(test)] -use super::LuaFanoutResult; use super::{ Access, Arc, AtomicU32, AtomicUsize, BTreeMap, DEFAULT_LUA_LOG_EVENTS, DEFAULT_LUA_MEMORY_BYTES, Error, Function, GuardNonce, InstructionBudget, IntoLuaMulti, Json, @@ -539,7 +537,7 @@ impl SectionVm { ) -> Result<()> where E: Fn(Value, Option, Json) -> std::result::Result + Send + 'static, - F: Fn(String, Vec, Json) -> std::result::Result, Error> + F: Fn(String, Vec, Json) -> std::result::Result, Error> + Send + 'static, L: Fn(String) -> std::result::Result, Error> + Send + 'static, diff --git a/crates/workshop-server/src/heartbeat.rs b/crates/workshop-server/src/heartbeat.rs index eeee63ef7..57b6b16ba 100644 --- a/crates/workshop-server/src/heartbeat.rs +++ b/crates/workshop-server/src/heartbeat.rs @@ -33,8 +33,6 @@ use std::time::Duration; use tokio::sync::{oneshot, watch}; use crate::backoff::ReconnectBackoff; -#[cfg(test)] -use crate::gateway::GatewayClient; use crate::gateway_binding::{GatewayBinding, GatewaySnapshot}; use crate::protocol::{Activity, Severity, StatusBarUpdate}; use crate::push::Push; @@ -343,6 +341,7 @@ async fn refresh_incomplete_sources( #[cfg(test)] mod tests { use super::*; + use crate::gateway::GatewayClient; use std::sync::Arc; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; diff --git a/crates/workshop-server/src/session_agents.rs b/crates/workshop-server/src/session_agents.rs index 76fdb65d3..10cb8b485 100644 --- a/crates/workshop-server/src/session_agents.rs +++ b/crates/workshop-server/src/session_agents.rs @@ -40,8 +40,6 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use promptforge_core_support::cancel::CancelHandle; use promptforge_core_support::events::{CallMetrics, RuntimeEventKind, ToolCallEvent}; use promptforge_core_support::observe::{Observation, Observer}; -#[cfg(test)] -use promptforge_model_client::client::GatewayClient as ModelClient; use promptforge_model_client::client::StreamDelta; use promptforge_model_client::model::{ModelCatalog, ModelDescriptor, ModelId, ThinkingMode}; use tokio::sync::{broadcast, mpsc}; @@ -731,7 +729,10 @@ fn fresh_session_id() -> String { /// [`LaunchRefusal::GatewayUnusable`] - when the key is empty (the model /// client refuses blank credentials) or the URL does not parse. #[cfg(test)] -fn model_client(base_url: &str, api_key: &str) -> Option { +fn model_client( + base_url: &str, + api_key: &str, +) -> Option { crate::gateway_binding::model_client(base_url, api_key) } diff --git a/vibe/2026-09-12-1-test-namespace-vfs-debt.md b/vibe/2026-09-12-1-test-namespace-vfs-debt.md index 7c381ce8e..98875c3c8 100644 --- a/vibe/2026-09-12-1-test-namespace-vfs-debt.md +++ b/vibe/2026-09-12-1-test-namespace-vfs-debt.md @@ -128,7 +128,7 @@ Each step is one commit containing its code and tests. -### Step 3: remove test-only re-exports workspace-wide and relocate test imports +### Step 3: remove test-only re-exports workspace-wide and relocate test imports [completed] - Component: cleanup - One behavior slice across five crates: delete every `#[cfg(test)]` import or re-export whose only consumers are test modules, and let each consumer import the name directly from its real home. The `execute.rs` and `execute/tests/mod.rs` portion is already applied in the worktree (coded before the re-decomposition); this step completes the remaining files and lands the whole slice as one commit. From a4c7876152dbf6629e20c4e772f19457840ba545 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 02:36:25 -0700 Subject: [PATCH 15/26] Remove the two tracked vibe ledger files Deletes the two tracked ledger files that recorded accumulated run history, one at the repository root and one under the run directory. That history is no longer kept in version control. - `vibe-ledger.md` - Deleted from the repository root; it held the accumulated run ledger. - `vibe/vibe-ledger.md` - Deleted; it held the same kind of accumulated run ledger under the run directory. Plan: vibe/2026-09-12-1-test-namespace-vfs-debt.md --- vibe-ledger.md | 89 ------------- vibe/2026-09-12-1-test-namespace-vfs-debt.md | 5 +- vibe/vibe-ledger.md | 131 ------------------- 3 files changed, 3 insertions(+), 222 deletions(-) delete mode 100644 vibe-ledger.md delete mode 100644 vibe/vibe-ledger.md diff --git a/vibe-ledger.md b/vibe-ledger.md deleted file mode 100644 index 418304342..000000000 --- a/vibe-ledger.md +++ /dev/null @@ -1,89 +0,0 @@ -# Vibe Ledger - -- Step 1 baseline - Clean detached HEAD `0d534eeb8ffed7370316f2950a39ba0e89cda9ea`; review anchor `84b2c9261f96642bb3fa02836d4e98b13cde8208` is not an ancestor; merge base `0c1e42cbb7ad1ebaaf702ec8265d1389749eb97f`. -- Step 1 Rust baseline - `cargo test -p gateway-config --lib; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p gateway-logging --lib; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p gateway-stt --test it batch; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p promptforge-core --test suite shipped; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p promptforge-webfetch --lib; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p shared-loopback --lib; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p shared-sidecar --test it; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p workshop-server --test it realtime_relay; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p workshop --lib` - preceding suites passed; Workshop has no library target, exit 101. -- Step 1 UI baseline - `node --test test/stt-stream.mjs test/take-registry.mjs` - 15 passed, 0 failed. -- Step 1 Workshop baseline - `cargo test -p workshop --bin promptforge-workshop gateway` - blocked because `binaries\promptforge-gateway-x86_64-pc-windows-msvc.exe` was not staged. -- Step 1 post-fix verification - `cargo build -p gateway`; `git diff --check HEAD^ HEAD` - pass. Decisions made alone: none. -- Step 2: Correct the cross-platform Gateway build helper - `clippy-driver crates/gateway/build.rs --crate-name gateway_build_script_build --edition=2024 --target=x86_64-unknown-linux-gnu --emit=metadata -o "$env:TEMP\gateway-build-script-linux.rmeta" -D warnings -W clippy::unnecessary_wraps` - pass; `cargo test -p gateway --test it icon::the_exe_carries_every_image_of_the_program_icon` - 1 passed, 103 filtered. Decisions made alone: none. -- Step 3: Replace Rust structural proxies with direct behavior evidence - `cargo build -p gateway`; `cargo test -p gateway-stt --test it architecture::; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p gateway-stt --test it generation::; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p gateway -F test-fixtures determinate_persistence_failure; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p gateway --test it profiles::; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p product-integration-tests real_model_client_completes_through_gateway; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p gateway-logging; if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }; cargo test -p promptforge-core --test suite shipped` - pass. -- Step 3 decision - Cross-product model-client compatibility lives in boundary-neutral `product-integration-tests` so neither product acquires a prohibited development dependency. Falsifier: an existing neutral test owner can carry the same real Gateway and PromptForge client coverage without crossing an approved product boundary. -- Step 4 focused verification - Both UI typecheck, production build, and test suites passed with 88 and 125 tests; JavaScript tools passed 21 tests; retained Rust behavior and compiler checks passed; `cargo tauri build --debug --bundles nsis --config tauri.nightly.conf.json` produced the unsigned NSIS installer. -- Step 4 component verification - `cargo build -p gateway`; `cargo test -p gateway-stt --test it architecture::`; `cargo test -p promptforge-core --test suite shipped`; `cargo test -p product-integration-tests real_model_client_completes_through_gateway`; Workshop UI `npm run typecheck`, `npm run build`, `npm test`; config UI `npm run typecheck`, `npm run build`, `npm test` - pass. Decisions made alone: none. -- Step 5: Preserve logging while removing internal duplication - `cargo build -p gateway-logging`; `cargo test -p gateway-logging` - 59 passed, 1 ignored; 11 doctests passed. Review fixes restored the production helper, surfaced synchronization failures, and removed the exact checkpoint allowlist. Decisions made alone: none. -- Step 6: Establish process-lifetime Gateway ownership - `cargo build -p gateway`; `cargo test -p gateway-logging`; `cargo test -p shared-sidecar`; `cargo test -p gateway relaunch::tests`; `cargo test -p gateway --bin promptforge-gateway`; `cargo test -p gateway --test it boot::`; `cargo test -p workshop-server a_named_fixture_releases_its_process_lease_when_terminated` - pass. Review fixes enforced one ownership deadline, preserved resolution error sources, and added deterministic process rendezvous. -- Step 6 decision - The process-lifetime lease uses dedicated run-directory file `gateway.instance.lock`, separate from `gateway.json.lock`. Falsifier: the name conflicts with an existing operator-managed path or cannot preserve independent parent and child lock ownership on a supported platform. -- Step 6 test-boundary correction - `cargo build -p gateway`; `cargo test -p gateway --test it the_default_binary_ignores_test_rendezvous_environment`; `cargo test --locked -p gateway --no-default-features --features test-fixtures --test it boot::`; `cargo test --locked -p shared-sidecar a_process_lifetime_lease_recovers_after_its_owner_is_terminated`; `cargo test -p gateway-logging`; parse `.github/workflows/ci.yml` with PyYAML - pass. -- Step 6 decision - Compile deterministic process rendezvous only with the existing `test-fixtures` feature and run the focused races in both Workshop platform jobs. Falsifier: the default binary can observe either rendezvous variable or a platform job cannot execute the feature-enabled race target. -- Step 7: Carry connection generation through Workshop dictation - Workshop UI `npm run build`, `npm run typecheck`, `node test/stt-stream.mjs`, `node test/take-registry.mjs`, `node test/take-registry-regressions.mjs`, and `node test/agent-stt.mjs` - pass. Review fix made every stale-generation input case nonvacuous. Decisions made alone: none. -- Step 8: Close publication and bound the Gateway lifecycle - `cargo build -p workshop-server`; `cargo test -p workshop-server gateway_binding`; `cargo test -p workshop-server serve::tests`; `cargo test -p workshop gateway::tests`; `cargo test -p workshop every_supervisor_outcome_continues_server_teardown`; `cargo test -p shared-sidecar cancellation`; `cargo test -p shared-sidecar shutdown`; Workshop UI `npm run typecheck`, `npm run build`, `node test/stt-stream.mjs`, `node test/take-registry.mjs`, `node test/take-registry-regressions.mjs`, and `node test/agent-stt.mjs` - pass. Review fixes preserved permanent closure, identity-authenticated recovery, private bounded stopping, and teardown continuation. Decisions made alone: none. -- Step 8 completion coverage - `cargo build -p workshop-server`; `cargo test -p workshop-server gateway_binding::tests`; `cargo test -p workshop-server serve::tests`; `cargo test -p workshop gateway::tests::boot`; `cargo test -p workshop gateway::tests::recovery`; `cargo test -p workshop gateway::tests::shutdown`; `cargo test -p workshop every_supervisor_outcome_continues_server_teardown`; `cargo test -p shared-sidecar shutdown`; Workshop UI typecheck, build, `stt-stream.mjs`, and `take-registry.mjs` - pass. Atomic close ordering and boot ownership fixtures are present; stop-bridge construction failures propagate under the supervisor deadline. -- Step 9 external prerequisite - [STT Miri run 34283748222](https://github.com/vinniefalco/promptforge/actions/runs/34283748222) passed under `NT AUTHORITY\NETWORK SERVICE`; `PROMPTFORGE_RUST_1_89_0_BIN` supplied the absolute directory and both direct tools reported Rust 1.89.0. -- Step 9: Migrate the native runner to one PowerShell contract - parse `tools/validate-rust-1.89.0.ps1`; `node --test tools/check-stt-native-workflow.test.mjs`; parse `.github/workflows/stt-miri.yml` with PyYAML - pass, 8 fixture tests. Review fix validates cargo and rustc provenance symmetrically. Decisions made alone: none. -- Step 10 focused package validation - `cargo metadata --locked --no-deps --format-version 1`; `cargo check --locked -p build-workshop`; `cargo test --locked -p build-workshop`; `cargo build --locked -p gateway --no-default-features`; `cargo tauri build --debug --bundles nsis --config tauri.nightly.conf.json`; parse release workflow with PyYAML - pass. Workspace metadata contains 34 version `0.3.0` packages, Gateway reports `0.3.0`, build-workshop passed 16 tests, and the unsigned package is `PromptForge_0.3.0_x64-setup.exe`. -- Step 10 review fixes - Installed Gateway and Workshop version checks compare complete expected output; the cumulative queue finding was rejected because commit-message history proves the authorized queue pipeline wrote N71. -- Step 10 Clippy recovery - Complete stable `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings` passed after one operator-authorized batch fixed seven remaining diagnostics. Focused profile, rollback, tray, formatting, and whitespace checks passed. -- Step 10 runner regression - Direct rustup-proxy probes now run in redirected child processes; realistic stderr fixtures reproduce Windows PowerShell native-command promotion. `node --test tools/check-stt-native-workflow.test.mjs` passed 8 tests and script parsing passed. -- Step 10 local suite remainder - Stable Gateway build; all-feature non-Workshop workspace tests; doctests; warnings-denied docs; featureless Gateway; Workshop UI typecheck, build, and tests; config UI typecheck, build, and tests - pass. -- Step 10 history attachment - Replayed the ten new commits onto finalized `master` and preserved both prior lines at `safety/remove-range-debt-detached-20260908-1637`, `safety/remove-range-debt-rebased-20260908-1640`, and `safety/pre-reattach-master-20260908-1637`. Finalized prior `master` remains the direct ancestor. -- Step 10 decision - The operator overrode the exhausted one-error-at-a-time verification cap and required all remaining Clippy diagnostics to be fixed as one batch. Falsifier: a repeated signature indicates a design failure or any focused behavior test regresses. -- Step 10 exact-SHA runner attempt - [STT Miri run 34293081202](https://github.com/vinniefalco/promptforge/actions/runs/34293081202): pure Miri passed; native Whisper failed because direct cargo and rustc handled the `+toolchain` proxy probe differently despite both direct executables reporting 1.89.0. -- Step 10 runner correction - Removed proxy inference from the explicit directory contract and added a regression proving neither executable receives a `+toolchain` argument. `node --test tools/check-stt-native-workflow.test.mjs` passed 7 tests and PowerShell parsing passed. Falsifier: the service-account job cannot validate the same regular files and exact versions without provenance inference. -- Step 10 second exact-SHA attempt - [CI run 34293854314](https://github.com/vinniefalco/promptforge/actions/runs/34293854314) reached one Linux-only Clippy failure; [STT Miri run 34293854362](https://github.com/vinniefalco/promptforge/actions/runs/34293854362) passed pure Miri and the corrected native validator before cancellation; [package run 34293865997](https://github.com/vinniefalco/promptforge/actions/runs/34293865997) was cancelled when the SHA became stale. -- Step 10 Linux correction - `sync_parent_directory` and its call compile only on Unix, removing the non-Unix unfulfilled `clippy::unnecessary_wraps` expectation. Source-specific Linux Clippy, Gateway Clippy, formatting, diff checks, and both focused persistence tests passed. Falsifier: the Linux `check` job still reports the expectation or a persistence behavior test fails. -- Step 10 third exact-SHA attempt - [STT Miri run 34294975958](https://github.com/vinniefalco/promptforge/actions/runs/34294975958) passed pure Miri and native Whisper completely; [CI run 34294975972](https://github.com/vinniefalco/promptforge/actions/runs/34294975972) reached Workshop fixture Clippy failures; [package run 34294986291](https://github.com/vinniefalco/promptforge/actions/runs/34294986291) completed all builds before cancellation when the SHA became stale. -- Step 10 Workshop fixture correction - Moved the existing test-only `expect_used` expectation onto the helper containing nine intentional panic assertions and removed the unfulfilled outer expectation. Focused fixture tests, focused Clippy, and the full staged `check-workshop` Clippy command passed. Falsifier: the Windows Workshop Clippy job still reports either expectation class. -- Step 10 fourth exact-SHA attempt - [CI run 34296875273](https://github.com/vinniefalco/promptforge/actions/runs/34296875273) and [STT Miri run 34296875516](https://github.com/vinniefalco/promptforge/actions/runs/34296875516) passed completely. [Package run 34296884049](https://github.com/vinniefalco/promptforge/actions/runs/34296884049) built and version-checked all five `0.3.0` artifacts, then every clean-machine runtime check failed because the smoke configuration had no selected profile. -- Step 10 package boot correction - Added one shared package-smoke `gateway.toml` with an empty `main` profile and `gateway.state.toml` selecting it; all platform jobs install those files before launching the sibling Gateway. The real-process regression failed before the fixture correction and then passed; Gateway boot passed 16 tests, Workshop boot passed 14 tests, shared-sidecar passed, and workflow YAML, formatting, and whitespace checks passed. Falsifier: a clean package cannot publish a validated Gateway connection without CLI profile arguments. -- Step 10 shutdown fixture correction - Replaced arbitrary shutdown-observation waits with named 10-second fixture phase timeout `FIXTURE_PHASE_TIMEOUT`. The flaky regression passed 20 consecutive runs and all 12 Gateway binding tests passed. Falsifier: accepted shutdown still fails to report under loaded CI scheduling. -- Step 10 plan contract migration - Added the six required plan ranges, complete project-survey status, and ten step ranges after the live execution protocol changed. No implementation contract changed. -- Step 10 fifth exact-SHA attempt - [STT Miri run 34300076435](https://github.com/vinniefalco/promptforge/actions/runs/34300076435) passed completely. [CI run 34300076420](https://github.com/vinniefalco/promptforge/actions/runs/34300076420) exposed one nondeterministic finalized-snapshot test. [Package run 34300084519](https://github.com/vinniefalco/promptforge/actions/runs/34300084519) passed both macOS clean-machine tests but Windows and both Linux jobs used ambiguous process discovery and never found the Workshop listener. -- Step 10 finalized-snapshot correction - Added a two-phase channel rendezvous that releases the writer only after the reader holds the finalized lock. The exact test passed 100 consecutive runs and the Gateway STT unit suite passed 122 tests with 1 ignored. Falsifier: a snapshot can still combine text and sample ownership from different finalization generations. -- Step 10 package listener correction - Windows tracks the exact `Start-Process` handle and retries all of its current listeners; Linux records the exact Workshop child PID through `xvfb-run` and retries that process's listeners. Early exits report captured output. Workflow YAML passed, Windows installed-package smoke reached the validated page, and `app.js` measured 2,640,781 bytes. Falsifier: a clean-machine job selects a wrapper, shell, stale process, or unvalidated first listener. -- Step 10 sixth exact-SHA attempt - [CI run 34301735868](https://github.com/vinniefalco/promptforge/actions/runs/34301735868) and [STT Miri run 34301735843](https://github.com/vinniefalco/promptforge/actions/runs/34301735843) passed completely. [Package run 34301744700](https://github.com/vinniefalco/promptforge/actions/runs/34301744700) passed macOS ARM, macOS Intel, Linux x64, and Linux ARM; Windows failed because its hosted runner job object denied `CREATE_BREAKAWAY_FROM_JOB`. -- Step 10 Windows package correction - Windows Gateway launch retries without `CREATE_BREAKAWAY_FROM_JOB` only after PermissionDenied, retaining `CREATE_NEW_PROCESS_GROUP` and `DETACHED_PROCESS`; success and other errors do not retry. Four exact flag tests, 18 Workshop boot tests, Clippy, formatting, and an installed debug NSIS smoke passed. Falsifier: the restricted hosted runner still cannot launch the packaged sibling or an unrelated spawn failure triggers fallback. -- Step 10 seventh exact-SHA attempt - [STT Miri run 34303902087](https://github.com/vinniefalco/promptforge/actions/runs/34303902087) and [package run 34303912978](https://github.com/vinniefalco/promptforge/actions/runs/34303912978) passed completely across native STT and all five package platforms. [CI run 34303902072](https://github.com/vinniefalco/promptforge/actions/runs/34303902072) passed every job except one flaky positive cleanup observation in the Windows Workshop boot suite. -- Step 10 boot cleanup correction - Added named 10-second positive fixture phase timeout in `crates/workshop/src/gateway/tests/boot.rs` while preserving the 100 ms negative observation. The exact test passed 10 consecutive runs, all 18 boot tests passed, and formatting and lints passed. Falsifier: authenticated unpublished-child cleanup still misses its marker under loaded Windows scheduling. - -- Async STT boot reset Step 1: Rename Gateway discovery - `cargo test -p shared-sidecar` (68 unit + 9 integration + 6 doc), focused Gateway diagnostics/relaunch/runner/sidecar/boot tests, Workshop gateway tests (43), Workshop Server resolve tests (12) - pass. Review fix swept remaining "connection file" prose to "gateway discovery file" across shared-sidecar, shared-loopback, gateway, workshop, workshop-server, and guides. Decision: kept diagnostics JSON key `connection_file` as a tested serialized contract; renamed only Rust types and prose. Falsifier: a later step or reviewer explicitly renames the diagnostics key contract. - -- Async STT boot reset Step 2: Make the pending deque authoritative - `cargo test -p gateway commands::` (24 passed), clippy clean. Review fix merged pop and activate into one critical section via `begin_next` so shutdown cannot slip between them. Decision: worker exits only via shutdown; notify after releasing the queue lock. Falsifier: a non-shutdown path must stop the worker, or measured wake latency traces to notify ordering. - -- Async STT boot reset Step 3: Build one-time speech publication - `cargo test -p gateway-stt` (122 unit + 63 integration), clippy and fmt clean, gateway crate compiles. Decision: one attempt is spent on any outcome (success, failure, cancellation, panic) so speech stays unavailable until restart; compat replacement APIs stay until Step 4. Falsifier: a retained test shows a failed boot load retried in-process, or Step 4 finds replacement callers beyond profile switch, Apply, runner, and gateway speech tests. - -- Async STT boot reset Step 4: Move STT to boot only and delete replacement - `cargo test -p gateway --lib` (283), `--test it` (103), `--bin` (34+3), `--doc` (4), `cargo test -p gateway-stt` (121+52), `workshop-server realtime_relay` (10), feature-disabled check, workspace clippy, fmt - pass. Decision: boot flag preserved through debounce attach and supersession; STT attempt follows both full commit and PartialStart; `loading-speech` leaf registers before the switch tree; replacement module renamed to `admission.rs`; realtime 1012 `engine_replaced` close deleted. Falsifier: a client expecting 1012 on shutdown, or a boot with partially failed local start loading no speech. - -- Async STT boot reset Step 5: Notify the browser when STT needs restart - Config UI typecheck, build, and complete suite (142 passed). Decision: toast kind `info`; catalog order ignored; membership filtered through each document's own catalog; combined process-owned plus STT Apply shows both the existing banner and one STT toast. Falsifier: reordering `[[stt_model]]` changes what boot loads, or a profile naming a non-STT model is a speech change. - -- Async STT boot reset Step 6: Reconcile documentation and qualify - Full verification passed: build, fmt, warnings-denied clippy (portable and staged Workshop), portable workspace and doc tests, feature-disabled Gateway, both UI suites, staged Workshop tests with guaranteed cleanup, both Miri lanes, all five native Whisper lanes with hash-pinned fixtures, guide assembler with zero drift, mdBook build, and `cargo workshop` package construction. Operator confirmed physical microphone hypothesis revision, authoritative completion, and second take on the release binaries. - -- Rulebook debt tiers Step 1: CI doctest coverage and dependency hygiene - component-scope verify pass: build, `cargo fmt --all --check`, warnings-denied clippy, and `cargo test --locked -p gateway-stt` (121 unit + 52 integration) green; nextest unavailable locally, cargo test fallback recorded in verify-step-1-round-1.log. Decision: `ci-green` treats cancelled as failure via an `if: always()` loop over `needs.*.result` | Falsifier: a cancelled or newly added job shows green as a required status check. Decision: doctest step placed in `check-workshop` (Windows) rather than the Linux `test` job | Falsifier: workshop doctests run twice or not at all in CI. - -- Rulebook debt tiers Step 2: Non-blocking destructors, restored error causes, bounded supervisor channel - component-scope verify pass: gateway and workshop builds, `cargo fmt --all --check`, both clippy invocations, nextest across the four touched crates, and the gateway-stt integration suite green (verify-step-2-round-1.log). Decision: `OperatorCancellation` is the one loss-tolerant event and alone rides `mpsc::channel(1)`; `AcceptedInput`/`TerminalSettlement`/`Close` keep guaranteed delivery on the unbounded queue | Falsifier: a reducer transition that awaits an operator cancellation. Decision: `RecoveryCandidate::shutdown` disarms the drop signal even on failure since the caller owns the outcome | Falsifier: a failure class where a silent drop retry recovers the child. Decision: `SessionError` dropped `Eq/PartialEq` because `TranscribeError` is not `PartialEq`; test comparisons became `matches!` | Falsifier: a production `SessionError` equality comparison. Decision: reject cargo-nextest install in the fix round, accept the passing cargo test fallback | Falsifier: a later round requires nextest-specific behavior. - -- Rulebook debt tiers Step 3: non_exhaustive attributes and expect conversions - component-scope verify pass: builds, `cargo fmt --all --check`, both clippy gates, and nextest across the touched crates green, 1925 tests, zero failures (verify-step-3-round-1.log). Decision: wildcard arms on downstream `DecodeMode` matches - `unreachable!` in test factories, `_ => None` fallbacks in initial_load.rs and backend-whisper model.rs | Falsifier: a third variant is added and the chosen fallback proves wrong. Decision: gateway bin `main.rs` wildcard exits `FAILURE` on an unrecognized future `GatewayStartup` variant | Falsifier: a new variant needs serving behavior there. Decision: 8 suppressions were stale (lint never fires) and deleted rather than converted | Falsifier: the `-D warnings` gate, which re-verified each. Decision: cfg-dependent sites use `#[cfg_attr(not(test), expect(...))]` | Falsifier: the gate. Decision: `app.rs` keeps its `#[allow]` per its in-code comment (expectation unfulfilled in some cfg permutations) | Falsifier: clippy behavior changes. - -- Rulebook debt tiers Step 4: Paused-time conversion for in-process async tests - component-scope verify pass: build, `cargo fmt --all --check`, clippy `-D warnings`, and nextest across promptforge-core, promptforge-lua, and gateway-stt-engine green (verify-step-4-round-1.log); converted tests proven deterministic over 60 repeated runs. Decision: `cancellation_interrupts_a_pending_input_wait` switched from `multi_thread` to `current_thread`, which tokio requires for `start_paused` | Falsifier: the cancel-during-pending-wait assertion is flavor-independent and passes 20/20. Decision: relied on tokio's idle auto-advance rather than explicit `advance()` calls, since the sleeps live in spawned canceller/tool tasks | Falsifier: 60/60 green repetitions. Decision: the gateway-stt-engine rendezvous test's late arrival was restructured to start after the rendezvous times out rather than relying on a timer race, because a paused clock auto-advances into the rendezvous window | Falsifier: tokio's paused runtime shown not to auto-advance while a `spawn_blocking` condvar wait is outstanding. - -- Rulebook debt tiers Step 5: FixtureError and typed sources in the take/finalization pipeline - focused verification: `cargo test -p gateway-stt -F test-fixtures` (125 lib + 53 integration, 0 failed), clippy `-D warnings`, fmt, and feature-off check green; review clean; no Verify dispatch (mid-component step with no fixes). Decision: take failure stored and shared as `Arc` since `pending_failure` clones it out of the mutex for session gating | Falsifier: if the failure were only ever moved, plain `TakeFailure` channels suffice. Decision: `PendingPrecommitFailure` carries `Arc` without `#[source]`; thiserror 2.0.19 `AsDynError` has no `Arc` impl | Falsifier: a thiserror release supporting Arc sources. Decision: `FixtureError` boxes pub(crate) sources as `Box`; `SpeechError`/`serde_json::Error` carried concretely | Falsifier: making `SessionError`/`RegisterError`/`ClientError` public. Decision: `ItemFailure` stays String-carrying (Clone+Eq wire terminal) but is classified from the typed `TakeFailure` | Falsifier: the wire protocol gains typed failure codes. - -- Rulebook debt tiers Step 6: anyhow for test and build code, typed errors in remaining production - FULL gate pass: build, fmt, clippy (pinned and newer stable), docs, full workspace tests, workshop nextest, and doctests all green (verify-step-6-round-1.log); review clean including the cumulative component diff. Decision: build-ui uses anyhow, not a new thiserror type | Falsifier: the contract allows exactly one new public error type (FixtureError) and routes build tooling to anyhow. Decision: menu.rs gains private `SwitchFailure` with Display byte-identical to the old strings | Falsifier: no existing workshop-server enum models switch lifecycle. Decision: dialect.rs gains private `ToolCallRejection` with Display matching the old wire warnings exactly | Falsifier: the reasons become `gateway_warning` wire strings, so the text had to survive. Decision: confine.rs `parse_whoami_user_sid` returns `LocalError::CacheNotPrivate` directly | Falsifier: keeps the existing variant and message while dropping the String channel. Decision: the two LazyLock statics hold `SharedSource`, replayed via a new `Error::shared` helper | Falsifier: SharedSource exists precisely for re-producing typed errors from non-Clone caches. - -- Unify toolchain Step 1: Unify toolchain, fix the three CI failures, and remove MSRV machinery - focused verify pass: `cargo build` and `cargo nextest run -p gateway-stt-engine --all-features decode_rendezvous_timeout` green (verify-step-1-round-1.log); both clippy scopes and fmt clean during coding. Decision: rewrote the `rust-toolchain.toml` comment, which described the removed MSRV pin | Falsifier: the old comment claims a pin and CI overrides that no longer exist. Decision: left "Rust 1.89 or later" in five crate README.md files untouched (outside the step's enumerated files) | Falsifier: plan step or contract names them. Decision: split four out-of-scope `Duration::from_mins`/`from_hours` hunks into a separate commit (`c1ef217f`) rather than amending the step to name them | Falsifier: the plan file is treated as amendable after review, making the split commit redundant history. - -- Unify toolchain Step 2: Full verification gate and CI confirmation - local gate run and the fixes pushed to origin/master (`23ffa642`); the `native-whisper` self-hosted lane failed on provisioning (the runner's NetworkService rustup home had only 1.89), fixed by junctioning its `stable-x86_64-pc-windows-msvc` toolchain directory to the operator's profile copy with a read ACL, so the runner tracks the operator's stable from now on. The operator closed the run assuming the in-flight CI passes rather than watching it. Falsifier: the CI run on `23ffa642`. - -## 2026-09-07-2-gateway-tts-phase-1 - -- Run re-seeded 2026-09-09 on branch `add-tts-phase-1` (rewrite base `5edcb3c9`, current cppalliance master). Plan: `vibe/2026-09-07-2-gateway-tts-phase-1.md` (corrected per the maintainer review of PR #21 before the rewrite; the correction commit lives on `backup/tts-phase-1-pre-rewrite`). This run rewrites the original 2026-09-07 run's history; the original series is preserved on the backup branch and under `d:\_cppalliance\promptforge-backups\2026-09-09-tts-rewrite\`. -- Decision (run-level): verify cadence. Every step gets a Verify dispatch running the build plus that step's focused tests; the full CI-exact gate suite runs at component ends (steps 2, 4, 6, 7, and 9) and after any fix round that changes the commit. Falsifier: a step lands red that its focused run could not catch, which indicts the cadence rather than the step. -- Decision (run-level): decomposition not re-run. The plan's steps are carried from the original run's executed plan with the review amendments folded in; the Run Mode defect pass re-validated ordering and ambiguity instead of a decomposition rewrite, which would have risked dropping the amendments. Falsifier: a step that cannot name what it receives from earlier steps; none found. -- Step 1 (Speech model kind and voices capability): `cargo build` + `cargo test -p gateway-config` green at `acfaa939` (log `vibe/verify-step-1.log`); config-UI npm gates (typecheck, build, 126 tests) green. Review: clean, 0 findings. Amendment beyond the original diff: `speech` added to both `kind()` doc comments in `accessors.rs` (review F12). Decisions made alone: none. -- Step 2 (launch_options goes fallible and refuses unknown kinds in gateway-local): component-ending verify at `eb7d3f25` green — fmt, clippy `-D warnings`, workspace tests + doctests, headless check, `cargo doc -D warnings`, `cargo deny` (0.20.2, no waiver), config-UI npm chain (log `vibe/verify-step-2.log`). Review: 1 Minor (`start_impl` provisioned the shared server before the per-model kind preflight), fixed by hoisting the preflight so the server is provisioned only when a launchable model exists, closed. Amendment beyond the original diff: the A6 preflight (`serve_mode_for` before any side effect in `start_impl` and `provision_artifacts_impl`) plus its no-side-effect tests (review F1). Decisions made alone: none. -- Step 3 (SpeechRequest wire type): `cargo build` + `cargo test -p shared-protocol` green at `b6cbf9d9` (log `vibe/verify-step-3.log`). Review: 1 Minor (the 4096-char cap table exercised only ASCII, so a byte-counting regression would pass untested), fixed with multi-byte boundary rows (4,096 x U+00E9 accepted at 8,192 bytes, 4,097 rejected), closed. Amendment beyond the original diff: the closed-format rejection test now pins Together's `raw` (review F8). Decisions made alone: none. -- Step 4 (Speech upstream and audio streaming client): component-ending verify at `e59f051a` green — fmt, clippy `-D warnings`, workspace tests + doctests, headless check, `cargo doc -D warnings`, `cargo deny` (manifests touched, so it really ran) (log `vibe/verify-step-4.log`). Review: 1 Important (the first-response deadline was untested — removing it kept all tests green), fixed with a cfg(test)-scaled budget and a stalled-headers rejection test, closed. Amendment beyond the original diff: the F4 deadline split (first-response ~120 s vs per-read body idle 30 s). Decision (made alone): both deadlines live in `send_speech`, not on the client, because reqwest 0.12 arms `read_timeout` during the header wait (`PendingRequest::poll`); the plan's Technical Design, Decision Record, and step text were revised in the same change naming the forcing behavior, per the plan's execution rule. Falsifier: a reqwest release that stops arming `read_timeout` during the header wait, which would let the idle budget move back onto the client. -- Step 5 (POST /v1/audio/speech route): focused verify green against the tree committed at `3cefa207` - fmt, clippy `-D warnings` (gateway, all-targets/all-features), full `cargo test -p gateway` (449 passed, 0 failed: 134 it + lib + doctests), warnings-denied docs over gateway/gateway-routing/shared-protocol, featureless `cargo check -p gateway --no-default-features`. Review: 3 findings (1 Important, 2 Minor), all closed: the vacuous permit-held guard became a bounded negative wait (`ADMISSION_GRACE` 100 ms, sized under the relay's scaled 200 ms upstream-idle budget, red-verified by dropping the permit at relay start); the queued-race `tokio::select!` is wrapped in `PHASE_TIMEOUT`; boot.rs's `not(test-fixtures)` rendezvous test records its retirement in its comment. Amendments beyond the original diff: the F2 bounded background relay (spawned task owning the upstream body, dominion permit, and cancellation guard; bounded channel; four named cfg(test)-scaled bounds; every terminal path emits one Err item and releases the permit), F3 cancellation emits a synthesized body error instead of clean EOF, F5 residual SSE-aware MIME fallback, F8 route pins (unknown-model 404 envelope, profile-switch mid-stream truncation, `` passthrough, `stream_format:"sse"` forwarding), F12 kind-list doc comments. Decision (made alone): a self dev-dependency (`gateway = { path = ".", features = ["test-fixtures"], default-features = false }`) lets the it harness drive the scaled relay bounds, retiring boot.rs's `not(test-fixtures)` test from its last runner (recorded in the test's comment). Falsifier: a gateway test invocation without `test-fixtures` reaches CI or local documentation, or the self dev-dep is removed, either of which re-enables the boot.rs test. -- Step 6 (GET /v1/audio/voices route): component-ending verify green against the tree committed at `6bded8ca` - fmt, clippy `-D warnings` (same workshop/workshop-server exclude substitution as steps 2 and 4, for the pre-existing missing Tauri sidecar binary), workspace tests + doctests (2843 + 275, 0 failures, 36 pre-existing env-gated ignores), featureless gateway check, warnings-denied docs, cargo deny, config-UI npm chain 126/126 (log `vibe/verify-step-6.log`). Review: clean, 0 findings. Amendment beyond the original diff: the unauthenticated-401 test on the voices route (review F8), red-verified against a commented-out `check_auth`. Decisions made alone: none. -- Step 7 (Live gateway speech probe and verification note): component-ending verify green against the tree committed at `aa138792` - fmt, clippy `-D warnings` (same workshop/workshop-server exclude substitution as steps 2/4/6), workspace tests + doctests (983 + 275; one gateway-logging timing assertion failed the first parallel run and passed isolated and full-package reruns - a flake, this commit carries zero Rust changes), featureless gateway check, warnings-denied docs, cargo deny, config-UI npm chain 126/126, `node --check` and `node --test tools/gateway-tts-live.test.mjs` 9/9 (log `vibe/verify-step-7.log`). Review: clean, 0 findings. Fresh work replacing the original run's parity step: the Python script never enters this history; the probe is the zero-dependency Node script `tools/gateway-tts-live.mjs` with paired offline tests, process-environment key ferrying only (F6/A19), always-builds-before-boot (F7), and the rewritten gateway-only verification note with Phase 3 provenance placeholders (F9/F10). Decisions made alone: none. -- Step 8 (Documentation): focused verify green against the tree committed at `447ba9e5` - fmt, `cargo run -p build-user-guide` byte-stable regen, F5/MIME-fallback inspection of README + speech chapter + export (log `vibe/verify-step-8.log`). Review: 1 Minor (the Content-Type fallback clause omitted the code's SSE-first `text/event-stream` path), fixed in the chapter, README, and regenerated export, closed. Amendments beyond the original diff: the F5 SSE wording (Together SSE is unrequestable because `raw` is rejected at the wire, phase-1 Together speech is non-streaming, `stream_format` forwarding is forward-looking) plus the SSE-first MIME fallback. Decisions made alone: none. -- Step 9 (As-built design document): component-ending verify green against the tree committed at `f44faebb` - fmt, clippy `-D warnings` (same workshop/workshop-server exclude), workspace tests + doctests (2568 + 275; one gateway-local readiness timing assertion fail-fasted the first parallel run and passed isolated and `--no-fail-fast` completion - a flake, this commit carries only the as-built markdown and the plan's F13 frontmatter), featureless gateway check, warnings-denied docs, cargo deny, config-UI npm chain 126/126, guide regen no-diff, `node --check` and `node --test tools/gateway-tts-live.test.mjs` 9/9 (log `vibe/verify-step-9.log`). Review: clean, 0 findings. Amendments beyond the original diff: the as-built reconciled with the bounded relay, the split send_speech deadlines, the serve_mode_for preflight, the Node live probe, A19 numbering, and the rewritten series SHAs (F9); the six frontmatter todos flipped to completed (F13). Decisions made alone: none. diff --git a/vibe/2026-09-12-1-test-namespace-vfs-debt.md b/vibe/2026-09-12-1-test-namespace-vfs-debt.md index 98875c3c8..f0c934af4 100644 --- a/vibe/2026-09-12-1-test-namespace-vfs-debt.md +++ b/vibe/2026-09-12-1-test-namespace-vfs-debt.md @@ -147,11 +147,12 @@ Each step is one commit containing its code and tests. -### Step 4: remove the three committed ledger files +### Step 4: remove the three committed ledger files [completed] - Component: cleanup - In the promptforge repository: move root `vibe-ledger.md`, root `vibe-review.md`, and `vibe/vibe-ledger.md` to `cabinet/_trash/`, stating the recovery sentence for each, and commit the removal. -- Verification: `git status` clean; the commit touches only those three files. +- Verification: `git status` clean; the commit touches only the tracked deletions (root `vibe-ledger.md` and `vibe/vibe-ledger.md`; `vibe-review.md` was never tracked and is moved to `cabinet/_trash/` alongside them). +- Component verification for `cleanup` (the survey's per-crate pattern cannot derive a cross-crate component target): `cargo check`, `cargo clippy --all-targets`, and `cargo test` for `promptforge-core`, `promptforge-lua`, `gateway-config`, `gateway-local`, and `workshop-server`. diff --git a/vibe/vibe-ledger.md b/vibe/vibe-ledger.md deleted file mode 100644 index cf8b897c0..000000000 --- a/vibe/vibe-ledger.md +++ /dev/null @@ -1,131 +0,0 @@ -# Vibe Ledger - -## 2026-09-10-1-unified-prompt-model - -- Step 1: message record validation in the Lua protocol - `cargo test -p promptforge-lua protocol` - 42 passed, 0 failed (also `cargo test -p promptforge-agent`: 23 passed; clippy and fmt clean). - - Decision: tool call records require string `id` and `name`, so the guide's legacy `{ id = call.id }` replay shape now errors | Falsifier: the plan's normalized `{id, name, arguments}` record is what `models.loop` appends and step 12 migrates the guide. - - Decision: cross-record checks (unique call IDs, call-result pairing, alternation) stay out of this parse | Falsifier: step 7 explicitly owns per-dispatch pairing/uniqueness validation. - - Decision: `ContentPart::ImageUrl` keeps only the URL string, dropping extras like `detail` | Falsifier: the Multimodal contract is data-URI image parts; no prompt or template consumes `detail`, and the type can grow if one does. -- Step 2: `messages.new()` builders module - COMPONENT verify: build, `cargo fmt --check`, clippy clean; `cargo test -p promptforge-lua` - 192 passed + 2 doc-tests, 0 failed. - - Decision: builder methods live behind the list metatable's `__index` rather than as direct fields, so serde conversion and protocol validation see only plain records | Falsifier: `lua.from_value` on builder output fails or serializes functions. - - Decision: `install_messages` runs during `inject_host_with_var` beside the H2 models table since the shim needs no privileged captures | Falsifier: a later step requires `messages` in a VM that never injects host values. - - Decision: used `cargo test -p promptforge-lua` instead of installing cargo-nextest | Falsifier: the project survey lists `cargo test -p ` as an accepted component test command, and installing a global toolchain binary is a heavier, host-level change than the failure warrants. -- Step 3: rename heading-based `execute` to `call` - FOCUSED verify: `cargo build` clean; 29 promptforge-lua and 66 promptforge-core filtered tests passed, 0 failed (full coding run: 192 lua + core suites + 23 agent). - - Decision: renamed internal depth machinery (`execute_depth` -> `call_depth`, `MAX_EXECUTE_DEPTH` -> `MAX_CALL_DEPTH`) since it is diagnostics-adjacent core naming for this op | Falsifier: any user-visible string or field still renders `execute` for a heading call. - - Decision: left `guide/`, READMEs, AGENTS.md, and `vibe/` docs untouched; the plan's `migrate-prompts-guides` todo owns doc migration | Falsifier: a doc example that runs in a test still uses heading `execute(`. - - Decision: kept generic verb uses of "execute" and the `crate::execute` module path unchanged | Falsifier: the executor module itself gets renamed. -- Step 4: namespace-only tool and model invocation - COMPONENT verify: build, `cargo fmt --check`, clippy `-D warnings`, and `cargo test -p promptforge-lua` / `-p promptforge-core` all passed (nextest unavailable, survey fallback used). - - Decision: internal protocol op string stays `"tool_call"` and Rust variants keep their names; only the Lua-facing surface moved | Falsifier: a later step renames protocol vocabulary to match the namespace. - - Decision: removed the proxy machinery outright (`wrap_handle`, H1 wrap chunk, `coro_shims` flag, unused `ModelInferHook`) since it existed solely for colon `infer` | Falsifier: a future per-handle Lua-callable shim need reappears. - - Decision: alias-or-Tool decodes once in `tools/decode.rs::tool_alias`, with the shim passing the raw value through the yield | Falsifier: a consumer needs different error wording per call site. - - Decision: legacy non-coroutine `models.infer` (hook path, test-only) keeps its single-arg form | Falsifier: the legacy engine is revived for production paths. - - Pre-existing (deferred): `cargo check -p workshop` fails on a missing staged gateway sidecar binary - environmental setup, untouched by this change. -- Step 5: parser pending Markdown capture - `cargo test -p promptforge-parser` green (87 + 2 doctests); after the review fix, `cargo test -p promptforge-core` green (397 lib + integration), workspace check clean, clippy and fmt clean. - - Decision: removed `off_walk`/`is_off_walk()` outright rather than leaving always-false stubs, per the decision record rejecting off-walk and reader-only break meanings | Falsifier: a later step reintroduces a thematic-break control meaning. - - Decision: parser keeps the interleaved Prose/Lua block stream with post-reset prose text; step 6's scheduler accumulates and installs it lazily | Falsifier: step 6 cannot express per-fence lazy prose from the block stream. - - Decision: H1 shares reset semantics, so `description_text` now comes from below a break | Falsifier: the product contract requires the H1 description from above a break. - - Decision: carried the minimal executor adjustments in this commit rather than re-add a parser shim (review finding close): the scheduler no longer skips off-walk sections and computes `loop_capable` locally as "last prose block in the section" until step 6 | Falsifier: a shim would require restoring the leading-`---` off-walk parse logic that step 5 deliberately removed, contradicting the plan's no-control-flow-meaning contract. -- Step 6: lazy `prose` and `reply` removal - COMPONENT verify: build, fmt, clippy `-D warnings`, `cargo test -p promptforge-lua` and `-p promptforge-core` all passed (382 core lib + 14 integration + 7 doctests; lua 200 + 2; agent 23). - - Decision: `_G`-metatable guard over userdata, so `models.infer(prose)` sees a plain string | Falsifier: a caller needs `prose` as a non-string Lua value. - - Decision: empty buffer installs an empty template (`prose == ""`) | Falsifier: authors need to distinguish "no prose" from "empty prose". - - Decision: `scope`/`tool_loop` kept as `#[cfg(test)]` modules for step 9 | Falsifier: step 9 rewrites rather than rewires them. - - Decision: deleted the prompt-level model-tool-loop tests rather than simulating them, because their only trigger (automatic prose inference) was removed and `models.loop` does not exist until step 9 | Falsifier: step 9's `models.loop` tests must re-add equivalent prompt-level coverage (tool scoping, local-tool VM routing, arm soft-degrade). - - Decision: fixed the clippy `too_many_lines` breakage from the step's own WIP commit (`prose.rs` `install` split into `guard_index`/`guard_newindex`) since the exit criteria require clippy `-D warnings` | Falsifier: the refactor is behavior-preserving - all lua/core tests and clippy pass. -- Step 7: provider-neutral projection and per-dispatch validation - COMPONENT verify: build, fmt, clippy, `cargo test -p promptforge-lua` all passed (nextest fallback). - - Decision: projection lives in `promptforge-lua` - it owns `MessageRecord`, both dispatch points (agent today, core's `models.loop` in step 9) depend on it, and agent cannot depend on core | Falsifier: step 9 cannot consume `project_messages` from core without rework. - - Decision: same-role runs are healed, not rejected; only unhealable structure errors (orphan record, duplicate ID, incomplete pairing, misplaced system/tool fields) | Falsifier: a provider or plan requirement demands rejecting same-role runs outright. - - Decision: user runs join with `\n\n`, assistant fragments with `""`, empty fragments absorbed | Falsifier: a guide or test pins different join text. - - Decision: projection failure observes `MODEL_TURN_FAILED` before riding back as the call's pcall-able answer (review finding close), so a pre-dispatch validation failure is operator-visible | Falsifier: an observer contract requires no failed-turn event for pre-dispatch validation. -- Step 8: minimum compactor surface with `compactors.fail` - COMPONENT verify: build, fmt, clippy `-D warnings`, `cargo test -p promptforge-lua` (223 + 2 doctests), `-p promptforge-core` (386 lib + 14 integration + 7 doctests), `-p promptforge-model-client` (97 + 18 doctests), `-p promptforge-agent` (26) all passed (nextest fallback). - - Decision: the compactor surface (`OverflowReason`, `Compactor`, `precheck`, `is_context_overflow`, `compactors.fail`) lives in `promptforge-lua`, same reasoning as projection: it sits on the message path, both dispatch points depend on it, and the agent cannot depend on core | Falsifier: step 9 cannot consume the surface from core's loop without rework. - - Decision: `OverflowReason` is a two-tag unit enum (`precheck`/`provider`); estimate numbers and budget records stay with the deferred framework | Falsifier: a shipped policy needs the estimate or window figures inside the reason. - - Decision: provider-overflow detection gates on status 400/413 plus case-insensitive body signatures; a 5xx never classifies, since a backend-reported overflow is indistinguishable from a fault | Falsifier: a real backend reports context overflow at 5xx or with unmatched wording, and the loop propagates a bare backend error instead of invoking the compactor. - - Decision: the precheck estimates chars/4 plus 4 tokens per message over text and tool-call arguments; image parts contribute nothing, so image-heavy requests rely on the provider path | Falsifier: the heuristic refuses requests that fit or admits requests that overflow without provider cover. - - Decision: `compactors.fail` is a Rust-backed Lua function that raises the typed error as an mlua external (downcastable per LUA-012), installed beside `messages` in `inject_host_with_var` | Falsifier: step 9 needs a non-callable policy token instead of a function. - - Decision: added `RunErrorKind::ContextExhausted` on the `LuaQuota`/`Quota` precedent so hosts can distinguish policy exhaustion from transport failure; the agent's `From` degrades it to `Program` until the agent path invokes compactors | Falsifier: a host or the agent needs the typed kind before step 9/11 wiring. -- Step 9: Rust-backed `models.loop` - COMPONENT verify: build, fmt, clippy, `cargo test -p promptforge-lua` and `-p promptforge-core` all passed (nextest fallback); 8/8 models_loop tests. - - Decision: the loop runs inline on the driver thread because it holds the section VM for appends and local-tool dispatch; a fanout arm's loop serializes sibling steps behind its rounds | Falsifier: a workload showing arms need concurrent model loops. - - Decision: `models.loop` is section-only (stashed shim, explicit section install), mirroring agent-only `models.chat` | Falsifier: the agent crate adopting `models.loop` before its step-11 rewrite. - - Decision: tool-call exchanges append atomically after the whole batch dispatches | Falsifier: an author workflow needing partial-round history after a caught tool failure. - - Decision: a compactor callback that returns instead of raising is rejected as the deferred replacement shape | Falsifier: the deferred framework defining a meaning for plain returns. - - Decision: `run`'s body future and `run_loop`'s future are `Box::pin`'d to stay under the workspace large-futures lint | Falsifier: measured allocation cost mattering at run/step granularity. -- Step 10: generic input broker - `cargo test -p promptforge-lua user_input` (4 passed) and `cargo test -p promptforge-core input` (11 passed); clippy and fmt clean. - - Decision: broker trait and InputTool live in `promptforge_core::input`, injected via `RunConfig::input_broker`; `None` is the unavailable-fallback policy | Falsifier: step 11 cannot adapt Workshop's WaitRegistry to the trait without core changes. - - Decision: waits recorded as `UserInputWaitStarted`, responses via existing `on_user_input`, on both direct and tool paths | Falsifier: step 11 shows double recording against the workshop's producer-side `on_user_input`. - - Decision: InputTool takes execution/section/observer at construction, per session like today's UserInputTool | Falsifier: a host needs one InputTool shared across sections with correct per-section coordinates. - - Decision: `user_input` is a section-only global; agent VMs get the unreachable guard | Falsifier: an agent program needs direct `user_input` before step 11's chat.md migration. -- Step 11: Agent window on the unified runtime - COMPONENT verify round 2: build, fmt, workspace and workshop clippy, `cargo test -p promptforge-core` and `-p workshop-server` all passed (round 1 failed on the unstaged gateway sidecar binary; repaired with `tools/stage-gateway-sidecar.mjs`, no code change; this also closes the pre-existing `cargo check -p workshop` gap noted at step 4). - - Decision: `RunConfig::ui(provider)` presence is the Agent-window context - installs `ui()` in section VMs and enables raw-gateway-id `models.get` | Falsifier: a second `ui()` consumer needs strict declared-alias resolution, then split the knob. - - Decision: `models.loop` reports content events (thinking/reply/tool-call batch/tool result with metrics) and forwards live deltas via `RunConfig::on_delta` | Falsifier: a host needs the loop silent on the content stream. - - Decision: the model-visible input tool auto-joins every `models.loop` scope when a broker is configured; a prompt-declared `user_input` alias wins | Falsifier: a brokered run needs the model to not see `user_input`. - - Decision: temporary dual-read version gate accepts `promptforge: 0` and `1` | Falsifier: the step-12 migration removes the 1 arm. - - Decision: Markdown sessions record input consumer-side only, skipping the producer-side `on_user_input` | Falsifier: double user_message events appear in the transcript. - - Decision: relaunch starts a fresh message list (the plan's accepted interim regression); gates 4/7/9/10 and the agents.rs catalog tests updated to pin it | Falsifier: the deferred persistence work lands and restores history. - - Decision: added a cheap `ToolPicker::empty` constructor rather than sharing one picker per server (review finding close: per-session embedding-model load) | Falsifier: if a Markdown agent ever binds tools through the picker, `rebuild` from an `empty`-built picker keeps the unloaded model and embeds with the deterministic fallback. -- Step 12: migrate affected prompts, fixtures, and guides - FULL verify round 3: build, fmt, workspace and workshop clippy, workspace suite, workshop crates, and doctests all passed (nextest fallback per the survey's documented host fallback). - - Decision: kept the dual-read 0/1 version gate and left inline unit-test prompt strings at `promptforge: 1`; the step scopes migration to fixtures, shipped prompts, APIs, READMEs, and guides | Falsifier: a follow-up flips the gate to 0-only and migrates test scaffolding. - - Decision: added criterion as the bench harness (dev-only, not in CI) | Falsifier: removing two `[[bench]]` targets and the workspace entry reverts it. - - Decision: lifted the once-per-section `models.use` restriction per the plan's decision record (review finding close): `ModelRuntime::select` records the latest selection unconditionally and re-selection steers the next round, pinned by an end-to-end test | Falsifier: a section needs selection locked after first use. - - Decision: made the build-workshop interruption test tolerate a pre-staged sidecar (move-aside and restore) instead of asserting absence, resolving the pre-existing local conflict between workshop clippy (needs the sidecar) and the test | Falsifier: the test must prove staging from a truly absent state rather than a moved-aside one. - -## 2026-09-10-2-debt-fixes - -- Step 1: Narrow the input tool contract (DEBT-UPM-02) - `cargo test -p promptforge-core input` (14 passed) and `cargo test -p workshop-server --test it chat_gate` (12 passed), nextest fallback per the survey. - - Decision: used the survey's `cargo test` fallback in place of the prescribed nextest command | Falsifier: host has no cargo-nextest binary, and the survey explicitly authorizes the fallback. -- Step 2: Flip the version gate to 0-only and migrate prompts (DEBT-UPM-04) - FULL verify: build, fmt, clippy, workspace suite, workshop crates, doctests all passed (nextest fallback). - - Decision: removed `SUPPORTED_MAJOR` outright rather than repurposing it, since the literal `Some(0)` arm left no uses | Falsifier: a future arm or message that needs a named supported-major constant. - - Decision: updated the `UnsupportedVersion` display text to "supports major 0" though not explicitly listed in the step, since the gate flip made "major 1" false | Falsifier: a reviewer who wants the message wording owned by a separate change. - -## 2026-09-11-3-vfs-foundation - -- Step 1: shared-vfs skeleton, value types, and canonical paths - `cargo nextest run -p shared-vfs` - 11 passed, 0 failed; clippy `-D warnings` and fmt clean. Review: clean. - - Decision: the virtual namespace requires a leading `/`; drive-letter-style paths (`C:/x`) are rejected as relative, with host-path translation deferred to the host backend | Falsifier: a later step requires identity-mount virtual paths of the form `C:/...` to canonicalize. - - Decision: case is preserved and comparison is case-sensitive (POSIX semantics) in the virtual namespace | Falsifier: a host-backend requirement mandates case-insensitive virtual-path comparison. - - Decision: `canonicalize`/`intern` carry targeted `#[allow(dead_code)]` until `Access` (a later step) becomes their caller | Falsifier: the next step wires `Access` and the allows remain. -- Step 2: Vfs and VfsAccess traits and policy types - `cargo nextest run -p shared-vfs traits` - 12 passed, 0 failed. Review: clean. - - Decision: `Op` covers all sixteen access operations (the contract's `// ...` resolved to Exists/Glob/List/Stat/Symlink/ReadLink/Chmod) | Falsifier: a later step needs an op the policy cannot name. - - Decision: str_replace zero/multiple-match failures use `VfsError::Backend` with a descriptive message (no dedicated kind exists) | Falsifier: the facade or model recovery path needs to match on a distinct kind. - - Decision: default grep returns `Unsupported` for `is_regex` (std-only crate cannot ship a regex engine) and skips non-UTF-8 files | Falsifier: a caller requires regex semantics from the memory backend's default grep. -- Step 3: handle, Access capability, and claims tables - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p shared-vfs` - pass (48/48 tests). Review: 1 Important (copy/rename dual-path claim kinds untested), closed with 5 tests; drift review against component base clean. Verification fixes: fmt normalization in handle.rs tests; clippy `# Errors` docs on 19 trait methods and a collapsed nested if in step-2 traits.rs. - - Decision: added `VfsRef::with_policy` as the policy-installation seam (contract sketch shows only `new`) | Falsifier: step 7 installs ModePolicy through it with no further handle change. - - Decision: `Ask` maps to `PermissionDenied(reason)` in v1; the approval dialog is a plan non-goal | Falsifier: a later step needs Ask distinguishable and adds a `VfsError` kind (non_exhaustive permits). - - Decision: policy stored as `Arc` so `VfsRef`/`Access` are Send+Sync despite `Policy: Send` | Falsifier: a Send-but-not-Sync policy impl forces revisiting the bound. - - Decision: invalid line ranges return `VfsError::Backend` (no InvalidRange kind) | Falsifier: step 8's parity suite demands a dedicated kind. - - Decision: alias test uses lexical spellings (`/a/./b.txt` vs `/a//b.txt`); facade-relative spelling belongs to the step-8 Store facade since `canonicalize` rejects relative paths here | Falsifier: step 8's alias test covers facade-relative vs mount-absolute. -- Step 4: router, builder, and overlays - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p shared-vfs` - pass (58/58 tests). Review: clean (with component-base drift check). Verification fixes: fmt normalization; clippy `#[must_use]` on builder/overlay/build, `VfsPath` by value in private helpers, let-else rewrite. - - Decision: backends see mount-relative rooted paths (router strips on dispatch, rejoins on glob/grep results) | Falsifier: a contract passage requiring backends to see full virtual paths. - - Decision: cross-mount rename/copy return `Unsupported` instead of read-plus-write | Falsifier: a caller needing atomic cross-mount moves. - - Decision: `Router::release` and mounted-handle `release` are no-ops; teardown flows through the routing session's `Drop` | Falsifier: a backend requiring explicit release independent of `Drop`. - - Decision: `overlay()` shares the base's policy `Arc` as well as its claims table | Falsifier: a requirement that overlays carry independent policy. -- Step 5: memory backend - `cargo nextest run -p shared-vfs` - 91 passed, 0 failed; clippy `-D warnings` clean. Review: 1 Critical (renaming a directory onto the namespace root bypassed the DirectoryNotEmpty guard and rewrote subtree keys to unreachable `//...` paths - silent data loss), closed by rejecting `dest == "/"` in the directory branch with a regression test. - - Decision: writes materialize ancestor directories instead of requiring mkdir (MemStore flat-map semantics carried over) | Falsifier: the Store facade or a host backend needs POSIX ENOENT-on-missing-parent behavior. - - Decision: glob results include directories, not just files | Falsifier: a caller (engine adapter, grep default) misbehaves when directories match. - - Decision: backend named `MemoryBackend` (plan pins no name; parallels `HostBackend`) | Falsifier: a later plan step or review pins a different name. -- Step 6: host backend, stage 1 thin - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p shared-vfs` - pass (100/100 tests). Review: 2 Important, both closed (`exists` mapped all errors to Ok(false), now NotFound-only; literal glob patterns never matched because walk_root did not resolve wildcard-free patterns to their parent). Verification fix: fmt drift in memory.rs. - - Decision: `rooted()` returns `Result` (canonicalization can fail) rather than panicking, matching the RealFs oracle's `open` | Falsifier: the contract's mount example shows `HostBackend::rooted("C:/work")` used without unwrap. - - Decision: read-only is set via `with_read_only(bool)` builder, not a `read_only()` setter, to avoid shadowing the `Vfs::read_only` trait method | Falsifier: a later step needing mid-run flips of the flag (the contract assigns mode flips to Policy, not the backend flag). - - Decision: containment canonicalizes the nearest existing ancestor and re-appends the missing tail; dangling-symlink resolution is documented as deferred to stage 2 | Falsifier: a write through a dangling symlink inside the root escaping containment. -- Step 7: promptforge-vfs policy crate - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p promptforge-vfs` - pass (5/5 tests). Review: clean (with component-base drift check). Verification fix: fmt normalization of two test assertions. - - Decision: Ask mode returns `Verdict::Ask` (not `Deny`) for mutations; the capability layer maps both to `PermissionDenied`, and the Ask string is the approval-dialog text per the contract | Falsifier: a later integration step asserts `Verdict::Deny` from `ModePolicy::check` in Ask mode. - - Decision: Plan's markdown rule is a case-sensitive `.md` suffix (clippy's case-insensitive suggestion explicitly `#[expect]`-overridden), matching the POSIX-strict virtual namespace | Falsifier: a host needs `.MD`/`.markdown` admitted in Plan mode. - - Decision: `ModeHandle` is a separate cloneable UI half (`policy.handle()`), per "one-way vs reversible is just who still holds the mode handle" | Falsifier: a caller needs to flip modes holding only the `ModePolicy`. -- Step 8: Store facade rewrite and parity suite - `cargo nextest run -p promptforge-store` - 44 passed, 0 failed, plus 15 doctests; `cargo build` and `cargo fmt --all --check` pass. Review: clean after a needs-context re-review supplied the decision-record extension-trait rationale; 1 Important (glob/stat claim conflicts surfaced as opaque Backend instead of WriteRace), closed with a regression test. DEVIATION: the COMPONENT clippy leg is deferred to run jointly after step 9 - the plan's step ordering leaves promptforge-lua referencing the removed StoreRef/WriteScope API, and the verification-fix round correctly returned blocked rather than perform step-9 migration inside step 8 | Falsifier: the joint verify after step 9 fails the promptforge-store component. - - Decision: `WriteRace` display text now names the claims model, not fanout arms | Falsifier: a caller matching on the old message text. - - Decision: facade implements anchor edits and line ranges itself (read/count/write) to preserve the exact `StoreError` vocabulary | Falsifier: a need for backend-atomic `str_replace`. - - Decision: glob delegates matching to the backend and stat-filters to files only | Falsifier: glob latency complaints on large trees. - - Decision: backslash glob rejection lives in the facade because the router canonicalizes patterns before the backend sees them | Falsifier: router forwarding verbatim patterns. - - Decision: `GlobSpyStore`, `with_files`, and `FileStore` tests dropped; their mechanisms no longer exist (matching moved to the backend, `with_files` had no external callers, `FileStore` superseded by `HostBackend`) | Falsifier: a host depending on file-backed store behavior through this crate. -- Step 8 deferred leg closed: workspace clippy `-D warnings` passed clean during step 9's coding run after the caller migration - the falsifier did not fire. -- Step 9: executor API pivot to VfsRef - FOCUSED verify: `cargo build`, `cargo nextest run -p promptforge-core execute::` - pass (314/314); coding run additionally passed 843 tests across shared-vfs, promptforge-vfs, promptforge-store, promptforge-lua, promptforge-agent, promptforge-core, 351 workshop-server tests, doctests including the run() VfsRef doc example, and workspace clippy/fmt. Review: 1 Important (missing mount-less-handle fallback test), 1 Minor (store-mount probe swallowed backend errors), both closed; probe now matches NotFound specifically and other errors fail the run via a new `Error::Store` variant mapped to the previously unreachable `RunErrorKind::Store`. - - Decision: access lives in the Chain (arena is append-only), taken at finish/abort, so a finished arm's claims never block the join's merge | Falsifier: a parent merge conflicting with a dead arm's claims. - - Decision: call chains Arc-borrow the parent's access; arms spawn from the fanout caller | Falsifier: a false conflict between a caller's claims and its blocking child. - - Decision: `run_agent` takes `&VfsRef` - the census missed promptforge-agent; StoreRef's deletion forced it | Falsifier: the four-crates API criterion read strictly. - - Decision: the fanout-store-writes fixture's ready-*.md rendezvous was removed - polling a live sibling's writes is exactly the cross-arm read-while-written pattern claims reject; interleaving coverage stays in fanout_arms_interleave_at_io_points_on_one_thread | Falsifier: a requirement that this fixture prove arm concurrency. - - Decision: run() probes stat(STORE_MOUNT) and overlays a fresh memory store only when absent | Falsifier: a router whose mount exists but stats error getting a shadowing overlay. -- Step 10: store operations as leaf yields - COMPONENT verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`, `cargo nextest run -p promptforge-core` - pass (429/429 core tests; coding run also green on promptforge-store 44, promptforge-lua 91 filtered, promptforge-agent 26). Review: 1 Critical (store yield shims installed before replay_shared broke load-time store calls in the shared library - main chunk cannot yield), closed by moving install_store_shims after replay_shared; 1 Minor (dead conflict_detail accessor), closed by routing classify_store_failure through it. Verification fix: clippy needless_pass_by_value on classify_store_failure. - - Decision: agent VMs keep direct store closures (single-identity driver, no interleaving) | Falsifier: the agent driver gains fanout or shares its VfsRef with a second live identity. - - Decision: non-conflict store failures resume as `Error::Lua` answers (legacy classification preserved) | Falsifier: a host needs `RunErrorKind::Store` for Lua-triggered store failures. - - Noted behavior change: cross-arm same-path writes now boom even with no other suspension point; the old two_arms_appending_one_path_succeed premise was inverted and the fanout_arms_take_global_ids fixture restructured to arm-scoped paths. -- Step 11: Bashkit adapter spike - FULL verify: `cargo build`, `cargo fmt --all --check`, `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings` plus workshop crates, `mdbook build guide`, `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server`, `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --all-features`, doctests - pass. Review: clean. Spike result: the Vfs trait subsumes Bashkit's FsBackend; ls/cat/grep scripts pass against memory and store mounts; no mapping failure. Verification fixes: fmt normalization; `to_io` consumes the VfsError into IoError::new preserving the source chain. - - Decision: absent `Stat` timestamps map to `SystemTime::UNIX_EPOCH`, not `now()` | Falsifier: a script's time-based behavior (e.g. `ls -t` ordering) needs real mtimes, requiring the memory backend to track them rather than the adapter fabricating them. - - Decision: claims conflicts and policy denials both map to `ErrorKind::PermissionDenied` | Falsifier: a builtin must distinguish "locked by another identity" from "denied by policy" to recover correctly, requiring a richer `to_io` mapping. From 1c550ee5bd57d8cd36606fdfc4100098182eb652 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 02:44:35 -0700 Subject: [PATCH 16/26] Remove the promptforge-bashkit spike crate Removes the spike crate that adapted the script engine's filesystem backend trait over the shared virtual filesystem handle. The spike existed to prove the filesystem trait subsumes the engine's storage contract; that evidence is recorded, so the crate has no remaining purpose. Its manifest pinned a path dependency on a sibling checkout outside the repository, which broke dependency resolution on any machine without that checkout. The lockfile is regenerated, dropping the engine and every dependency that existed only to serve it. - `crates/promptforge-bashkit` is removed outright rather than relocated into a spikes directory or excluded from the workspace manifest; both alternatives keep dead weight whose evidence is already recorded. - `Cargo.lock` drops seventeen packages with the regeneration, including the engine crate and its exclusive transitive dependencies, and collapses the duplicate sha1 and fancy-regex entries each to one version. - `crates/promptforge-bashkit/src/lib.rs` needs no caller edits alongside its deletion; no source outside the removed crate changes. Plan: vibe/2026-09-12-1-test-namespace-vfs-debt.md --- Cargo.lock | 186 +---------- crates/promptforge-bashkit/Cargo.toml | 24 -- crates/promptforge-bashkit/src/lib.rs | 317 ------------------- vibe/2026-09-12-1-test-namespace-vfs-debt.md | 2 +- 4 files changed, 4 insertions(+), 525 deletions(-) delete mode 100644 crates/promptforge-bashkit/Cargo.toml delete mode 100644 crates/promptforge-bashkit/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 182838ae3..5e6bfa885 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -346,7 +346,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1 0.10.7", + "sha1", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -393,56 +393,6 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" -[[package]] -name = "base64" -version = "0.23.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac07cdecf99051d9a5238b80f35af32cdeba5b336e55d957b318b50137e18da5" - -[[package]] -name = "bashkit" -version = "0.18.0" -dependencies = [ - "anyhow", - "async-trait", - "base64 0.23.1", - "bigdecimal", - "bitflags 2.13.1", - "bzip2", - "chrono", - "clap", - "fancy-regex 0.19.1", - "flate2", - "futures-util", - "getrandom 0.4.3", - "hmac", - "md-5", - "num-traits", - "os_display", - "regex", - "serde", - "serde_json", - "sha1 0.11.0", - "sha2 0.11.0", - "thiserror 2.0.19", - "tokio", - "unit-prefix", - "url", -] - -[[package]] -name = "bigdecimal" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d6867f1565b3aad85681f1015055b087fcfd840d6aeee6eee7f2da317603695" -dependencies = [ - "autocfg", - "libm", - "num-bigint", - "num-integer", - "num-traits", -] - [[package]] name = "bit-set" version = "0.8.0" @@ -670,15 +620,6 @@ dependencies = [ "serde", ] -[[package]] -name = "bzip2" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" -dependencies = [ - "libbz2-rs-sys", -] - [[package]] name = "cairo-rs" version = "0.18.5" @@ -764,7 +705,7 @@ dependencies = [ "byteorder", "candle-core", "candle-nn", - "fancy-regex 0.18.0", + "fancy-regex", "num-traits", "rand 0.9.5", "rayon", @@ -938,7 +879,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" dependencies = [ "clap_builder", - "clap_derive", ] [[package]] @@ -951,18 +891,6 @@ dependencies = [ "clap_lex", ] -[[package]] -name = "clap_derive" -version = "4.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" -dependencies = [ - "heck 0.5.0", - "proc-macro2", - "quote", - "syn 3.0.3", -] - [[package]] name = "clap_lex" version = "1.1.0" @@ -978,12 +906,6 @@ dependencies = [ "cc", ] -[[package]] -name = "cmov" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" - [[package]] name = "colored" version = "3.1.1" @@ -1338,15 +1260,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "ctutils" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" -dependencies = [ - "cmov", -] - [[package]] name = "darling" version = "0.20.11" @@ -1564,7 +1477,6 @@ dependencies = [ "block-buffer 0.12.1", "const-oid", "crypto-common 0.2.2", - "ctutils", ] [[package]] @@ -1866,17 +1778,6 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "fancy-regex" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e0387578e845beb7a1acff126228499f26cb18edf12919cc513bb863266464" -dependencies = [ - "bit-set", - "regex-automata", - "regex-syntax", -] - [[package]] name = "fastrand" version = "2.5.0" @@ -2914,15 +2815,6 @@ dependencies = [ "xet-runtime", ] -[[package]] -name = "hmac" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" -dependencies = [ - "digest 0.11.3", -] - [[package]] name = "hound" version = "3.5.1" @@ -3602,12 +3494,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "libbz2-rs-sys" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" - [[package]] name = "libc" version = "0.2.189" @@ -3821,16 +3707,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" -[[package]] -name = "md-5" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" -dependencies = [ - "cfg-if 1.0.4", - "digest 0.11.3", -] - [[package]] name = "memchr" version = "2.8.3" @@ -4101,16 +3977,6 @@ dependencies = [ "windows-sys 0.61.2", ] -[[package]] -name = "num-bigint" -version = "0.4.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367" -dependencies = [ - "num-integer", - "num-traits", -] - [[package]] name = "num-complex" version = "0.4.6" @@ -4127,15 +3993,6 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" -[[package]] -name = "num-integer" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce2d95d4b3734dc35aa2f45e1aa22cd416814592a4f9d9205e11affd5b8e10b" -dependencies = [ - "num-traits", -] - [[package]] name = "num-traits" version = "0.2.19" @@ -4533,15 +4390,6 @@ dependencies = [ "pin-project-lite", ] -[[package]] -name = "os_display" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad5fd71b79026fb918650dde6d125000a233764f1c2f1659a1c71118e33ea08f" -dependencies = [ - "unicode-width", -] - [[package]] name = "os_str_bytes" version = "6.6.1" @@ -4976,17 +4824,6 @@ dependencies = [ "tokio", ] -[[package]] -name = "promptforge-bashkit" -version = "0.3.0" -dependencies = [ - "bashkit", - "promptforge-vfs", - "shared-vfs", - "tokio", - "tracing", -] - [[package]] name = "promptforge-core" version = "0.3.0" @@ -6199,17 +6036,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if 1.0.4", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - [[package]] name = "sha2" version = "0.10.9" @@ -7647,7 +7473,7 @@ dependencies = [ "httparse", "log", "rand 0.9.5", - "sha1 0.10.7", + "sha1", "thiserror 2.0.19", ] @@ -7772,12 +7598,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" -[[package]] -name = "unit-prefix" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81e544489bf3d8ef66c953931f56617f423cd4b5494be343d9b9d3dda037b9a3" - [[package]] name = "unsafe-libyaml" version = "0.2.11" diff --git a/crates/promptforge-bashkit/Cargo.toml b/crates/promptforge-bashkit/Cargo.toml deleted file mode 100644 index 8e04d17c3..000000000 --- a/crates/promptforge-bashkit/Cargo.toml +++ /dev/null @@ -1,24 +0,0 @@ -[package] -name = "promptforge-bashkit" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -publish = false - -description = "Spike: the Bashkit engine's FsBackend implemented over the shared VFS handle, evidencing that the VFS trait subsumes Bashkit" - -[dependencies] -shared-vfs.workspace = true -tracing.workspace = true -# Spike-only path dependency on the local Bashkit clone (a sibling of this -# repository). Default features stay off: the spike needs the embeddable -# interpreter only, not the LLM tool wrapper or the timezone database. -bashkit = { path = "../../../bashkit/crates/bashkit", default-features = false } - -[dev-dependencies] -promptforge-vfs.workspace = true -tokio.workspace = true - -[lints] -workspace = true diff --git a/crates/promptforge-bashkit/src/lib.rs b/crates/promptforge-bashkit/src/lib.rs deleted file mode 100644 index 38bbd636c..000000000 --- a/crates/promptforge-bashkit/src/lib.rs +++ /dev/null @@ -1,317 +0,0 @@ -//! Spike: the Bashkit engine's `FsBackend` trait implemented over the -//! shared VFS handle. -//! -//! The deliverable is evidence that `Vfs` subsumes Bashkit's storage -//! contract: whole-file reads serve from `read`, `symlink`/`chmod` return -//! the engine's unsupported error, the first four file types map directly -//! and the three specials map to `File` with a trace, and an absent `Stat` -//! mode emits the 0o644/0o755 defaults. The adapter captures the current -//! [`ExecId`] at exec start: one identity per engine session, so the -//! claims model attributes every script operation to that session. - -use std::io::{Error as IoError, ErrorKind}; -use std::path::{Path, PathBuf}; -use std::time::SystemTime; - -use bashkit::{DirEntry, Error, FileType as BashFileType, FsBackend, Metadata, Result}; -use shared_vfs::{Access, ExecId, FileType as VfsFileType, Stat, VfsError, VfsRef}; - -/// A Bashkit storage backend serving from a VFS handle. -/// -/// Constructing one acquires an [`Access`] capability: the adapter holds -/// one [`ExecId`] for the engine session's lifetime, so every script -/// operation is attributed to that identity and the claims model sees -/// the session as one thread of execution. The engine's `PosixFs` -/// wrapper enforces POSIX semantics above this raw storage layer. -#[derive(Debug)] -pub struct VfsBackend { - access: Access, -} - -impl VfsBackend { - /// Captures a fresh identity from `vfs`: call at exec start. - #[must_use] - pub fn new(vfs: &VfsRef) -> VfsBackend { - VfsBackend { - access: vfs.acquire(), - } - } - - /// Binds the backend to an existing capability: a host that already - /// holds the run's [`Access`] keeps the script under that identity. - #[must_use] - pub fn from_access(access: Access) -> VfsBackend { - VfsBackend { access } - } - - /// The identity this backend's operations are attributed to. - #[must_use] - pub fn id(&self) -> ExecId { - self.access.id() - } -} - -/// The engine hands `Path` values; the virtual namespace is POSIX-shaped -/// text. Lossy conversion with separator normalization is sufficient: -/// the engine never produces host-native paths here. -fn vfs_path(path: &Path) -> String { - path.to_string_lossy().replace('\\', "/") -} - -/// Maps the VFS error onto the engine's io-error channel, preserving the -/// kind so builtins report the right failure (`PermissionDenied` for a -/// claims conflict or policy denial, `Unsupported` for unimplemented -/// operations, and so on). -fn to_io(error: VfsError) -> Error { - let kind = match &error { - VfsError::NotFound(_) => ErrorKind::NotFound, - VfsError::PermissionDenied(_) | VfsError::Conflict(_) => ErrorKind::PermissionDenied, - VfsError::AlreadyExists(_) => ErrorKind::AlreadyExists, - VfsError::InvalidPath(_) => ErrorKind::InvalidInput, - VfsError::NotADirectory(_) => ErrorKind::NotADirectory, - VfsError::IsADirectory(_) => ErrorKind::IsADirectory, - VfsError::DirectoryNotEmpty(_) => ErrorKind::DirectoryNotEmpty, - VfsError::Unsupported(_) => ErrorKind::Unsupported, - _ => ErrorKind::Other, - }; - IoError::new(kind, error).into() -} - -/// The first four kinds map directly; the three specials map to `File` -/// with a trace (unreachable in practice: neither our v1 backends nor -/// the engine's ever produce them). -fn file_type(kind: VfsFileType) -> BashFileType { - match kind { - VfsFileType::File => BashFileType::File, - VfsFileType::Directory => BashFileType::Directory, - VfsFileType::Symlink => BashFileType::Symlink, - VfsFileType::Fifo => BashFileType::Fifo, - special => { - tracing::warn!( - ?special, - "VFS special file type reported to the engine as File" - ); - BashFileType::File - } - } -} - -/// The VFS says `None` rather than fabricating; the engine's `Metadata` -/// has no options, so an absent mode emits the 0o644/0o755 defaults and -/// absent timestamps become the epoch - deterministic, never an invented -/// `now()`. -fn metadata(stat: &Stat) -> Metadata { - let mode = stat.mode.unwrap_or(match stat.file_type { - VfsFileType::Directory => 0o755, - _ => 0o644, - }); - Metadata { - file_type: file_type(stat.file_type), - size: stat.size, - mode, - modified: stat.modified.unwrap_or(SystemTime::UNIX_EPOCH), - created: stat.created.unwrap_or(SystemTime::UNIX_EPOCH), - } -} - -/// The engine's unsupported error, matching its own convention of an -/// io error with `ErrorKind::Unsupported`. -fn unsupported(op: &str) -> Error { - IoError::new( - ErrorKind::Unsupported, - format!("{op} is not supported by the VFS adapter"), - ) - .into() -} - -#[bashkit::async_trait] -impl FsBackend for VfsBackend { - async fn read(&self, path: &Path) -> Result> { - self.access.read(&vfs_path(path)).map_err(to_io) - } - - async fn write(&self, path: &Path, content: &[u8]) -> Result<()> { - self.access.write(&vfs_path(path), content).map_err(to_io) - } - - async fn append(&self, path: &Path, content: &[u8]) -> Result<()> { - self.access.append(&vfs_path(path), content).map_err(to_io) - } - - async fn mkdir(&self, path: &Path, recursive: bool) -> Result<()> { - self.access.mkdir(&vfs_path(path), recursive).map_err(to_io) - } - - async fn remove(&self, path: &Path, recursive: bool) -> Result<()> { - self.access - .remove(&vfs_path(path), recursive) - .map_err(to_io) - } - - async fn stat(&self, path: &Path) -> Result { - self.access - .stat(&vfs_path(path)) - .map(|stat| metadata(&stat)) - .map_err(to_io) - } - - async fn read_dir(&self, path: &Path) -> Result> { - let entries = self.access.list(&vfs_path(path)).map_err(to_io)?; - Ok(entries - .into_iter() - .map(|entry| DirEntry { - name: entry.name, - metadata: metadata(&entry.stat), - }) - .collect()) - } - - async fn exists(&self, path: &Path) -> Result { - self.access.exists(&vfs_path(path)).map_err(to_io) - } - - async fn rename(&self, from: &Path, to: &Path) -> Result<()> { - self.access - .rename(&vfs_path(from), &vfs_path(to)) - .map_err(to_io) - } - - async fn copy(&self, from: &Path, to: &Path) -> Result<()> { - self.access - .copy(&vfs_path(from), &vfs_path(to)) - .map_err(to_io) - } - - async fn symlink(&self, _target: &Path, _link: &Path) -> Result<()> { - Err(unsupported("symlink")) - } - - async fn read_link(&self, _path: &Path) -> Result { - Err(unsupported("read_link")) - } - - async fn chmod(&self, _path: &Path, _mode: u32) -> Result<()> { - Err(unsupported("chmod")) - } -} - -#[cfg(test)] -mod tests { - use std::io::ErrorKind; - use std::path::Path; - use std::sync::Arc; - - use bashkit::{Bash, Error, FsBackend, PosixFs}; - use shared_vfs::{FileType as VfsFileType, MemoryBackend, VfsRef}; - - use super::{VfsBackend, file_type}; - use bashkit::FileType as BashFileType; - - type TestResult = Result<(), Box>; - - /// An engine whose entire filesystem is the VFS handle, with POSIX - /// semantics enforced by the engine's own wrapper. - fn engine(vfs: &VfsRef) -> Bash { - let backend = VfsBackend::new(vfs); - let fs = Arc::new(PosixFs::new(backend)); - Bash::builder().fs(fs).build() - } - - #[tokio::test] - async fn an_ls_cat_grep_script_runs_against_a_mounted_memory_backend() -> TestResult { - let vfs = VfsRef::builder().mount("/", MemoryBackend::new()).build(); - let mut bash = engine(&vfs); - let result = bash - .exec( - "mkdir -p /tmp/docs && echo hello > /tmp/docs/a.txt \ - && ls /tmp/docs && cat /tmp/docs/a.txt \ - && grep hello /tmp/docs/a.txt", - ) - .await?; - assert_eq!(result.exit_code, 0, "stderr: {}", result.stderr); - let stdout = result.stdout.text_lossy().into_owned(); - assert!(stdout.contains("a.txt"), "ls lists the file: {stdout}"); - assert!( - stdout.contains("hello"), - "cat and grep serve reads: {stdout}" - ); - Ok(()) - } - - #[tokio::test] - async fn an_ls_cat_grep_script_runs_against_the_store_mount() -> TestResult { - let vfs = promptforge_vfs::empty(); - vfs.acquire() - .write("/_promptforge/store/paper.md", b"# Draft\nhello world\n")?; - let mut bash = engine(&vfs); - let result = bash - .exec( - "ls /_promptforge/store && cat /_promptforge/store/paper.md \ - && grep hello /_promptforge/store/paper.md", - ) - .await?; - assert_eq!(result.exit_code, 0, "stderr: {}", result.stderr); - let stdout = result.stdout.text_lossy().into_owned(); - assert!(stdout.contains("paper.md"), "ls lists the store: {stdout}"); - assert!( - stdout.contains("hello world"), - "cat and grep read the store: {stdout}" - ); - Ok(()) - } - - #[tokio::test] - async fn symlink_and_chmod_return_the_engine_unsupported_error() -> TestResult { - let vfs = VfsRef::new(MemoryBackend::new()); - let backend = VfsBackend::new(&vfs); - match backend.symlink(Path::new("/a"), Path::new("/b")).await { - Err(Error::Io(io)) => assert_eq!(io.kind(), ErrorKind::Unsupported), - other => panic!("expected an unsupported io error, got {other:?}"), - } - match backend.read_link(Path::new("/a")).await { - Err(Error::Io(io)) => assert_eq!(io.kind(), ErrorKind::Unsupported), - other => panic!("expected an unsupported io error, got {other:?}"), - } - match backend.chmod(Path::new("/a"), 0o600).await { - Err(Error::Io(io)) => assert_eq!(io.kind(), ErrorKind::Unsupported), - other => panic!("expected an unsupported io error, got {other:?}"), - } - Ok(()) - } - - #[tokio::test] - async fn an_absent_stat_mode_emits_the_posix_defaults() -> TestResult { - let vfs = VfsRef::new(MemoryBackend::new()); - let backend = VfsBackend::new(&vfs); - backend.write(Path::new("/f.txt"), b"x").await?; - backend.mkdir(Path::new("/d"), false).await?; - // The memory backend honestly reports mode None; the adapter - // emits the engine's expected defaults instead. - assert_eq!(backend.stat(Path::new("/f.txt")).await?.mode, 0o644); - assert_eq!(backend.stat(Path::new("/d")).await?.mode, 0o755); - Ok(()) - } - - #[test] - fn special_file_types_map_to_file_and_the_first_four_map_directly() { - assert_eq!(file_type(VfsFileType::File), BashFileType::File); - assert_eq!(file_type(VfsFileType::Directory), BashFileType::Directory); - assert_eq!(file_type(VfsFileType::Symlink), BashFileType::Symlink); - assert_eq!(file_type(VfsFileType::Fifo), BashFileType::Fifo); - assert_eq!(file_type(VfsFileType::Socket), BashFileType::File); - assert_eq!(file_type(VfsFileType::CharDevice), BashFileType::File); - assert_eq!(file_type(VfsFileType::BlockDevice), BashFileType::File); - } - - #[test] - fn each_adapter_captures_a_fresh_exec_identity_at_exec_start() { - let vfs = VfsRef::new(MemoryBackend::new()); - let first = VfsBackend::new(&vfs); - let second = VfsBackend::new(&vfs); - assert_ne!(first.id(), second.id()); - // A host that already holds a capability binds it explicitly. - let access = vfs.acquire(); - let bound = VfsBackend::from_access(access); - assert_ne!(bound.id(), first.id()); - } -} diff --git a/vibe/2026-09-12-1-test-namespace-vfs-debt.md b/vibe/2026-09-12-1-test-namespace-vfs-debt.md index f0c934af4..0880e307d 100644 --- a/vibe/2026-09-12-1-test-namespace-vfs-debt.md +++ b/vibe/2026-09-12-1-test-namespace-vfs-debt.md @@ -158,7 +158,7 @@ Each step is one commit containing its code and tests. -### Step 5: remove the promptforge-bashkit crate +### Step 5: remove the promptforge-bashkit crate [completed] - Component: vfs-debt-removal - Remove `promptforge/crates/promptforge-bashkit/` by moving it to `cabinet/_trash/` (stating the recovery sentence), regenerate `Cargo.lock`, and sweep remaining bashkit references in CI, guide, and READMEs. From fca9d4e405968679c214ebe29685874746bd841f Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 02:51:23 -0700 Subject: [PATCH 17/26] De-intern VfsPath and close the manifest test bypass Virtual paths now hold their canonical string in a reference-counted allocation per value lineage instead of an id into a process-global interner, so a dropped path's string frees with its last owner rather than leaking for the life of the process. The zero-dependency manifest check now recognizes dependency sub-tables and target-qualified tables, closing a bypass that let a dependency sub-table declare a dependency undetected. The architecture document now describes the store facade and the virtual filesystem layer as separate components with their real dependency directions. - `VfsPath` replaces its u32 interner id with an `Arc` field; the type loses `Copy`, and `as_str` returns a borrow of self instead of a static string. - `is_dependency_table` extracts the manifest test's section matching into one helper that counts exact dependency tables, their sub-tables, and target-qualified tables as dependency declarations. - `vibe/archdoc.md` replaces the store component line with a Store facade entry that depends on the VFS layer and adds a VFS layer entry covering the shared machinery and the policy gate. - `gate` hands the claims table its own clone of the canonical path, so the string frees when the claim and every other owner drop. - `canonicalize` allocates one shared string per call; there is no global table and no lock. - `crates/shared-vfs/src/router.rs` and the host and handle internals now pass paths by reference, a mechanical adaptation to the loss of Copy. - `Interner` is gone along with its process-global mutex and the u32-exhaustion panic. Design: removes global-state @ crates/shared-vfs/src/path.rs::interner Design: new pure-function @ crates/shared-vfs/src/lib.rs::tests::is_dependency_table deps: &str Repairs: a dropped path's string frees with its last owner @ crates/shared-vfs/src/path.rs::canonicalize - the interner leaked every distinct canonicalized path for the process lifetime Repairs: the zero-dependency manifest check rejects every dependency table @ crates/shared-vfs/src/lib.rs::tests::is_dependency_table - a [dependencies.foo] sub-table declared a dependency while passing the exact-match section check Plan: vibe/2026-09-12-1-test-namespace-vfs-debt.md --- crates/promptforge-lua/src/tests.rs | 22 ++-- crates/promptforge-vfs/src/lib.rs | 4 +- crates/shared-vfs/Cargo.toml | 2 +- crates/shared-vfs/src/handle.rs | 28 +++-- crates/shared-vfs/src/host.rs | 56 ++++----- crates/shared-vfs/src/lib.rs | 40 ++++++- crates/shared-vfs/src/path.rs | 111 ++++++++--------- crates/shared-vfs/src/router.rs | 120 +++++++++---------- vibe/2026-09-12-1-test-namespace-vfs-debt.md | 2 +- vibe/archdoc.md | 3 +- 10 files changed, 203 insertions(+), 185 deletions(-) diff --git a/crates/promptforge-lua/src/tests.rs b/crates/promptforge-lua/src/tests.rs index dce7bf208..bd90c1272 100644 --- a/crates/promptforge-lua/src/tests.rs +++ b/crates/promptforge-lua/src/tests.rs @@ -63,7 +63,7 @@ fn lua_error_message(error: &Error) -> &str { struct FailingBackend; impl FailingBackend { - fn error(path: VfsPath) -> VfsError { + fn error(path: &VfsPath) -> VfsError { VfsError::Backend(format!( "the failing backend rejects every operation: {path}" )) @@ -86,23 +86,23 @@ struct FailingAccess; impl VfsAccess for FailingAccess { fn read(&self, path: &VfsPath) -> std::result::Result, VfsError> { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn write(&mut self, path: &VfsPath, _contents: &[u8]) -> std::result::Result<(), VfsError> { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn append(&mut self, path: &VfsPath, _contents: &[u8]) -> std::result::Result<(), VfsError> { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn remove(&mut self, path: &VfsPath, _recursive: bool) -> std::result::Result<(), VfsError> { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn exists(&self, path: &VfsPath) -> std::result::Result { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn glob(&self, pattern: &str) -> std::result::Result, VfsError> { @@ -112,23 +112,23 @@ impl VfsAccess for FailingAccess { } fn list(&self, path: &VfsPath) -> std::result::Result, VfsError> { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn stat(&self, path: &VfsPath) -> std::result::Result { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn mkdir(&mut self, path: &VfsPath, _recursive: bool) -> std::result::Result<(), VfsError> { - Err(FailingBackend::error(*path)) + Err(FailingBackend::error(path)) } fn rename(&mut self, from: &VfsPath, _to: &VfsPath) -> std::result::Result<(), VfsError> { - Err(FailingBackend::error(*from)) + Err(FailingBackend::error(from)) } fn copy(&mut self, from: &VfsPath, _to: &VfsPath) -> std::result::Result<(), VfsError> { - Err(FailingBackend::error(*from)) + Err(FailingBackend::error(from)) } } diff --git a/crates/promptforge-vfs/src/lib.rs b/crates/promptforge-vfs/src/lib.rs index 6d96c135b..a8aed462f 100644 --- a/crates/promptforge-vfs/src/lib.rs +++ b/crates/promptforge-vfs/src/lib.rs @@ -112,7 +112,7 @@ fn is_mutation(op: Op) -> bool { clippy::case_sensitive_file_extension_comparisons, reason = "virtual paths are POSIX-strict; case-insensitive extension matching is a host-OS notion" )] -fn is_markdown(path: VfsPath) -> bool { +fn is_markdown(path: &VfsPath) -> bool { path.as_str().ends_with(".md") } @@ -130,7 +130,7 @@ impl Policy for ModePolicy { // so Plan refuses to even read a non-markdown source. The // policy cannot tell the two apart; conservative refusal // is the safe side. - Mode::Plan if is_markdown(*path) => Verdict::Allow, + Mode::Plan if is_markdown(path) => Verdict::Allow, Mode::Plan => Verdict::Deny(format!( "{op:?} on {path} is refused: the Plan mode allows mutations only to markdown paths" )), diff --git a/crates/shared-vfs/Cargo.toml b/crates/shared-vfs/Cargo.toml index aa3cbf3fb..c4fea701f 100644 --- a/crates/shared-vfs/Cargo.toml +++ b/crates/shared-vfs/Cargo.toml @@ -6,7 +6,7 @@ license.workspace = true repository.workspace = true publish = false -description = "PromptForge shared virtual filesystem machinery: canonical interned paths, claims, routing, and backends" +description = "PromptForge shared virtual filesystem machinery: canonical shared paths, claims, routing, and backends" # Zero-dependency rule: std only. No dependencies, workspace or external. # The manifest test enforces this; never weaken it. diff --git a/crates/shared-vfs/src/handle.rs b/crates/shared-vfs/src/handle.rs index b9332f741..840a8a20c 100644 --- a/crates/shared-vfs/src/handle.rs +++ b/crates/shared-vfs/src/handle.rs @@ -78,13 +78,13 @@ impl Claims { /// never conflicts. An identity never conflicts with itself. fn claim(&self, path: VfsPath, id: ExecId, kind: ClaimKind) -> Result<(), VfsError> { let mut tables = self.tables(); - if let Some(other) = other_claimant(&tables.writers, path, id) { - return Err(conflict(path, id, kind, other, ClaimKind::Write)); + if let Some(other) = other_claimant(&tables.writers, &path, id) { + return Err(conflict(&path, id, kind, other, ClaimKind::Write)); } if kind == ClaimKind::Write - && let Some(other) = other_claimant(&tables.readers, path, id) + && let Some(other) = other_claimant(&tables.readers, &path, id) { - return Err(conflict(path, id, kind, other, ClaimKind::Read)); + return Err(conflict(&path, id, kind, other, ClaimKind::Read)); } let map = match kind { ClaimKind::Read => &mut tables.readers, @@ -115,10 +115,10 @@ impl Claims { /// Returns the first claimant of `path` in `map` other than `id`. fn other_claimant( map: &HashMap>, - path: VfsPath, + path: &VfsPath, id: ExecId, ) -> Option { - map.get(&path)?.iter().find(|&&other| other != id).copied() + map.get(path)?.iter().find(|&&other| other != id).copied() } /// Removes every claim held by `id`, dropping emptied path entries. @@ -137,7 +137,7 @@ fn delete_claims(tables: &mut ClaimsTables, id: ExecId) { /// kinds: the executor maps it to a fatal run error, and the message is /// the whole diagnosis. fn conflict( - path: VfsPath, + path: &VfsPath, id: ExecId, kind: ClaimKind, other: ExecId, @@ -527,26 +527,28 @@ impl Access { /// backend fails. pub fn grep(&self, query: &GrepQuery) -> Result { let root = canonicalize(query.root.as_str())?; - self.check_policy(Op::Grep, root)?; + self.check_policy(Op::Grep, &root)?; self.volume.claims.claim(root, self.id, ClaimKind::Read)?; self.inner().grep(query) } /// Canonicalizes at receipt, consults the policy, then registers the /// claim - in that order, so a denied operation never registers a - /// claim and every claim key is the canonical interned path. + /// claim and every claim key is the canonical path. fn gate(&self, op: Op, path: &str, claim: ClaimKind) -> Result { let path = canonicalize(path)?; - self.check_policy(op, path)?; - self.volume.claims.claim(path, self.id, claim)?; + self.check_policy(op, &path)?; + // The claims table holds its own clone: the string frees when + // the claim and every other owner drop. + self.volume.claims.claim(path.clone(), self.id, claim)?; Ok(path) } /// Consults the handle's policy. v1 maps `Ask` to `PermissionDenied`: /// the approval dialog is a host concern above this layer, and the /// reason string still names what was asked and which rule fired. - fn check_policy(&self, op: Op, path: VfsPath) -> Result<(), VfsError> { - match self.policy.check(op, &path) { + fn check_policy(&self, op: Op, path: &VfsPath) -> Result<(), VfsError> { + match self.policy.check(op, path) { Verdict::Allow => Ok(()), Verdict::Deny(reason) | Verdict::Ask(reason) => Err(VfsError::PermissionDenied(reason)), } diff --git a/crates/shared-vfs/src/host.rs b/crates/shared-vfs/src/host.rs index 8c4d343aa..185670298 100644 --- a/crates/shared-vfs/src/host.rs +++ b/crates/shared-vfs/src/host.rs @@ -103,7 +103,7 @@ fn join_virtual(root: &Path, virtual_path: &str) -> PathBuf { /// is canonicalized and must sit under the (already canonical) root; /// the missing tail is re-appended lexically. This catches link escapes /// for existing paths while still resolving paths yet to be created. -fn contain(root: &Path, candidate: &Path, original: VfsPath) -> Result { +fn contain(root: &Path, candidate: &Path, original: &VfsPath) -> Result { let denied = || VfsError::PermissionDenied(format!("{original} escapes the mounted root")); let mut ancestor = candidate; let mut tail: Vec<&std::ffi::OsStr> = Vec::new(); @@ -169,7 +169,7 @@ fn atomic_write(dest: &Path, contents: &[u8]) -> Result<(), VfsError> { /// Creates the destination's ancestor directories, matching the memory /// backend's materialize-on-write semantics. -fn create_parent(host: &Path, path: VfsPath) -> Result<(), VfsError> { +fn create_parent(host: &Path, path: &VfsPath) -> Result<(), VfsError> { if let Some(parent) = host.parent() { fs::create_dir_all(parent).map_err(|err| map_io(path.as_str(), &err))?; } @@ -358,7 +358,7 @@ struct HostAccess { impl HostAccess { /// Resolves a canonical virtual path to its host path, applying /// containment in rooted mode. - fn resolve(&self, path: VfsPath) -> Result { + fn resolve(&self, path: &VfsPath) -> Result { match &self.root { HostRoot::Identity => Ok(identity_to_host(path.as_str())), HostRoot::Rooted(root) => { @@ -390,7 +390,7 @@ impl HostAccess { /// Rejects mutations on a read-only backend before anything is /// touched: a denied operation never partially applies. - fn check_writable(&self, path: VfsPath) -> Result<(), VfsError> { + fn check_writable(&self, path: &VfsPath) -> Result<(), VfsError> { if self.read_only { return Err(VfsError::PermissionDenied(format!( "the host backend is read-only, so {path} cannot be mutated" @@ -402,7 +402,7 @@ impl HostAccess { impl VfsAccess for HostAccess { fn read(&self, path: &VfsPath) -> Result, VfsError> { - let host = self.resolve(*path)?; + let host = self.resolve(path)?; if host.is_dir() { return Err(VfsError::IsADirectory(path.to_string())); } @@ -411,7 +411,7 @@ impl VfsAccess for HostAccess { fn read_range(&self, path: &VfsPath, offset: u64, len: u64) -> Result, VfsError> { // Seek, never materialize: the host can position directly. - let host = self.resolve(*path)?; + let host = self.resolve(path)?; if host.is_dir() { return Err(VfsError::IsADirectory(path.to_string())); } @@ -426,22 +426,22 @@ impl VfsAccess for HostAccess { } fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { - self.check_writable(*path)?; - let host = self.resolve(*path)?; + self.check_writable(path)?; + let host = self.resolve(path)?; if host.is_dir() { return Err(VfsError::IsADirectory(path.to_string())); } - create_parent(&host, *path)?; + create_parent(&host, path)?; atomic_write(&host, contents) } fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { - self.check_writable(*path)?; - let host = self.resolve(*path)?; + self.check_writable(path)?; + let host = self.resolve(path)?; if host.is_dir() { return Err(VfsError::IsADirectory(path.to_string())); } - create_parent(&host, *path)?; + create_parent(&host, path)?; let mut file = fs::OpenOptions::new() .create(true) .append(true) @@ -452,13 +452,13 @@ impl VfsAccess for HostAccess { } fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { - self.check_writable(*path)?; + self.check_writable(path)?; if path.as_str() == "/" { return Err(VfsError::PermissionDenied( "the mounted root cannot be removed".into(), )); } - let host = self.resolve(*path)?; + let host = self.resolve(path)?; let metadata = fs::symlink_metadata(&host).map_err(|err| map_io(path.as_str(), &err))?; // symlink_metadata does not follow links: a symlink is removed // as a link, never its target. @@ -475,7 +475,7 @@ impl VfsAccess for HostAccess { } fn exists(&self, path: &VfsPath) -> Result { - let host = self.resolve(*path)?; + let host = self.resolve(path)?; // symlink_metadata counts a dangling link as existing. Only a // confirmed absence is Ok(false); every other failure (a // denied permission, a genuine I/O error) surfaces as Err, as @@ -499,7 +499,7 @@ impl VfsAccess for HostAccess { ))); } let tokens = compile_glob(pattern.as_bytes()); - let root = self.resolve(canonicalize(walk_root(pattern))?)?; + let root = self.resolve(&canonicalize(walk_root(pattern))?)?; if !root.is_dir() { return Ok(Vec::new()); } @@ -515,7 +515,7 @@ impl VfsAccess for HostAccess { } fn list(&self, path: &VfsPath) -> Result, VfsError> { - let host = self.resolve(*path)?; + let host = self.resolve(path)?; let entries = fs::read_dir(&host).map_err(|err| map_io(path.as_str(), &err))?; let mut result = Vec::new(); for entry in entries { @@ -535,14 +535,14 @@ impl VfsAccess for HostAccess { } fn stat(&self, path: &VfsPath) -> Result { - let host = self.resolve(*path)?; + let host = self.resolve(path)?; let metadata = fs::symlink_metadata(&host).map_err(|err| map_io(path.as_str(), &err))?; Ok(stat_of(&metadata)) } fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { - self.check_writable(*path)?; - let host = self.resolve(*path)?; + self.check_writable(path)?; + let host = self.resolve(path)?; if fs::symlink_metadata(&host).is_ok() { return Err(VfsError::AlreadyExists(path.to_string())); } @@ -555,7 +555,7 @@ impl VfsAccess for HostAccess { } fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { - self.check_writable(*from)?; + self.check_writable(from)?; if from.as_str() == "/" { return Err(VfsError::PermissionDenied( "the mounted root cannot be renamed".into(), @@ -571,19 +571,19 @@ impl VfsAccess for HostAccess { "cannot rename {from} into its own descendant {to}" ))); } - let host_from = self.resolve(*from)?; - let host_to = self.resolve(*to)?; + let host_from = self.resolve(from)?; + let host_to = self.resolve(to)?; // Validation finishes before the rename syscall, so a failed // rename changes nothing; the rename itself is atomic. fs::symlink_metadata(&host_from).map_err(|err| map_io(from.as_str(), &err))?; - create_parent(&host_to, *to)?; + create_parent(&host_to, to)?; fs::rename(&host_from, &host_to).map_err(|err| map_io(from.as_str(), &err)) } fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { - self.check_writable(*to)?; - let host_from = self.resolve(*from)?; - let host_to = self.resolve(*to)?; + self.check_writable(to)?; + let host_from = self.resolve(from)?; + let host_to = self.resolve(to)?; if host_from.is_dir() { return Err(VfsError::IsADirectory(from.to_string())); } @@ -591,7 +591,7 @@ impl VfsAccess for HostAccess { if host_to.is_dir() { return Err(VfsError::IsADirectory(to.to_string())); } - create_parent(&host_to, *to)?; + create_parent(&host_to, to)?; atomic_write(&host_to, &bytes) } } diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs index 9ec946bc4..9f17c1791 100644 --- a/crates/shared-vfs/src/lib.rs +++ b/crates/shared-vfs/src/lib.rs @@ -26,6 +26,20 @@ pub use types::{Entry, FileType, GrepMatch, GrepQuery, GrepResults, Stat}; #[cfg(test)] mod tests { + /// A manifest section is a dependency table when it is exactly one of + /// the three dependency tables, a sub-table of one + /// (`[dependencies.foo]` declares a dependency the same way), or a + /// target-qualified dependency table. + fn is_dependency_table(section: &str) -> bool { + const TABLES: [&str; 3] = ["dependencies", "dev-dependencies", "build-dependencies"]; + TABLES.iter().any(|table| { + section == *table + || section + .strip_prefix(*table) + .is_some_and(|rest| rest.starts_with('.')) + }) || (section.starts_with("target.") && section.ends_with(".dependencies")) + } + /// The zero-dependency rule is load-bearing: this crate compiles alone /// and never rebuilds for a dependency rev, so the manifest must never /// declare a dependency. This test reads the crate's own Cargo.toml and @@ -45,15 +59,31 @@ mod tests { if line.is_empty() || line.starts_with('#') { continue; } - let is_dependency_table = section == "dependencies" - || section == "dev-dependencies" - || section == "build-dependencies" - || (section.starts_with("target.") && section.ends_with(".dependencies")); assert!( - !is_dependency_table, + !is_dependency_table(§ion), "zero-dependency rule violated: [{section}] declares `{line}`" ); } Ok(()) } + + #[test] + fn dependency_sub_tables_count_as_dependency_tables() { + // Regression: `[dependencies.foo]` once slipped past the exact- + // match section check while still declaring a dependency. + for section in [ + "dependencies", + "dependencies.foo", + "dev-dependencies", + "dev-dependencies.foo", + "build-dependencies", + "build-dependencies.foo", + "target.'cfg(windows)'.dependencies", + ] { + assert!(is_dependency_table(section), "[{section}] must be caught"); + } + for section in ["package", "lints", "features", "dependenciesfoo"] { + assert!(!is_dependency_table(section), "[{section}] must pass"); + } + } } diff --git a/crates/shared-vfs/src/path.rs b/crates/shared-vfs/src/path.rs index 76085d30b..a3d5f0015 100644 --- a/crates/shared-vfs/src/path.rs +++ b/crates/shared-vfs/src/path.rs @@ -1,77 +1,30 @@ -//! Canonical, interned virtual paths. +//! Canonical, shared virtual paths. //! -//! Paths are canonicalized at the moment the API receives them and interned, -//! so claim lookups are pointer-cheap and aliases cannot slip past the +//! Paths are canonicalized at the moment the API receives them, so claim +//! lookups compare one canonical form and aliases cannot slip past the //! claims tables. The only way to form a [`VfsPath`] is through //! `canonicalize`, which is crate-private: canonicalization at receipt is //! enforced by visibility, not convention. -use std::collections::HashMap; use std::fmt; -use std::sync::{Mutex, MutexGuard, OnceLock, PoisonError}; +use std::sync::Arc; use crate::error::VfsError; -/// Hand-rolled string interner on std. Strings are leaked once each, so -/// resolution is a vector index and equality is an integer compare. -struct Interner { - ids: HashMap<&'static str, u32>, - strings: Vec<&'static str>, -} - -impl Interner { - fn new() -> Self { - Self { - ids: HashMap::new(), - strings: Vec::new(), - } - } - - fn intern(&mut self, s: &str) -> u32 { - if let Some(&id) = self.ids.get(s) { - return id; - } - let leaked: &'static str = Box::leak(s.into()); - let Ok(id) = u32::try_from(self.strings.len()) else { - panic!("vfs path interner exhausted") - }; - self.strings.push(leaked); - self.ids.insert(leaked, id); - id - } - - fn resolve(&self, id: u32) -> &'static str { - match self.strings.get(id as usize) { - Some(s) => s, - None => panic!("vfs path id {id} was never interned"), - } - } -} - -/// Poison-safe lock: each guard scope is one complete mutation, so a -/// panicking writer cannot leave the tables half-updated and recovery is -/// safe. -fn interner() -> MutexGuard<'static, Interner> { - static INTERNER: OnceLock> = OnceLock::new(); - INTERNER - .get_or_init(|| Mutex::new(Interner::new())) - .lock() - .unwrap_or_else(PoisonError::into_inner) -} - -/// Canonical, interned virtual path. Produced by `canonicalize` at the -/// moment the API receives a path; interning makes claim lookups -/// pointer-cheap and guarantees alias detection. -#[derive(Clone, Copy, PartialEq, Eq, Hash)] +/// Canonical virtual path. Produced by `canonicalize` at the moment the +/// API receives a path. The string is `Arc`-shared per value lineage: +/// clones share one allocation, and the string frees when its last +/// owner drops. There is no global table and no lock. +#[derive(Clone, PartialEq, Eq, Hash)] pub struct VfsPath { - id: u32, + text: Arc, } impl VfsPath { /// Returns the canonical string for this path. #[must_use] - pub fn as_str(&self) -> &'static str { - interner().resolve(self.id) + pub fn as_str(&self) -> &str { + &self.text } /// Returns an owned copy of this path. @@ -162,12 +115,15 @@ pub(crate) fn canonicalize(path: &str) -> Result { } s }; - let id = interner().intern(&canonical); - Ok(VfsPath { id }) + Ok(VfsPath { + text: canonical.into(), + }) } #[cfg(test)] mod tests { + use std::sync::Arc; + use super::{VfsPath, canonicalize}; use crate::VfsError; @@ -235,11 +191,40 @@ mod tests { } #[test] - fn identical_paths_intern_to_one_entry() -> Result<(), VfsError> { + fn identical_paths_canonicalize_to_equal_values() -> Result<(), VfsError> { let first = canonicalize("/a/b")?; let second = canonicalize("/a/./b/")?; assert_eq!(first, second); - assert!(std::ptr::eq(first.as_str(), second.as_str())); + assert_eq!(first.as_str(), second.as_str()); + Ok(()) + } + + #[test] + fn clones_share_one_allocation() -> Result<(), VfsError> { + let path = canonicalize("/a/b")?; + let clone = path.clone(); + assert_eq!(Arc::strong_count(&path.text), 2); + drop(clone); + assert_eq!(Arc::strong_count(&path.text), 1); + Ok(()) + } + + #[test] + fn canonicalizing_distinct_paths_in_a_loop_does_not_retain_their_strings() + -> Result<(), VfsError> { + // Regression: the interner leaked every distinct string for the + // process's life, so a loop like this grew the heap + // monotonically. Each path's string must free with its last + // owner - here, at the end of its own iteration. + let mut dangling = Vec::new(); + for index in 0..1000 { + let path = canonicalize(&format!("/loop/{index}"))?; + dangling.push(Arc::downgrade(&path.text)); + } + assert!( + dangling.iter().all(|weak| weak.upgrade().is_none()), + "a dropped path's string must free with its last owner" + ); Ok(()) } } diff --git a/crates/shared-vfs/src/router.rs b/crates/shared-vfs/src/router.rs index 00e9a1a43..67b793495 100644 --- a/crates/shared-vfs/src/router.rs +++ b/crates/shared-vfs/src/router.rs @@ -131,7 +131,7 @@ impl RoutingAccess { /// acquiring that session on first touch. fn with_mount( &self, - path: VfsPath, + path: &VfsPath, op: impl FnOnce(&mut dyn VfsAccess) -> Result, ) -> Result { let (prefix, backend) = resolve(&self.mounts, path.as_str()) @@ -148,7 +148,7 @@ impl RoutingAccess { } /// The mount-relative path the serving backend sees. - fn strip(&self, path: VfsPath) -> Result { + fn strip(&self, path: &VfsPath) -> Result { let (prefix, _) = resolve(&self.mounts, path.as_str()) .ok_or_else(|| VfsError::NotFound(format!("no mount serves {path}")))?; canonicalize(strip_mount(prefix.as_str(), path.as_str())) @@ -156,7 +156,7 @@ impl RoutingAccess { /// Rejects mutations on read-only mounts before anything is /// touched: a denied operation never partially applies. - fn check_writable(&self, path: VfsPath) -> Result<(), VfsError> { + fn check_writable(&self, path: &VfsPath) -> Result<(), VfsError> { let (prefix, backend) = resolve(&self.mounts, path.as_str()) .ok_or_else(|| VfsError::NotFound(format!("no mount serves {path}")))?; if lock(backend).read_only() { @@ -169,7 +169,7 @@ impl RoutingAccess { /// Two-path operations require one mount: backend atomicity /// guarantees stop at the mount boundary. - fn one_mount(&self, from: VfsPath, to: VfsPath, op: &str) -> Result<(), VfsError> { + fn one_mount(&self, from: &VfsPath, to: &VfsPath, op: &str) -> Result<(), VfsError> { let from_prefix = resolve(&self.mounts, from.as_str()) .ok_or_else(|| VfsError::NotFound(format!("no mount serves {from}")))? .0; @@ -187,38 +187,38 @@ impl RoutingAccess { impl VfsAccess for RoutingAccess { fn read(&self, path: &VfsPath) -> Result, VfsError> { - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.read(&stripped)) + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.read(&stripped)) } fn read_range(&self, path: &VfsPath, offset: u64, len: u64) -> Result, VfsError> { // Delegated, not defaulted, so backends that can seek never // materialize the file. - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.read_range(&stripped, offset, len)) + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.read_range(&stripped, offset, len)) } fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { - self.check_writable(*path)?; - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.write(&stripped, contents)) + self.check_writable(path)?; + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.write(&stripped, contents)) } fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { - self.check_writable(*path)?; - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.append(&stripped, contents)) + self.check_writable(path)?; + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.append(&stripped, contents)) } fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { - self.check_writable(*path)?; - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.remove(&stripped, recursive)) + self.check_writable(path)?; + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.remove(&stripped, recursive)) } fn exists(&self, path: &VfsPath) -> Result { - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.exists(&stripped)) + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.exists(&stripped)) } fn glob(&self, pattern: &str) -> Result, VfsError> { @@ -229,7 +229,7 @@ impl VfsAccess for RoutingAccess { let (prefix, _) = resolve(&self.mounts, canonical.as_str()) .ok_or_else(|| VfsError::NotFound(format!("no mount serves {canonical}")))?; let scoped = strip_mount(prefix.as_str(), canonical.as_str()).to_owned(); - let mut matches = self.with_mount(canonical, |session| session.glob(&scoped))?; + let mut matches = self.with_mount(&canonical, |session| session.glob(&scoped))?; for path in &mut matches { *path = rejoin(prefix.as_str(), path); } @@ -237,46 +237,46 @@ impl VfsAccess for RoutingAccess { } fn list(&self, path: &VfsPath) -> Result, VfsError> { - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.list(&stripped)) + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.list(&stripped)) } fn stat(&self, path: &VfsPath) -> Result { - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.stat(&stripped)) + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.stat(&stripped)) } fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { - self.check_writable(*path)?; - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.mkdir(&stripped, recursive)) + self.check_writable(path)?; + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.mkdir(&stripped, recursive)) } fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { - self.check_writable(*from)?; - self.check_writable(*to)?; - self.one_mount(*from, *to, "rename")?; - let from_stripped = self.strip(*from)?; - let to_stripped = self.strip(*to)?; - self.with_mount(*from, |session| { + self.check_writable(from)?; + self.check_writable(to)?; + self.one_mount(from, to, "rename")?; + let from_stripped = self.strip(from)?; + let to_stripped = self.strip(to)?; + self.with_mount(from, |session| { session.rename(&from_stripped, &to_stripped) }) } fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { - self.check_writable(*to)?; - self.one_mount(*from, *to, "copy")?; - let from_stripped = self.strip(*from)?; - let to_stripped = self.strip(*to)?; - self.with_mount(*from, |session| session.copy(&from_stripped, &to_stripped)) + self.check_writable(to)?; + self.one_mount(from, to, "copy")?; + let from_stripped = self.strip(from)?; + let to_stripped = self.strip(to)?; + self.with_mount(from, |session| session.copy(&from_stripped, &to_stripped)) } fn str_replace(&mut self, path: &VfsPath, old: &str, new: &str) -> Result<(), VfsError> { // Delegated so backends can push down; the read-only check here // covers the backend's default read-plus-write as well. - self.check_writable(*path)?; - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.str_replace(&stripped, old, new)) + self.check_writable(path)?; + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.str_replace(&stripped, old, new)) } fn grep(&self, query: &GrepQuery) -> Result { @@ -285,7 +285,7 @@ impl VfsAccess for RoutingAccess { .ok_or_else(|| VfsError::NotFound(format!("no mount serves {root}")))?; let mut scoped = query.clone(); scoped.root = canonicalize(strip_mount(prefix.as_str(), root.as_str()))?.to_buf(); - let mut results = self.with_mount(root, |session| session.grep(&scoped))?; + let mut results = self.with_mount(&root, |session| session.grep(&scoped))?; for hit in &mut results.matches { hit.path = rejoin(prefix.as_str(), &hit.path); } @@ -293,21 +293,21 @@ impl VfsAccess for RoutingAccess { } fn symlink(&mut self, target: &VfsPath, link: &VfsPath) -> Result<(), VfsError> { - self.check_writable(*link)?; - let stripped = self.strip(*link)?; + self.check_writable(link)?; + let stripped = self.strip(link)?; // The target is a stored name, not resolved: it passes verbatim. - self.with_mount(*link, |session| session.symlink(target, &stripped)) + self.with_mount(link, |session| session.symlink(target, &stripped)) } fn read_link(&self, path: &VfsPath) -> Result { - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.read_link(&stripped)) + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.read_link(&stripped)) } fn chmod(&mut self, path: &VfsPath, mode: u32) -> Result<(), VfsError> { - self.check_writable(*path)?; - let stripped = self.strip(*path)?; - self.with_mount(*path, |session| session.chmod(&stripped, mode)) + self.check_writable(path)?; + let stripped = self.strip(path)?; + self.with_mount(path, |session| session.chmod(&stripped, mode)) } } @@ -460,7 +460,7 @@ mod tests { self.files.lock().unwrap_or_else(PoisonError::into_inner) } - fn record(&self, path: VfsPath) { + fn record(&self, path: &VfsPath) { self.seen .lock() .unwrap_or_else(PoisonError::into_inner) @@ -470,7 +470,7 @@ mod tests { impl VfsAccess for StubAccess { fn read(&self, path: &VfsPath) -> Result, VfsError> { - self.record(*path); + self.record(path); self.files() .get(path.as_str()) .cloned() @@ -478,13 +478,13 @@ mod tests { } fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { - self.record(*path); + self.record(path); self.files().insert(path.to_string(), contents.to_vec()); Ok(()) } fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { - self.record(*path); + self.record(path); self.files() .entry(path.to_string()) .or_default() @@ -494,7 +494,7 @@ mod tests { fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { let _ = recursive; - self.record(*path); + self.record(path); self.files() .remove(path.as_str()) .map(|_| ()) @@ -502,7 +502,7 @@ mod tests { } fn exists(&self, path: &VfsPath) -> Result { - self.record(*path); + self.record(path); Ok(self.files().contains_key(path.as_str())) } @@ -534,8 +534,8 @@ mod tests { } fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { - self.record(*from); - self.record(*to); + self.record(from); + self.record(to); let bytes = self .files() .remove(from.as_str()) @@ -545,8 +545,8 @@ mod tests { } fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { - self.record(*from); - self.record(*to); + self.record(from); + self.record(to); let bytes = self .files() .get(from.as_str()) diff --git a/vibe/2026-09-12-1-test-namespace-vfs-debt.md b/vibe/2026-09-12-1-test-namespace-vfs-debt.md index 0880e307d..06799f12b 100644 --- a/vibe/2026-09-12-1-test-namespace-vfs-debt.md +++ b/vibe/2026-09-12-1-test-namespace-vfs-debt.md @@ -168,7 +168,7 @@ Each step is one commit containing its code and tests. -### Step 6: de-intern VfsPath, close the manifest test bypass, and correct archdoc +### Step 6: de-intern VfsPath, close the manifest test bypass, and correct archdoc [completed] - Component: vfs-debt-removal - In `promptforge/crates/shared-vfs/src/path.rs`: remove the `Interner` and the `OnceLock>` global; give `VfsPath` an `Arc` field so `canonicalize` allocates one `Arc` per call and the string frees when its last owner drops. `VfsPath` loses `Copy`; `VfsPath::as_str` returns `&str` borrowed from self instead of `&'static str`. Update claim sites to hold clones, change the `identical_paths_intern_to_one_entry` property test's pointer-equality assertion to content equality, and add the regression check that a loop canonicalizing distinct paths does not grow the heap monotonically. diff --git a/vibe/archdoc.md b/vibe/archdoc.md index e992195b0..c37919541 100644 --- a/vibe/archdoc.md +++ b/vibe/archdoc.md @@ -10,7 +10,8 @@ PromptForge is a Rust system for executing Markdown prompt pipelines and Lua age - gateway: independent server process that owns model routing, provider access, and local inference lifecycle; exposes protocol data and discovery; depends on: shared substrate - CLI: thin shell adapter that supplies inputs and host resources to the executor; depends on: executor, gateway, store, shared substrate - workshop UI: desktop authoring shell and in-process server that host the executor and attach over the gateway protocol; depends on: executor, gateway, store, shared substrate -- store: run-scoped virtual filesystem contract with interchangeable memory and file backends; depends on: none +- store: run-scoped Store facade over the VFS layer, exposed as `vfs.store(&access)`; depends on: VFS layer +- VFS layer: canonical paths, claims, routing, and memory and host backends (`shared-vfs`), plus the policy gate (`promptforge-vfs`); depends on: none - Lua VM boundary: sandbox and coroutine bridge between prompt code and host capabilities; depends on: gateway, store, shared substrate - shared substrate: cross-product progress, loopback discovery, protocol, and sidecar facilities; depends on: none From 432f268d0bc3e0479c7be4c849e3984163a6419c Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 03:06:10 -0700 Subject: [PATCH 18/26] Make handle acquisition fallible instead of panicking The virtual filesystem handle used to panic when a backend refused to vend an execution identity, even though the backend trait's contract has always allowed refusal. Acquisition and spawn now return errors instead: the executor surfaces a refusal as its store error, and the agent surfaces it as a program failure before the run starts. A failed spawn leaves the parent capability's claims registered, so the conflict bookkeeping stays intact for later identities. Callers, tests, and doc examples update mechanically to the fallible signatures. - `VfsRef::acquire` and `acquire_with` now return `Result` and propagate a backend refusal with `?`; the panic-on-refusal path is deleted. - `spawn` returns `Result`, and a failed spawn leaves the parent's claims registered: `a_backend_refusal_fails_spawn_and_keeps_the_parents_claims` shows a second identity still conflicting with the parent's write. - `install_root_slots` and the fan-out arm refill map a refusal to `Error::Store`; the agent's `drive` maps it to `AgentError::Program` with the source error attached. Design: new surface-growth @ crates/shared-vfs/src/handle.rs::VfsRef::acquire boundary: pub Design: new surface-growth @ crates/shared-vfs/src/handle.rs::Access::spawn boundary: pub Design: new shotgun-surgery @ crates/shared-vfs/src/handle.rs::VfsRef::acquire Repairs: Vfs::acquire's Result contract permits backend refusal @ crates/shared-vfs/src/handle.rs::VfsRef::acquire - a refusing backend panicked the process instead of returning the error Repairs: Vfs::acquire's Result contract permits backend refusal @ crates/shared-vfs/src/handle.rs::Access::spawn - a refusing backend panicked instead of returning the error Plan: vibe/2026-09-12-1-test-namespace-vfs-debt.md --- crates/promptforge-agent/src/agent.rs | 14 +- crates/promptforge-agent/src/tests.rs | 2 +- crates/promptforge-core/src/execute.rs | 2 +- .../promptforge-core/src/execute/scheduler.rs | 29 ++- .../src/execute/tests/exec_flow.rs | 4 +- .../promptforge-core/src/execute/tests/mod.rs | 10 +- crates/promptforge-core/src/lua/coro_tests.rs | 6 +- .../promptforge-core/src/model/tests/mod.rs | 6 +- .../promptforge-core/tests/suite/support.rs | 2 +- crates/promptforge-core/tests/suite/vfs.rs | 8 +- crates/promptforge-lua/benches/surface.rs | 6 +- crates/promptforge-lua/src/messages/tests.rs | 6 +- crates/promptforge-lua/src/models/tests.rs | 6 +- crates/promptforge-lua/src/tests.rs | 12 +- crates/promptforge-lua/src/tools/tests.rs | 6 +- crates/promptforge-lua/src/vm.rs | 4 +- crates/promptforge-store/src/error.rs | 6 +- crates/promptforge-store/src/lib.rs | 22 +- crates/promptforge-store/src/tests.rs | 10 +- crates/promptforge-vfs/src/lib.rs | 15 +- crates/shared-vfs/src/handle.rs | 213 ++++++++++++------ crates/shared-vfs/src/memory.rs | 2 +- crates/shared-vfs/src/router.rs | 31 ++- vibe/2026-09-12-1-test-namespace-vfs-debt.md | 2 +- 24 files changed, 276 insertions(+), 148 deletions(-) diff --git a/crates/promptforge-agent/src/agent.rs b/crates/promptforge-agent/src/agent.rs index aa5ffda3c..e78994854 100644 --- a/crates/promptforge-agent/src/agent.rs +++ b/crates/promptforge-agent/src/agent.rs @@ -61,8 +61,9 @@ pub enum AgentError { Interrupted, /// The agent program failed: a Lua compile or runtime error, an - /// exhausted Lua resource quota, a failed host contract, or a dispatch - /// failure the program did not catch. + /// exhausted Lua resource quota, a failed host contract, a dispatch + /// failure the program did not catch, or the store capability's + /// acquisition failure before the program started. #[error("{message}")] Program { /// The failure rendered as its location-tagged diagnostic. @@ -230,7 +231,10 @@ async fn drive( vm.apply_lua_limits(limits.lua_memory_bytes, limits.lua_log_events)?; // The agent is one serial thread of execution: one capability for the // whole run, released when it drops at the run's end. - let access = Arc::new(vfs.acquire()); + let access = Arc::new(vfs.acquire().map_err(|error| AgentError::Program { + message: format!("the store capability acquisition failed: {error}"), + source: Some(Box::new(error)), + })?); let (counts, events) = match setup_agent_vm(&mut vm, &access, &observer, &name, &tool_set, event_log, ui) { Ok(installed) => installed, @@ -905,7 +909,9 @@ mod tests { vfs: &VfsRef, path: &str, ) -> std::result::Result { - let access = vfs.acquire(); + let access = vfs + .acquire() + .map_err(promptforge_store::StoreError::backend)?; promptforge_store::StoreExt::store(vfs, &access).read(path) } diff --git a/crates/promptforge-agent/src/tests.rs b/crates/promptforge-agent/src/tests.rs index 344e3b421..4d32ea9e6 100644 --- a/crates/promptforge-agent/src/tests.rs +++ b/crates/promptforge-agent/src/tests.rs @@ -562,7 +562,7 @@ impl FixtureRun { /// immediately dropped access: the run's identity dropped with it, so /// nothing it wrote conflicts with the extraction. fn read(&self, path: &str) -> String { - let access = self.vfs.acquire(); + let access = self.vfs.acquire().expect("the stock backend acquires"); self.vfs .store(&access) .read(path) diff --git a/crates/promptforge-core/src/execute.rs b/crates/promptforge-core/src/execute.rs index fe4d71bca..2a419866f 100644 --- a/crates/promptforge-core/src/execute.rs +++ b/crates/promptforge-core/src/execute.rs @@ -272,7 +272,7 @@ pub async fn run( /// throwaway overlay. The probe's identity and claim release with the /// access. fn store_mount_present(vfs: &VfsRef) -> std::result::Result { - match vfs.acquire().stat(promptforge_vfs::STORE_MOUNT) { + match vfs.acquire()?.stat(promptforge_vfs::STORE_MOUNT) { Ok(_) => Ok(true), Err(shared_vfs::VfsError::NotFound(_)) => Ok(false), Err(error) => Err(error), diff --git a/crates/promptforge-core/src/execute/scheduler.rs b/crates/promptforge-core/src/execute/scheduler.rs index d3d7a46ab..bdca7175c 100644 --- a/crates/promptforge-core/src/execute/scheduler.rs +++ b/crates/promptforge-core/src/execute/scheduler.rs @@ -750,10 +750,11 @@ impl<'a> Scheduler<'a> { /// enqueues it. /// /// # Errors - /// Returns [`Error::Internal`] when the run's chain count exceeds `u32`. + /// Returns [`Error::Internal`] when the run's chain count exceeds `u32`, + /// or [`Error::Store`] when the backend refuses acquisition. fn start_root_walk(&mut self, sections: &'a [Section], var: &serde_json::Value) -> Result<()> { let root = self.start_chain(self.ctx.clone(), sections, 0, None, var, 0, None)?; - self.install_root_slots(root); + self.install_root_slots(root)?; self.ready.push_back(root); Ok(()) } @@ -765,9 +766,14 @@ impl<'a> Scheduler<'a> { /// the run's configured client, as the legacy walk's slot is seeded /// from run()'s client: a prose block before any infer must use it /// rather than fall back to building an environment client. - fn install_root_slots(&mut self, root: ChainId) { - self.chains[root.index()].access = Some(Arc::new(self.ctx.vfs().acquire())); + /// + /// # Errors + /// Returns [`Error::Store`] when the backend refuses acquisition. + fn install_root_slots(&mut self, root: ChainId) -> Result<()> { + let access = self.ctx.vfs().acquire().map_err(Error::Store)?; + self.chains[root.index()].access = Some(Arc::new(access)); self.chains[root.index()].client = self.client.ready().cloned(); + Ok(()) } /// Starts the live H1 pass as the driver loop's first chain: the @@ -775,7 +781,8 @@ impl<'a> Scheduler<'a> { /// coroutine machinery as any section under the live pass's rules. /// /// # Errors - /// Returns [`Error::Internal`] when the run's chain count exceeds `u32`. + /// Returns [`Error::Internal`] when the run's chain count exceeds `u32`, + /// or [`Error::Store`] when the backend refuses acquisition. fn start_live_h1(&mut self) -> Result { let id = ChainId( u32::try_from(self.chains.len()) @@ -784,9 +791,10 @@ impl<'a> Scheduler<'a> { // The pass owns its client slot, seeded from the run's configured // client, exactly as the legacy pass seeds its own. let client = self.client.ready().cloned(); + let access = self.ctx.vfs().acquire().map_err(Error::Store)?; self.chains.push(Chain { ctx: self.ctx.clone(), - access: Some(Arc::new(self.ctx.vfs().acquire())), + access: Some(Arc::new(access)), frame: None, slice: &[], index: 0, @@ -815,6 +823,7 @@ impl<'a> Scheduler<'a> { /// # Errors /// Returns [`Error::Lua`] when the final `var` read-back fails, /// [`Error::TimestampFormat`] when the walk's `when` fails to format, + /// [`Error::Store`] when the backend refuses the walk's acquisition, /// or [`Error::Internal`] when the chain holds no frame. fn end_live_h1(&mut self, id: ChainId, root_result: &mut Option>) -> Result<()> { let chain = &mut self.chains[id.index()]; @@ -836,7 +845,7 @@ impl<'a> Scheduler<'a> { let when = now_rfc3339_checked()?; let walk_ctx = self.ctx.with_walk_state(&when); let root = self.start_chain(walk_ctx, sections, 0, None, &var, 0, None)?; - self.install_root_slots(root); + self.install_root_slots(root)?; self.ready.push_back(root); Ok(()) } @@ -2239,7 +2248,8 @@ impl<'a> Scheduler<'a> { /// /// # Errors /// Returns [`Error::Internal`] when the join is not live or the run's - /// chain count exceeds `u32`. + /// chain count exceeds `u32`, or [`Error::Store`] when the backend + /// refuses an arm's acquisition. fn refill_fanout(&mut self, fanout: FanoutId) -> Result<()> { loop { let (index, item, template) = { @@ -2295,7 +2305,8 @@ impl<'a> Scheduler<'a> { // caller's claims (the happens-before edge), and drops with // the chain so a finished arm's claims never linger into the // join's merge. - self.chains[chain.index()].access = Some(Arc::new(template.access.spawn())); + let access = template.access.spawn().map_err(Error::Store)?; + self.chains[chain.index()].access = Some(Arc::new(access)); // The arm inherits the caller's client slot: an // already-resolved client is shared, an unresolved one stays // lazy. diff --git a/crates/promptforge-core/src/execute/tests/exec_flow.rs b/crates/promptforge-core/src/execute/tests/exec_flow.rs index c96d66264..5f9c2568f 100644 --- a/crates/promptforge-core/src/execute/tests/exec_flow.rs +++ b/crates/promptforge-core/src/execute/tests/exec_flow.rs @@ -2318,7 +2318,9 @@ async fn a_mount_less_handle_runs_on_the_defensive_store_overlay() { // or the run's writes. assert!( matches!( - vfs.acquire().stat(promptforge_vfs::STORE_MOUNT), + vfs.acquire() + .expect("the stock backend acquires") + .stat(promptforge_vfs::STORE_MOUNT), Err(shared_vfs::VfsError::NotFound(_)) ), "the run's writes must land on the overlay, not the caller's backend" diff --git a/crates/promptforge-core/src/execute/tests/mod.rs b/crates/promptforge-core/src/execute/tests/mod.rs index 399057b9b..6ebfa0a6a 100644 --- a/crates/promptforge-core/src/execute/tests/mod.rs +++ b/crates/promptforge-core/src/execute/tests/mod.rs @@ -36,7 +36,11 @@ use crate::untrusted::GuardNonce; /// A fresh stock handle's access capability, for tests that inject host /// values into a standalone VM. fn fresh_access() -> Arc { - Arc::new(promptforge_vfs::empty().acquire()) + Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ) } const EXECUTION: &str = "execute-test"; @@ -227,12 +231,12 @@ impl TestStore { } fn read(&self, path: &str) -> std::result::Result { - let access = self.0.acquire(); + let access = self.0.acquire().map_err(StoreError::backend)?; self.0.store(&access).read(path) } fn glob(&self, pattern: &str) -> std::result::Result, StoreError> { - let access = self.0.acquire(); + let access = self.0.acquire().map_err(StoreError::backend)?; self.0.store(&access).glob(pattern) } } diff --git a/crates/promptforge-core/src/lua/coro_tests.rs b/crates/promptforge-core/src/lua/coro_tests.rs index 0f794c717..7963eda7c 100644 --- a/crates/promptforge-core/src/lua/coro_tests.rs +++ b/crates/promptforge-core/src/lua/coro_tests.rs @@ -114,7 +114,11 @@ fn scheduler_vm_with_tools( .expect("the section VM builds"); let shared = LuaProgram::empty().expect("the empty shared program compiles"); let sys = json!({}); - let access = Arc::new(promptforge_vfs::empty().acquire()); + let access = Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ); let setup = SectionVmSetup { args: "", sys: &sys, diff --git a/crates/promptforge-core/src/model/tests/mod.rs b/crates/promptforge-core/src/model/tests/mod.rs index 06e8a3652..8f51332a1 100644 --- a/crates/promptforge-core/src/model/tests/mod.rs +++ b/crates/promptforge-core/src/model/tests/mod.rs @@ -21,7 +21,11 @@ const EXECUTION: &str = "model-bind-test"; /// A fresh stock handle's access capability, for tests that inject host /// values into a standalone VM. fn fresh_access() -> Arc { - Arc::new(promptforge_vfs::empty().acquire()) + Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ) } fn ctx(window: u32) -> NonZeroU32 { diff --git a/crates/promptforge-core/tests/suite/support.rs b/crates/promptforge-core/tests/suite/support.rs index 9d72096c5..b5dae12e3 100644 --- a/crates/promptforge-core/tests/suite/support.rs +++ b/crates/promptforge-core/tests/suite/support.rs @@ -111,7 +111,7 @@ pub(super) struct FixtureStore(VfsRef); impl FixtureStore { /// Reads a store path through a fresh, immediately dropped access. pub(super) fn read(&self, path: &str) -> Result { - let access = self.0.acquire(); + let access = self.0.acquire().map_err(StoreError::backend)?; self.0.store(&access).read(path) } } diff --git a/crates/promptforge-core/tests/suite/vfs.rs b/crates/promptforge-core/tests/suite/vfs.rs index 5c2d3cee3..e74cf08ca 100644 --- a/crates/promptforge-core/tests/suite/vfs.rs +++ b/crates/promptforge-core/tests/suite/vfs.rs @@ -79,7 +79,7 @@ fn seed_declared_input(vfs: &VfsRef, prompt: &Prompt, contents: &str) { .frontmatter() .input() .expect("the fixture declares an input"); - let access = vfs.acquire(); + let access = vfs.acquire().expect("the stock backend acquires"); vfs.store(&access) .write(input.path(), contents) .expect("the declared input seeds"); @@ -141,7 +141,7 @@ return store.read('handoff.txt')\n\ assert_eq!(result, "across the reset"); // Extraction after the run takes a fresh access: the run's identities // dropped with it, so nothing the run touched can conflict here. - let access = vfs.acquire(); + let access = vfs.acquire().expect("the stock backend acquires"); assert_eq!( vfs.store(&access) .read("handoff.txt") @@ -165,7 +165,7 @@ async fn a_host_seeds_and_extracts_through_the_stock_handle_with_no_real_files() .await .expect("the seeded run executes offline"); assert_eq!(result, "done"); - let access = vfs.acquire(); + let access = vfs.acquire().expect("the stock backend acquires"); let report = extract_declared_output(&vfs.store(&access), &prompt) .expect("the run left its promised output"); assert_eq!(report, "report on: the paper body"); @@ -188,7 +188,7 @@ async fn a_missing_declared_output_is_a_contract_error_naming_the_prompts_promis .await .expect("the run itself succeeds"); assert_eq!(result, "read: the paper body"); - let access = vfs.acquire(); + let access = vfs.acquire().expect("the stock backend acquires"); let error = extract_declared_output(&vfs.store(&access), &prompt) .expect_err("the missing output is a contract error"); assert!( diff --git a/crates/promptforge-lua/benches/surface.rs b/crates/promptforge-lua/benches/surface.rs index 4894a5507..6583438f8 100644 --- a/crates/promptforge-lua/benches/surface.rs +++ b/crates/promptforge-lua/benches/surface.rs @@ -44,7 +44,11 @@ fn builder_vm() -> SectionVm { vm.inject_host( "", &json!({}), - &std::sync::Arc::new(promptforge_vfs::empty().acquire()), + &std::sync::Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ), ) .expect("host injection installs the messages namespace"); vm diff --git a/crates/promptforge-lua/src/messages/tests.rs b/crates/promptforge-lua/src/messages/tests.rs index b27c04edf..99744669d 100644 --- a/crates/promptforge-lua/src/messages/tests.rs +++ b/crates/promptforge-lua/src/messages/tests.rs @@ -9,7 +9,11 @@ use crate::{Error, SectionVm}; /// A fresh stock handle's access capability for a test VM. fn fresh_access() -> std::sync::Arc { - std::sync::Arc::new(promptforge_vfs::empty().acquire()) + std::sync::Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ) } fn lua_with_messages() -> Lua { diff --git a/crates/promptforge-lua/src/models/tests.rs b/crates/promptforge-lua/src/models/tests.rs index b6e47957a..e2d133a44 100644 --- a/crates/promptforge-lua/src/models/tests.rs +++ b/crates/promptforge-lua/src/models/tests.rs @@ -335,7 +335,11 @@ fn h2_vm(raw_ids: bool) -> crate::SectionVm { vm.inject_host( "", &serde_json::json!({}), - &std::sync::Arc::new(promptforge_vfs::empty().acquire()), + &std::sync::Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ), ) .expect("host injection installs the H2 models table"); vm diff --git a/crates/promptforge-lua/src/tests.rs b/crates/promptforge-lua/src/tests.rs index bd90c1272..a87601bf3 100644 --- a/crates/promptforge-lua/src/tests.rs +++ b/crates/promptforge-lua/src/tests.rs @@ -15,7 +15,11 @@ const EXECUTION: &str = "lua-test"; /// exists and the vended identity is the test's own, so seeding through the /// facade and the VM's store ops never meet a second live identity. fn fresh_access() -> Arc { - Arc::new(promptforge_vfs::empty().acquire()) + Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ) } #[derive(Default)] @@ -134,7 +138,11 @@ impl VfsAccess for FailingAccess { /// The access a failing backend vends, for tests driving the error path. fn failing_access() -> Arc { - Arc::new(VfsRef::new(FailingBackend).acquire()) + Arc::new( + VfsRef::new(FailingBackend) + .acquire() + .expect("the failing backend still acquires"), + ) } struct BoundaryRecorder { diff --git a/crates/promptforge-lua/src/tools/tests.rs b/crates/promptforge-lua/src/tools/tests.rs index 5bdc3767a..e3bdb46e2 100644 --- a/crates/promptforge-lua/src/tools/tests.rs +++ b/crates/promptforge-lua/src/tools/tests.rs @@ -14,7 +14,11 @@ use std::sync::{Arc, Mutex}; /// A fresh stock handle's access capability for a test VM. fn fresh_access() -> Arc { - Arc::new(promptforge_vfs::empty().acquire()) + Arc::new( + promptforge_vfs::empty() + .acquire() + .expect("the stock backend acquires"), + ) } fn echo_handle() -> LuaToolHandle { diff --git a/crates/promptforge-lua/src/vm.rs b/crates/promptforge-lua/src/vm.rs index cd76b411d..3ee80cf0c 100644 --- a/crates/promptforge-lua/src/vm.rs +++ b/crates/promptforge-lua/src/vm.rs @@ -410,7 +410,7 @@ impl SectionVm { /// /// let nonce = GuardNonce::fresh(); /// let vfs = promptforge_vfs::empty(); - /// let access = std::sync::Arc::new(vfs.acquire()); + /// let access = std::sync::Arc::new(vfs.acquire().expect("the stock backend acquires")); /// let mut vm = SectionVm::new(&nonce, "example-run", &NullObserver::default(), "Example")?; /// vm.inject_host("input", &serde_json::json!({ "id": 1 }), &access)?; /// vm.teardown(&NullObserver::default(), "Example"); @@ -783,7 +783,7 @@ impl SectionVm { /// /// let nonce = GuardNonce::fresh(); /// let vfs = promptforge_vfs::empty(); - /// let access = std::sync::Arc::new(vfs.acquire()); + /// let access = std::sync::Arc::new(vfs.acquire().expect("the stock backend acquires")); /// let mut vm = SectionVm::new(&nonce, "example-run", &NullObserver::default(), "Example")?; /// vm.inject_host("", &serde_json::json!({}), &access)?; /// assert_eq!(vm.var()?, serde_json::json!({})); diff --git a/crates/promptforge-store/src/error.rs b/crates/promptforge-store/src/error.rs index 6605003af..6e914390f 100644 --- a/crates/promptforge-store/src/error.rs +++ b/crates/promptforge-store/src/error.rs @@ -189,7 +189,7 @@ impl StoreError { /// use promptforge_store::{StoreErrorKind, StoreExt}; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().expect("the stock backend acquires"); /// let store = vfs.store(&access); /// let err = store.read("missing.txt").unwrap_err(); /// assert_eq!(err.kind(), StoreErrorKind::NotFound); @@ -217,7 +217,7 @@ impl StoreError { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().expect("the stock backend acquires"); /// let store = vfs.store(&access); /// let err = store.read("missing.txt").unwrap_err(); /// assert!(err.is_not_found()); @@ -234,7 +234,7 @@ impl StoreError { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().expect("the stock backend acquires"); /// let store = vfs.store(&access); /// let err = store.read("missing.txt").unwrap_err(); /// assert_eq!(err.path(), Some("missing.txt")); diff --git a/crates/promptforge-store/src/lib.rs b/crates/promptforge-store/src/lib.rs index 032438e51..b24a99bd3 100644 --- a/crates/promptforge-store/src/lib.rs +++ b/crates/promptforge-store/src/lib.rs @@ -52,7 +52,7 @@ pub(crate) const MAX_GLOB_PATTERN_BYTES: usize = 1024; /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); -/// let access = vfs.acquire(); +/// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("shared.txt", "state")?; /// assert_eq!(store.read("shared.txt")?, "state"); @@ -90,7 +90,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "hi")?; /// # Ok::<(), promptforge_store::StoreError>(()) @@ -114,7 +114,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.append("a.txt", "hi")?; /// # Ok::<(), promptforge_store::StoreError>(()) @@ -140,7 +140,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "hi\n")?; /// assert_eq!(store.read("a.txt")?, "hi\n"); @@ -171,7 +171,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "one\ntwo\nthree\n")?; /// assert_eq!(store.read_range("a.txt", 2, None)?, "two\nthree"); @@ -210,7 +210,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "one\ntwo\nthree\n")?; /// assert_eq!( @@ -264,7 +264,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "one two")?; /// store.str_replace("a.txt", "two", "three")?; @@ -314,7 +314,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "hi")?; /// store.delete("a.txt")?; @@ -350,7 +350,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "")?; /// store.write("b.md", "")?; @@ -427,7 +427,7 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// assert!(!store.exists("a.txt")?); /// store.write("a.txt", "hi")?; @@ -457,7 +457,7 @@ pub trait StoreExt { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire(); + /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("seeded.txt", "input")?; /// # Ok::<(), promptforge_store::StoreError>(()) diff --git a/crates/promptforge-store/src/tests.rs b/crates/promptforge-store/src/tests.rs index e848cc35b..585bb2ab2 100644 --- a/crates/promptforge-store/src/tests.rs +++ b/crates/promptforge-store/src/tests.rs @@ -17,7 +17,7 @@ use super::{MAX_GLOB_PATTERN_BYTES, PathReason, Store, StoreError, StoreErrorKin /// single-identity test starts from. fn stock() -> (VfsRef, Access) { let vfs = promptforge_vfs::empty(); - let access = vfs.acquire(); + let access = vfs.acquire().expect("the stock backend acquires"); (vfs, access) } @@ -80,7 +80,7 @@ fn a_second_identitys_write_to_a_claimed_path_races() { store .write("a.txt", "uno") .expect("one identity may rewrite its own path"); - let second = vfs.acquire(); + let second = vfs.acquire().expect("the stock backend acquires"); let contender = vfs.store(&second); let err = contender .write("a.txt", "two") @@ -115,7 +115,7 @@ fn a_glob_over_a_claimed_path_races_like_a_read() { let (vfs, first) = stock(); let store = vfs.store(&first); store.write("a.txt", "one").expect("first write"); - let second = vfs.acquire(); + let second = vfs.acquire().expect("the stock backend acquires"); let contender = vfs.store(&second); let err = contender .glob("*.txt") @@ -147,7 +147,7 @@ fn identities_share_backing_state_once_claims_are_released() { .write("shared.txt", "written by the first") .expect("write"); drop(first); - let second = vfs.acquire(); + let second = vfs.acquire().expect("the stock backend acquires"); let reader = vfs.store(&second); assert_eq!( reader.read("shared.txt").expect("read"), @@ -777,7 +777,7 @@ fn a_panicking_operation_does_not_wedge_the_store() { let vfs = VfsRef::builder() .mount(STORE_MOUNT, PanicBackend(MemoryBackend::new())) .build(); - let access = vfs.acquire(); + let access = vfs.acquire().expect("the stock backend acquires"); let store = vfs.store(&access); let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| store.write("a.txt", "x"))); diff --git a/crates/promptforge-vfs/src/lib.rs b/crates/promptforge-vfs/src/lib.rs index a8aed462f..77995d522 100644 --- a/crates/promptforge-vfs/src/lib.rs +++ b/crates/promptforge-vfs/src/lib.rs @@ -147,7 +147,7 @@ mod tests { #[test] fn empty_carries_the_store_mount() -> Result<(), VfsError> { let vfs = empty(); - let access = vfs.acquire(); + let access = vfs.acquire()?; let path = format!("{STORE_MOUNT}/paper.md"); access.write(&path, b"# draft")?; assert_eq!(access.read(&path)?, b"# draft"); @@ -157,13 +157,14 @@ mod tests { } #[test] - fn empty_serves_nothing_outside_the_store_mount() { + fn empty_serves_nothing_outside_the_store_mount() -> Result<(), VfsError> { let vfs = empty(); - let access = vfs.acquire(); + let access = vfs.acquire()?; assert!(matches!( access.read("/elsewhere.txt"), Err(VfsError::NotFound(_)) )); + Ok(()) } #[test] @@ -172,7 +173,7 @@ mod tests { let policy = ModePolicy::new(Mode::Ask); let handle = policy.handle(); let vfs = VfsRef::with_policy(empty(), policy); - let access = vfs.acquire(); + let access = vfs.acquire()?; let path = format!("{STORE_MOUNT}/notes.md"); match access.write(&path, b"x") { Err(VfsError::PermissionDenied(reason)) => { @@ -195,7 +196,7 @@ mod tests { fn plan_mode_allows_mutations_only_to_markdown_paths() -> Result<(), VfsError> { let policy = ModePolicy::new(Mode::Plan); let vfs = VfsRef::with_policy(empty(), policy); - let access = vfs.acquire(); + let access = vfs.acquire()?; let markdown = format!("{STORE_MOUNT}/notes.md"); let binary = format!("{STORE_MOUNT}/data.bin"); access.write(&markdown, b"# ok")?; @@ -219,10 +220,10 @@ mod tests { let handle = policy.handle(); let vfs = VfsRef::with_policy(empty(), policy); let path = format!("{STORE_MOUNT}/paper.md"); - vfs.acquire().write(&path, b"text")?; + vfs.acquire()?.write(&path, b"text")?; // Even in Ask, the strictest mode, reads flow. handle.set(Mode::Ask); - let access = vfs.acquire(); + let access = vfs.acquire()?; assert_eq!(access.read(&path)?, b"text"); assert!(access.exists(&path)?); Ok(()) diff --git a/crates/shared-vfs/src/handle.rs b/crates/shared-vfs/src/handle.rs index 840a8a20c..0e4c73208 100644 --- a/crates/shared-vfs/src/handle.rs +++ b/crates/shared-vfs/src/handle.rs @@ -240,11 +240,9 @@ impl VfsRef { /// Acquires the capability for a new serial thread of execution. /// This is the only way in: every acquire vends a fresh [`ExecId`]. /// - /// # Panics - /// Panics when the backend fails to acquire the identity. Backends - /// are expected to accept attribution; a refusal is a backend bug, - /// not a runtime condition. - pub fn acquire(&self) -> Access { + /// # Errors + /// Returns an error when the backend refuses to acquire the identity. + pub fn acquire(&self) -> Result { self.acquire_with(ExecId::vend()) } @@ -253,21 +251,18 @@ impl VfsRef { /// as live in this handle's claims table, so conflicts are detected /// across both views of the same storage. /// - /// # Panics - /// Panics when the backend fails to acquire the identity; see - /// [`VfsRef::acquire`]. - pub(crate) fn acquire_with(&self, id: ExecId) -> Access { - let inner = self - .backend() - .acquire(id) - .unwrap_or_else(|err| panic!("the backend refused to acquire identity {id:?}: {err}")); + /// # Errors + /// Returns an error when the backend refuses to acquire the identity; + /// see [`VfsRef::acquire`]. + pub(crate) fn acquire_with(&self, id: ExecId) -> Result { + let inner = self.backend().acquire(id)?; self.volume.claims.register_live(id); - Access { + Ok(Access { id, volume: self.volume.clone(), policy: self.policy.clone(), inner: Mutex::new(inner), - } + }) } /// Builds a handle over a router with a fresh claims table and the @@ -309,23 +304,21 @@ impl Access { /// construction, so a retired claim can never conflict again. The /// spawn IS the happens-before edge - no fence call, no epochs. /// - /// # Panics - /// Panics when the backend fails to acquire the child's identity; - /// see [`VfsRef::acquire`]. - pub fn spawn(&self) -> Access { + /// # Errors + /// Returns an error when the backend refuses to acquire the child's + /// identity; see [`VfsRef::acquire`]. A failed spawn leaves this + /// capability's claims untouched. + pub fn spawn(&self) -> Result { let id = ExecId::vend(); - let inner = self - .backend() - .acquire(id) - .unwrap_or_else(|err| panic!("the backend refused to acquire identity {id:?}: {err}")); + let inner = self.backend().acquire(id)?; self.volume.claims.retire(self.id); self.volume.claims.register_live(id); - Access { + Ok(Access { id, volume: self.volume.clone(), policy: self.policy.clone(), inner: Mutex::new(inner), - } + }) } /// Returns this capability's identity. @@ -620,7 +613,7 @@ impl Drop for Access { /// base's policy and claims under the caller's identity. impl Vfs for VfsRef { fn acquire(&mut self, id: ExecId) -> Result, VfsError> { - Ok(Box::new(HandleAccess(self.acquire_with(id)))) + Ok(Box::new(HandleAccess(self.acquire_with(id)?))) } fn release(&mut self, id: ExecId) -> Result<(), VfsError> { @@ -860,7 +853,7 @@ mod tests { #[test] fn an_access_reads_and_writes_through_the_handle() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let access = vfs.acquire(); + let access = vfs.acquire()?; access.write("/notes/a.txt", b"hello")?; assert_eq!(access.read("/notes/a.txt")?, b"hello"); assert!(access.exists("/notes/a.txt")?); @@ -868,11 +861,12 @@ mod tests { } #[test] - fn every_acquire_vends_a_process_unique_identity() { + fn every_acquire_vends_a_process_unique_identity() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let first = vfs.acquire(); - let second = vfs.acquire(); + let first = vfs.acquire()?; + let second = vfs.acquire()?; assert_ne!(first.id(), second.id()); + Ok(()) } #[test] @@ -881,7 +875,7 @@ mod tests { // access, so sequential ops on one path by one identity stay // legal - no new identity, no false conflict. let vfs = handle(&StubFs::default()); - let access = vfs.acquire(); + let access = vfs.acquire()?; access.write("/f.txt", b"one")?; access.write("/f.txt", b"two")?; access.append("/f.txt", b"!")?; @@ -892,8 +886,8 @@ mod tests { #[test] fn a_write_conflicts_with_another_identitys_read_claim() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); - let reader = vfs.acquire(); - let writer = vfs.acquire(); + let reader = vfs.acquire()?; + let writer = vfs.acquire()?; reader.read("/f.txt")?; let message = conflict_message(writer.write("/f.txt", b"new")); assert!(message.contains("/f.txt"), "names the path: {message}"); @@ -919,9 +913,9 @@ mod tests { #[test] fn a_read_conflicts_with_another_identitys_write_claim() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let writer = vfs.acquire(); + let writer = vfs.acquire()?; writer.write("/f.txt", b"x")?; - let reader = vfs.acquire(); + let reader = vfs.acquire()?; match reader.read("/f.txt") { Err(VfsError::Conflict(_)) => {} other => panic!("expected a conflict, got {other:?}"), @@ -932,9 +926,9 @@ mod tests { #[test] fn two_writes_by_two_identities_conflict() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let first = vfs.acquire(); + let first = vfs.acquire()?; first.write("/f.txt", b"1")?; - let second = vfs.acquire(); + let second = vfs.acquire()?; let message = conflict_message(second.write("/f.txt", b"2")); assert!(message.contains("write claim"), "{message}"); Ok(()) @@ -943,8 +937,8 @@ mod tests { #[test] fn reads_by_two_identities_never_conflict() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); - let first = vfs.acquire(); - let second = vfs.acquire(); + let first = vfs.acquire()?; + let second = vfs.acquire()?; first.read("/f.txt")?; assert_eq!(second.read("/f.txt")?, b"data"); Ok(()) @@ -955,9 +949,9 @@ mod tests { // Copy claims the source as a read, and a read booms on another // live identity's write claim. let vfs = handle(&StubFs::default()); - let writer = vfs.acquire(); + let writer = vfs.acquire()?; writer.write("/src.txt", b"data")?; - let copier = vfs.acquire(); + let copier = vfs.acquire()?; let message = conflict_message(copier.copy("/src.txt", "/dst.txt")); assert!(message.contains("/src.txt"), "names the source: {message}"); Ok(()) @@ -969,9 +963,9 @@ mod tests { // read claim on the source must not block the copy. Were the // source claimed as a write, this copy would conflict. let vfs = handle(&StubFs::seeded(&[("/src.txt", "data")])); - let reader = vfs.acquire(); + let reader = vfs.acquire()?; reader.read("/src.txt")?; - let copier = vfs.acquire(); + let copier = vfs.acquire()?; copier.copy("/src.txt", "/dst.txt")?; assert_eq!(copier.read("/dst.txt")?, b"data"); Ok(()) @@ -985,9 +979,9 @@ mod tests { ("/src.txt", "data"), ("/dst.txt", "old"), ])); - let reader = vfs.acquire(); + let reader = vfs.acquire()?; reader.read("/dst.txt")?; - let copier = vfs.acquire(); + let copier = vfs.acquire()?; let message = conflict_message(copier.copy("/src.txt", "/dst.txt")); assert!( message.contains("/dst.txt"), @@ -999,9 +993,9 @@ mod tests { #[test] fn a_rename_conflicts_with_a_claim_on_the_source_path() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/from.txt", "data")])); - let reader = vfs.acquire(); + let reader = vfs.acquire()?; reader.read("/from.txt")?; - let renamer = vfs.acquire(); + let renamer = vfs.acquire()?; let message = conflict_message(renamer.rename("/from.txt", "/to.txt")); assert!(message.contains("/from.txt"), "names the source: {message}"); Ok(()) @@ -1015,9 +1009,9 @@ mod tests { ("/from.txt", "data"), ("/to.txt", "old"), ])); - let reader = vfs.acquire(); + let reader = vfs.acquire()?; reader.read("/to.txt")?; - let renamer = vfs.acquire(); + let renamer = vfs.acquire()?; let message = conflict_message(renamer.rename("/from.txt", "/to.txt")); assert!( message.contains("/to.txt"), @@ -1030,12 +1024,12 @@ mod tests { fn dropping_an_access_releases_its_identity_and_claims() -> Result<(), VfsError> { let stub = StubFs::default(); let vfs = handle(&stub); - let first = vfs.acquire(); + let first = vfs.acquire()?; let first_id = first.id(); first.write("/f.txt", b"1")?; drop(first); assert!(stub.released().contains(&first_id)); - let second = vfs.acquire(); + let second = vfs.acquire()?; second.write("/f.txt", b"2")?; assert_eq!(second.read("/f.txt")?, b"2"); Ok(()) @@ -1044,9 +1038,9 @@ mod tests { #[test] fn spawn_deletes_the_parents_claims() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let parent = vfs.acquire(); + let parent = vfs.acquire()?; parent.write("/f.txt", b"1")?; - let child = parent.spawn(); + let child = parent.spawn()?; assert_ne!(parent.id(), child.id()); // The parent's pre-spawn write claim is retired: the child can // touch the same path without a false conflict. @@ -1061,11 +1055,11 @@ mod tests { // arm in turn; a dropped arm releases its claims, so the next arm // can merge onto the same path. let vfs = handle(&StubFs::default()); - let parent = vfs.acquire(); - let arm_one = parent.spawn(); + let parent = vfs.acquire()?; + let arm_one = parent.spawn()?; arm_one.write("/evidence.md", b"one\n")?; drop(arm_one); - let arm_two = parent.spawn(); + let arm_two = parent.spawn()?; arm_two.append("/evidence.md", b"two\n")?; assert_eq!(arm_two.read("/evidence.md")?, b"one\ntwo\n"); Ok(()) @@ -1074,12 +1068,12 @@ mod tests { #[test] fn transfer_of_control_moves_the_claims_with_the_access() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); - let original = vfs.acquire(); + let original = vfs.acquire()?; original.read("/f.txt")?; // Transfer of control moves the access object; the identity and // its claims move with it. let moved = original; - let other = vfs.acquire(); + let other = vfs.acquire()?; let message = conflict_message(other.write("/f.txt", b"new")); assert!(message.contains(&format!("{:?}", moved.id()))); assert_eq!(moved.read("/f.txt")?, b"data"); @@ -1089,9 +1083,9 @@ mod tests { #[test] fn alias_spellings_of_one_file_land_on_one_claim_key() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/a/b.txt", "x")])); - let reader = vfs.acquire(); + let reader = vfs.acquire()?; reader.read("/a/./b.txt")?; - let writer = vfs.acquire(); + let writer = vfs.acquire()?; let message = conflict_message(writer.write("/a//b.txt", b"y")); assert!(message.contains("/a/b.txt"), "the canonical key: {message}"); Ok(()) @@ -1101,9 +1095,9 @@ mod tests { fn claims_are_shared_across_handle_clones() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); let clone = vfs.clone(); - let first = vfs.acquire(); + let first = vfs.acquire()?; first.write("/f.txt", b"1")?; - let second = clone.acquire(); + let second = clone.acquire()?; let message = conflict_message(second.write("/f.txt", b"2")); assert!(message.contains("/f.txt"), "{message}"); Ok(()) @@ -1133,7 +1127,7 @@ mod tests { verdict: Arc::clone(&verdict), }, ); - let denied = vfs.acquire(); + let denied = vfs.acquire()?; match denied.write("/f.txt", b"x") { Err(VfsError::PermissionDenied(reason)) => { assert_eq!(reason, "writes are sealed"); @@ -1142,7 +1136,7 @@ mod tests { } // The host flips the policy mid-run through shared state. *verdict.lock().unwrap_or_else(PoisonError::into_inner) = Verdict::Allow; - let allowed = vfs.acquire(); + let allowed = vfs.acquire()?; // Had the denied attempt registered a write claim, this write // would conflict with it. allowed.write("/f.txt", b"x")?; @@ -1153,7 +1147,7 @@ mod tests { #[test] fn read_range_slices_lines_one_based_and_inclusive() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\nthree\n")])); - let access = vfs.acquire(); + let access = vfs.acquire()?; assert_eq!(access.read_range("/f.txt", 2, None)?, "two\nthree"); assert_eq!(access.read_range("/f.txt", 2, Some(99))?, "two\nthree"); assert_eq!(access.read_range("/f.txt", 99, None)?, ""); @@ -1162,17 +1156,18 @@ mod tests { } #[test] - fn read_range_rejects_invalid_bounds() { + fn read_range_rejects_invalid_bounds() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\n")])); - let access = vfs.acquire(); + let access = vfs.acquire()?; assert!(access.read_range("/f.txt", 0, None).is_err()); assert!(access.read_range("/f.txt", 2, Some(1)).is_err()); + Ok(()) } #[test] fn read_range_numbered_numbers_absolutely_from_start() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\nthree\n")])); - let access = vfs.acquire(); + let access = vfs.acquire()?; assert_eq!( access.read_range_numbered("/f.txt", 1, None)?, "1| one\n2| two\n3| three" @@ -1190,7 +1185,7 @@ mod tests { let lines: Vec = (1..=10).map(|n| format!("line{n}")).collect(); let text = lines.join("\n"); let vfs = handle(&StubFs::seeded(&[("/f.txt", &text)])); - let access = vfs.acquire(); + let access = vfs.acquire()?; assert_eq!( access.read_range_numbered("/f.txt", 9, Some(10))?, " 9| line9\n10| line10" @@ -1201,7 +1196,7 @@ mod tests { #[test] fn read_string_rejects_non_utf8() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let access = vfs.acquire(); + let access = vfs.acquire()?; access.write("/bin.dat", &[0xff, 0xfe])?; match access.read_string("/bin.dat") { Err(VfsError::Backend(_)) => {} @@ -1216,4 +1211,82 @@ mod tests { assert_send_sync::(); assert_send_sync::(); } + + /// A backend that refuses every acquisition: the trait's contract + /// allows refusal, so the handle must surface it as an error rather + /// than panic. + struct RefusingFs; + + impl Vfs for RefusingFs { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + let _ = id; + Err(VfsError::Backend( + "the backend refuses acquisition".to_owned(), + )) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + let _ = id; + Ok(()) + } + } + + #[test] + fn a_backend_refusal_fails_acquire_with_an_error_instead_of_panicking() { + let vfs = VfsRef::new(RefusingFs); + match vfs.acquire() { + Err(VfsError::Backend(message)) => { + assert_eq!(message, "the backend refuses acquisition"); + } + other => panic!("expected a backend refusal, got {other:?}"), + } + } + + #[test] + fn a_backend_refusal_fails_spawn_and_keeps_the_parents_claims() -> Result<(), VfsError> { + /// A backend that refuses exactly its second acquisition: the + /// spawn is the second. + struct RefuseSecond { + vended: Arc>, + } + + impl Vfs for RefuseSecond { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + let _ = id; + let mut vended = self.vended.lock().unwrap_or_else(PoisonError::into_inner); + *vended += 1; + if *vended == 2 { + return Err(VfsError::Backend( + "the backend refuses acquisition".to_owned(), + )); + } + Ok(Box::new(StubAccess { + files: Arc::new(Mutex::new(BTreeMap::new())), + })) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + let _ = id; + Ok(()) + } + } + + let vfs = VfsRef::new(RefuseSecond { + vended: Arc::new(Mutex::new(0)), + }); + let parent = vfs.acquire()?; + parent.write("/f.txt", b"1")?; + match parent.spawn() { + Err(VfsError::Backend(message)) => { + assert_eq!(message, "the backend refuses acquisition"); + } + other => panic!("expected a backend refusal, got {other:?}"), + } + // The failed spawn did not retire the parent's claims: a second + // identity still conflicts with the parent's write. + let other = vfs.acquire()?; + let message = conflict_message(other.write("/f.txt", b"2")); + assert!(message.contains("/f.txt"), "{message}"); + Ok(()) + } } diff --git a/crates/shared-vfs/src/memory.rs b/crates/shared-vfs/src/memory.rs index 443db5ab5..903d49446 100644 --- a/crates/shared-vfs/src/memory.rs +++ b/crates/shared-vfs/src/memory.rs @@ -146,7 +146,7 @@ impl Tree { /// use shared_vfs::{MemoryBackend, VfsRef}; /// /// let vfs = VfsRef::new(MemoryBackend::new()); -/// let access = vfs.acquire(); +/// let access = vfs.acquire()?; /// access.write("/notes.md", b"todo")?; /// assert_eq!(access.read("/notes.md")?, b"todo"); /// # Ok::<(), shared_vfs::VfsError>(()) diff --git a/crates/shared-vfs/src/router.rs b/crates/shared-vfs/src/router.rs index 67b793495..336830b5d 100644 --- a/crates/shared-vfs/src/router.rs +++ b/crates/shared-vfs/src/router.rs @@ -258,9 +258,7 @@ impl VfsAccess for RoutingAccess { self.one_mount(from, to, "rename")?; let from_stripped = self.strip(from)?; let to_stripped = self.strip(to)?; - self.with_mount(from, |session| { - session.rename(&from_stripped, &to_stripped) - }) + self.with_mount(from, |session| session.rename(&from_stripped, &to_stripped)) } fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { @@ -567,7 +565,7 @@ mod tests { .mount("/a/b", inner.clone()) .mount("/elsewhere", untouched.clone()) .build(); - let access = vfs.acquire(); + let access = vfs.acquire()?; access.write("/a/b/f.txt", b"inner")?; access.write("/a/f.txt", b"outer")?; // Each backend keyed the file by its mount-relative path. @@ -590,7 +588,7 @@ mod tests { .mount("/", base.clone()) .mount("/mnt", shadow.clone()) .build(); - let access = vfs.acquire(); + let access = vfs.acquire()?; // The shadow mount owns everything under /mnt. assert_eq!(access.read("/mnt/f.txt")?, b"shadow-mnt"); // The base still owns the rest of the namespace. @@ -614,14 +612,14 @@ mod tests { .mount("/base", base.clone()) .mount("/local", local.clone()) .build(); - let writer = child.acquire(); + let writer = child.acquire()?; writer.write("/base/f.txt", b"nested")?; // The child router stripped its mount prefix: the base backend // keyed the file at its own root. assert!(base_storage.files().contains_key("/f.txt")); // A second child identity conflicts on the same path: the claim // registered through the mounted handle is visible. - let reader = child.acquire(); + let reader = child.acquire()?; match reader.read("/base/f.txt") { Err(VfsError::Conflict(_)) => {} other => panic!("expected a conflict, got {other:?}"), @@ -641,7 +639,7 @@ mod tests { .mount("/", rw.clone()) .mount("/ro", ro.clone()) .build(); - let access = vfs.acquire(); + let access = vfs.acquire()?; // Reads are not gated. assert_eq!(access.read("/ro/a.txt")?, b"keep"); // A write is denied with a clear read-only error. @@ -673,9 +671,9 @@ mod tests { } #[test] - fn traversal_that_escapes_the_namespace_root_is_rejected() { + fn traversal_that_escapes_the_namespace_root_is_rejected() -> Result<(), VfsError> { let vfs = VfsRef::builder().mount("/mnt", StubFs::default()).build(); - let access = vfs.acquire(); + let access = vfs.acquire()?; assert!(matches!( access.read("/mnt/../../etc/passwd"), Err(VfsError::InvalidPath(_)) @@ -684,6 +682,7 @@ mod tests { access.glob("/mnt/../../*"), Err(VfsError::InvalidPath(_)) )); + Ok(()) } #[test] @@ -691,7 +690,7 @@ mod tests { // No root mount: the only storage lives at /mnt. let storage = StubFs::seeded(&[("/f.txt", "inside")]); let vfs = VfsRef::builder().mount("/mnt", storage.clone()).build(); - let access = vfs.acquire(); + let access = vfs.acquire()?; // Dot segments within the mount resolve within the mount: the // backend sees the clean mount-relative path. assert_eq!(access.read("/mnt/sub/../f.txt")?, b"inside"); @@ -718,7 +717,7 @@ mod tests { .mount("/a", StubFs::default()) .mount("/a/b", inner.clone()) .build(); - let access = vfs.acquire(); + let access = vfs.acquire()?; let matches = access.glob("/a/b/*.txt")?; assert_eq!(matches, vec!["/a/b/x.txt".to_owned()]); Ok(()) @@ -735,7 +734,7 @@ mod tests { .build(); let overlay = base.overlay("/overlay", extra.clone()); - let writer = overlay.acquire(); + let writer = overlay.acquire()?; writer.write("/store/doc.md", b"store")?; writer.write("/scratch/tmp.txt", b"scratch")?; writer.write("/overlay/x.txt", b"overlay")?; @@ -743,7 +742,7 @@ mod tests { drop(writer); // The base handle serves its own mounts from the same storage. - let reader = base.acquire(); + let reader = base.acquire()?; assert_eq!(reader.read("/store/doc.md")?, b"store"); assert_eq!(reader.read("/scratch/tmp.txt")?, b"scratch"); // The overlay mount exists only in the overlay's view. @@ -758,11 +757,11 @@ mod tests { fn an_overlay_shares_the_bases_claims_table() -> Result<(), VfsError> { let base = VfsRef::builder().mount("/store", StubFs::default()).build(); let overlay = base.overlay("/overlay", StubFs::default()); - let first = base.acquire(); + let first = base.acquire()?; first.write("/store/shared.txt", b"1")?; // A write claim registered through the base conflicts with a // write attempted through the overlay: one claims table. - let second = overlay.acquire(); + let second = overlay.acquire()?; match second.write("/store/shared.txt", b"2") { Err(VfsError::Conflict(message)) => { assert!(message.contains("/store/shared.txt"), "{message}"); diff --git a/vibe/2026-09-12-1-test-namespace-vfs-debt.md b/vibe/2026-09-12-1-test-namespace-vfs-debt.md index 06799f12b..bd5d0912b 100644 --- a/vibe/2026-09-12-1-test-namespace-vfs-debt.md +++ b/vibe/2026-09-12-1-test-namespace-vfs-debt.md @@ -180,7 +180,7 @@ Each step is one commit containing its code and tests. -### Step 7: make handle acquisition fallible +### Step 7: make handle acquisition fallible [completed] - Component: vfs-debt-removal - Depends on step 5: the described caller set assumes the bashkit adapter (three acquisition call sites) is gone. From ec145916e5bb54ca7930cf3d6139f82e19b232cb Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 03:24:25 -0700 Subject: [PATCH 19/26] Close plan: test-namespace vfs-debt Plan: vibe/2026-09-12-1-test-namespace-vfs-debt.md --- vibe/ACTIVE | 1 - 1 file changed, 1 deletion(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 9a9660250..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-09-12-1-test-namespace-vfs-debt.md \ No newline at end of file From a7494586edd5c5a1ce6c78cafb3f7be5da2bdcb8 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 07:30:40 -0700 Subject: [PATCH 20/26] [WIP] Plan: Dependency rules, PR 35 code fixes, and the VFS observation hook --- .../2026-09-12-2-dependency-rules-vfs-hook.md | 167 ++++++++++++++++++ vibe/ACTIVE | 1 + 2 files changed, 168 insertions(+) create mode 100644 vibe/2026-09-12-2-dependency-rules-vfs-hook.md create mode 100644 vibe/ACTIVE diff --git a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md new file mode 100644 index 000000000..3155e2561 --- /dev/null +++ b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md @@ -0,0 +1,167 @@ +--- +name: Dependency rules, PR 35 code fixes, and the VFS observation hook +overview: Land the AGENTS.md dependency-rule revisions, rename shared-protocol to gateway-protocol, extend the architecture test to enforce the shared-* rule, fix the two code failures CI exposed on PR +todos: + - id: land-rules + content: Commit the AGENTS.md rule revisions together with the two design/ deletions + status: pending + - id: rename-crate + content: Rename shared-protocol to gateway-protocol (crate, manifests, imports, AGENTS.md, lockfile) + status: pending + - id: extend-arch-test + content: "Extend architecture.rs: Shared set + fifth rule, workshop prefix fix, fixture coverage" + status: pending + - id: fix-unix-clippy + content: "Add #[expect(unnecessary_wraps)] to the cfg(unix) mode_of in shared-vfs host.rs" + status: pending + - id: fix-writerace + content: "dispatch_store: drop the access clone before sending the answer" + status: pending + - id: vfs-hook + content: "shared-vfs: op-observation sink on the handle plus mandatory acquire identity label" + status: pending + - id: verify + content: "Verify: local gates plus PR #35 CI green across clippy, test, native-whisper" + status: pending +isProject: false +--- + +# Dependency rules, PR #35 code fixes, and the VFS observation hook + + + +## Product Requirements + +The promptforge repository's AGENTS.md gained a dependency-rule block whose shared-* rule one crate violates today and no check enforces. The first CI run on PR #35 (cppalliance/promptforge, head vinniefalco:master) then exposed three failures: a unix-only clippy lint invisible on Windows, a store claim-release race exposed by Linux scheduling, and a self-hosted runner environment fault on native-whisper (repaired on the machine and green since). This plan lands the rule revisions, renames the violating crate, extends the existing architecture test to enforce the shared-* rule, fixes the two code failures, and adds the VFS operation-observation hook to shared-vfs (the seam only - the event log, Lua query, and enrichment policies are deferred). + +- Problem and users: `shared-protocol` depends on `gateway-config`, violating the new AGENTS.md rule "Shared crates must not depend on any product crates" and the crate's own AGENTS.md rule; the architecture test's Workshop classification is a hardcoded pair while the rule says `workshop-*`. Users are the maintainers and every CI consumer of the PR. +- Goals: the written dependency rules are true of the code and enforced by CI. +- Non-goals: no merge of `shared-protocol` into another crate; no new shared types crate; no `dtolnay/rust-toolchain` workflow step (the runner repair holds; the operator removed it); no VFS event log, Lua exposure, or enrichment policy (deferred - only the observation hook itself lands); no edits to historical `vibe/*.md` records. +- Success criteria: no `shared-*` crate depends on a product crate; `cargo test -p gateway-stt --test it architecture` enforces all five rules including the Shared rule and prefix-based Workshop classification; the unix clippy lint is expected-out; the WriteRace test passes a 100-iteration local stress loop; a sink installed on a VFS handle receives op, canonical path, and identity label for every admitted operation and never for a denied one; PR #35 CI is green across `clippy`, `test`, and `native-whisper`. +- Constraints: the initial commit contains exactly the `AGENTS.md` revisions and the two `design/` deletions and nothing else; the `Cargo.lock` regeneration after the rename is minimal (rename only, no version changes); the workspace lint policy wants `#[expect(..., reason = "...")]`, never `#[allow(...)]`. +- Open questions: None + +## Functional Specification + +The work is rule landing, one crate rename, one test extension, two targeted code fixes, and one new observation seam. Two behavior-adjacent changes: store claims release inside `run()` before it returns instead of at the blocking pool's leisure, and a VFS handle with an installed sink reports every admitted operation to it. Nothing changes what any operation does. + +- Actors and workflows: the executing agent applies the work items; CI on PR #35 validates the result. +- Inputs and outputs: inputs are the working tree (which carries the uncommitted AGENTS.md revisions and the two `design/` deletions), the `shared-protocol` crate and its four consumers, and the architecture test; outputs are the same trees corrected, plus the initial commit. +- States and validation: after the rename, the workspace has no `shared-protocol` references outside `vibe/`; the architecture test's rule table holds five rules. +- Errors and recovery: the `dispatch_store` fix changes when claims release, never whether an operation succeeds; the clippy fix changes no behavior; the observation sink is fire-and-forget and never consulted for a decision. +- Security and privacy behavior: the WriteRace fix tightens the determinism boundary by bounding claim release to the run's lifetime; the observation sink sees op kind, canonical path, and identity label - no content - and is installed only by the host that owns the handle. +- Acceptance criteria: the success criteria above, with per-item verification in the Testing Plan. + + + + +## Technical Design + +The cross-module design is the crate rename, which changes no dependency direction - `gateway-protocol` sits where `shared-protocol` sat, consumed by the same four gateway crates - and the observation hook, which extends the shared-vfs public surface. The architecture test extension adds a package set and a rule row to an existing approved structural check. The two CI code fixes are local: a cfg-gated lint expectation and an ordering constraint in one closure. + +- Architecture: `shared-protocol` becomes `gateway-protocol`; its four consumers (`gateway`, `gateway-local`, `gateway-routing`, `gateway-web-search`) are all gateway crates, so no product boundary moves. The accepted consequence: the wire vocabulary is gateway-owned, so `promptforge-model-client`'s duplicate `ThinkingMode` can never converge onto it (promptforge crates may not depend on gateway crates) - the duplication is permanent by design. The observation hook lives in `shared-vfs` at the capability layer, not the backend layer, because backends cannot see identity: `acquire` mints the `ExecId` and now also carries the caller-supplied label. +- Modules and interfaces: `crates/shared-protocol/` moves to `crates/gateway-protocol/` with manifest, import, and lockfile renames; `crates/gateway-stt/tests/it/architecture.rs` gains `PackageSet::Shared` (prefix `shared-`), a combined `AnyProduct` forbidden set, a fifth rule (Shared cannot depend on any product), and prefix-based Workshop classification; `crates/shared-vfs/src/host.rs`'s `#[cfg(unix)]` `mode_of` gains an `#[expect]`; `crates/promptforge-core/src/execute/scheduler.rs`'s `dispatch_store` blocking closure drops its `Arc` clone before sending the answer; `crates/shared-vfs/src/handle.rs` gains the op sink and the labeled acquisition. +- File and public API changes: `VfsRef::acquire` and `Access::spawn` gain a mandatory caller-supplied `Origin` (a `#[non_exhaustive]` struct: `label`, `file`, `line`, all mandatory - `Origin::new(label)` is `#[track_caller]` and stamps the Rust call site via `Location::caller()`, `Origin::at(label, file, line)` sets an explicit position, which the executor and agent use to substitute the prompt's position for the Rust one, so every event's position is present and is the most precise thing the caller knows; claims still key on the internal `ExecId`; the `Origin` is for observability); `VfsRef::builder()` gains the op-sink installation. The sink fires on every admitted operation - after policy and claims pass, before the backend executes - with the op kind, the canonical path, and the origin; fire-and-forget, no outcome, and a policy-denied operation never fires. Reads, writes, and enumeration (list, glob) all fire. The sink is `Send + Sync` and must be cheap: store ops fire it from the blocking pool. Module docs name the deferred consumers (the bounded event log, the Lua pull query, enrichment policies). +- Data, persistence, failure, security, and privacy constraints: no persisted or wire format changes; the claims-release ordering is a lifecycle constraint - the drop must precede the send, and the fix carries a comment naming that constraint per the repository comment rule. + + + + +## Testing Plan + +Each work item carries a focused check; the WriteRace fix adds a stress loop; the final gate is PR #35 CI. No new product behavior means no new product tests beyond the architecture fixtures and the stress regression. + +- Unit: the architecture test's adversarial fixtures extended to trigger the new Shared rule and a `workshop-`-prefixed violator; the existing fixture tests keep passing. New shared-vfs tests: a sink receives events in order with op, path, and label; no sink means no events; a policy-denied operation never fires the sink. +- Integration and end-to-end: `cargo test -p gateway-stt --test it architecture` against the post-rename workspace; builds and test suites of the five rename-touched crates; the workspace builds with the new `acquire` signature (every call site labels itself). +- Regression, security, and performance: `cargo nextest run -p promptforge-core jump_inside_a_fanout_arm_to_a_silent_chain_returns_empty_text` in a 100-iteration loop, every iteration green; `rustup target add x86_64-unknown-linux-gnu` then `cargo clippy -p shared-vfs --target x86_64-unknown-linux-gnu --all-targets -- -D warnings` to see the unix-only lint without CI. +- Exit criteria: `git show --stat` of the initial commit lists exactly `AGENTS.md` and the two `design/` deletions; a repo grep for `shared[-_]protocol` outside `vibe/` finds nothing; local build, clippy, and test gates green for the touched crates; PR #35 CI green across `clippy`, `test`, and `native-whisper`. + + + + +## Decision Record + +- Decisions: + - Rename `shared-protocol` to `gateway-protocol` rather than merging: all four consumers are gateway crates, and merge has no viable home (`gateway` would cycle through its optional `gateway-local` feature edge; `gateway-routing` would force an unrelated `gateway-web-search` edge; `gateway-config`'s lean charter forbids the reqwest/tokio upstream client). The user's words: "if gateway is the only product consuming shared-protocol then shared-protocol needs to be either renamed to gateway-{something} or merged into one of the gateway-* crates". + - The two `design/` deletions ride the initial commit with the AGENTS.md revisions. The user's words: "the plan must merge the design deletions into the intiial commit". + - The PR #35 CI failures split by runner: `native-whisper` (self-hosted) was runner configuration - repaired on the machine and green since. `clippy` and `test` run on GitHub-hosted `ubuntu-latest` (`ci.yml`), so they are repo code problems: a unix-only clippy lint in `shared-vfs/src/host.rs` and a store claim-release race in `dispatch_store`. Both fixes are in scope. The user's call, after the runs-on evidence: restore both fixes. + - The VFS observation hook lands, but only the hook: the seam, the identity label, and its documentation are in scope now; the event log, the Lua query, and all enrichment policy are deferred. The user's words: "I want the hook in place but I do not want to build out the rest of the Rust and Lua". + - The AGENTS.md rule revisions themselves (Principles, Roles, Structure, Engineering sections, including the shared-* dependency rule and the all-kinds binding) are the operator's own edits; this plan lands and enforces them. +- Rejected alternatives: + - A new lean shared types crate holding `Capabilities`, `ModelKind`, `ThinkingMode`, and `Secret`: correct layering but a new facility for one edge. Revisit if a consumer outside the gateway product ever needs the wire vocabulary. + - Splitting `shared-protocol` (wire types stay shared, upstream client moves to a gateway crate): the most faithful to the crate's original name, but a much larger reshuffle than the rulebreak requires. Revisit if the upstream abstraction gains a non-gateway consumer. + - Merging into `gateway`, `gateway-routing`, or `gateway-config`: rejected for the cycle, the unrelated edge, and the lean charter respectively, as above. +- Assumptions, risks, and notes: + - The runner repair is already applied to the self-hosted Windows runner (the operator's machine) and proven: the native-whisper job went green on the re-run after the fix. What was applied: NETWORK SERVICE has read-and-execute on `C:\Users\Vinnie\.cargo\bin` and the stable toolchain directory; both runner roots (`D:\actions-runner`, `C:\actions-runner`) carry a `.env` file whose `PATH=` line is the full current machine PATH plus the service account's WindowsApps entry, which the runner's listener reads at startup (`LoadAndSetEnv`); both runner services were restarted after each change. Windows services inherit the SCM's boot-time environment, so the `.env` bootstrap - not a service restart - is what refreshes a runner's PATH. + - rust-cache's `rustc -vV` validation failure logs `##[error]` but does not fail its step (its `reportError` only annotates), so a green cache step never proved the toolchain resolved; only the run steps did. + - The WriteRace mechanism is grounded in code reading of `dispatch_store`: the `spawn_blocking` closure's `Arc` clone outlives the answer send, and claims release only when the last clone drops. The fix bounds release to chain teardown inside `run()`. + - The unix clippy lint is invisible on Windows by `cfg`; the Linux-target clippy check in the Testing Plan is the local proxy. + + + + +## Project Survey + +- Status: complete +- Build: `cargo build --locked` (default member is the gateway only); desktop app is explicit: `cargo build --locked -p workshop`. UI bundles need `npm ci --prefix crates/workshop-server/ui` and `npm ci --prefix crates/gateway-config-ui/ui` first. +- Focused test: `cargo nextest run -p ` or `cargo test -p `; single integration target: `cargo test -p --test it `. +- Component test: `cargo nextest run -p --all-features` (doctests separately: `cargo test -p --doc`). +- Full suite: `cargo nextest run --locked --workspace --exclude workshop --exclude workshop-server --all-features`, then `cargo test --workspace --exclude workshop --exclude workshop-server --all-features --doc`; workshop crates run on Windows: `cargo nextest run --locked -p workshop -p workshop-server` plus `cargo test --doc -p workshop -p workshop-server`. +- Linter: `cargo clippy --workspace --exclude workshop --exclude workshop-server --all-targets --all-features -- -D warnings`; workshop: `cargo clippy -p workshop -p workshop-server --all-targets -- -D warnings`. Supply chain: `cargo deny check`, `cargo audit`. +- Formatter check: `cargo fmt --all --check`. +- Docs: `cargo doc --workspace --no-deps --all-features --exclude workshop --exclude workshop-server` with `RUSTDOCFLAGS: -D warnings`; user guide: `mdbook build guide`. +- Test placement and naming: unit tests live in `src` beside the code; integration tests live in per-crate `tests/`, most often as a single `it` target (`tests/it/`) with shared `common`/`fixtures` helpers; cross-product end-to-end tests live in the `product-integration-tests` crate. Test names are snake_case sentences describing behavior (e.g. `a_process_lifetime_lease_recovers_after_its_owner_is_terminated`). Nextest is configured in `.config/nextest.toml` with a `heavy` test group (max 2 threads) for `promptforge-tool-picker`, `gateway-stt`, and `gateway-stt-backend-whisper`. +- Directory map: `crates/` holds all workspace members (`crates/*` glob; `shared-ui` excluded, it is a TypeScript+CSS package); `guide/` is the mdBook user guide; `prompts/` prompt pipelines; `tools/` Node helper scripts (sidecar staging, TTS live checks); `vibe/` architecture docs (`archdoc.md`); `images/` assets; `local/` local config; `.config/` nextest config; `.github/workflows/` CI; `target/` and `target-msrv/` build output. +- Component boundaries: three products plus shared substrate, per `AGENTS.md` and `vibe/archdoc.md`. `promptforge-*` crates (executor, parser, Lua boundary, tools, store, vfs policy) must not depend on gateway or workshop crates; `gateway-*` crates (routing, config, STT, local inference) must not depend on promptforge or workshop crates; `workshop-*` crates (Tauri shell, in-process server) must not depend on gateway crates; `shared-*` crates (protocol, vfs, loopback, progress, sidecar, ui) hold the public API surface and depend on no product crates; `build-*` crates build specific outputs. Dependency rules bind normal, dev, build, and target-specific dependencies. +- Conventions summary: Rust 2024 edition workspace, BSL-1.0; `unsafe_code` forbidden at workspace level (explicitly owned FFI boundaries excepted, e.g. `gateway-whisper-ffi`), `unwrap_used`/`expect_used` denied, clippy `all` denied and `pedantic` warn; behavior tests ship with behavior changes in the same change; structural enforcement (parsers, snapshots, allowlists, topology checks) requires explicit user approval; features gate real constraints (toolchain, native builds), not product shape; library/serve paths return failures instead of exiting; long-running work reports through `shared-progress`; comments cite upstream issue URLs for workarounds; no build step may write into the repository (CI enforces a clean tree). + + + + +## Execution Instructions + +Components in dependency order: rule-landing first (the initial-commit constraint requires it to precede every other commit); rename-and-enforcement next (the architecture test's new Shared rule only passes against the post-rename workspace, so the rename and the test extension land as one coupled commit); ci-code-fixes and vfs-observation-hook after (independent of each other, fixes first so the hook lands on a clean tree). Per-step verification below is the gating; the run's close-out confirms PR #35 CI green across `clippy`, `test`, and `native-whisper`. + + + +### Step 1: Land the dependency-rule revisions + +- Component: rule-landing + +Commit exactly the working tree's `AGENTS.md` revisions together with the two `design/` deletions (`design/design-gateway-tts-phase-1.md`, `design/note-gateway-tts-phase-1-verification.md`, the operator's file move to `promptforge-design/`) and nothing else. Verification: `git show --stat` of the commit lists exactly those three paths. + + + + + +### Step 2: Rename shared-protocol to gateway-protocol and enforce the Shared rule + +- Component: rename-and-enforcement + +One coupled commit - the new Shared rule fails against the pre-rename workspace, so the rename and the test extension land together. Rename: `git mv crates/shared-protocol crates/gateway-protocol`; rename the package in `crates/gateway-protocol/Cargo.toml` (drop the product qualifier from the description, keep `publish = false`); rename the root `Cargo.toml` `workspace.dependencies` entry and the four consumer manifests (`gateway`, `gateway-local`, `gateway-routing`, `gateway-web-search`); change `shared_protocol::` to `gateway_protocol::` in about 15 source files (`gateway/src/{profile_switch,lib,error,hf}.rs`, `gateway-routing/src/model.rs`, `gateway-local/src/{runtime,upstream,lib,dialect}.rs` plus `server/tests.rs` and `server/support.rs`, `gateway-web-search/src/{brave,error,service}.rs`); reword the crate's `AGENTS.md` (remove the "no dependency points back into Gateway code" rule, restate the purpose as a gateway crate) and its `README.md` name reference; check `tools/document.md`'s mention; regenerate `Cargo.lock` minimally (rename only, no version changes). Enforcement: in `crates/gateway-stt/tests/it/architecture.rs` (the test behind CI's "Check product dependency boundaries" step), make `PackageSet::Workshop` prefix-based (`package == "workshop" || package.starts_with("workshop-")`); add `PackageSet::Shared` (`starts_with("shared-")`); add a fifth rule forbidding Shared from depending on any product set (a combined `AnyProduct` forbidden set or three rule rows); extend the adversarial fixtures to trigger the new Shared rule and a `workshop-`-prefixed violator. Verification: build, clippy, and test green for the five rename-touched crates; a repo grep for `shared[-_]protocol` outside `vibe/` finds nothing; the fixture tests pass and `cargo test -p gateway-stt --test it architecture` passes against the post-rename workspace. + + + + + +### Step 3: Fix the two PR #35 code failures + +- Component: ci-code-fixes + +Both fixes are tiny, share the PR #35 CI provenance (the GitHub-hosted `ubuntu-latest` `clippy` and `test` jobs), and land as one commit. In `crates/shared-vfs/src/host.rs`, add `#[expect(clippy::unnecessary_wraps, reason = "the not(unix) variant returns None; the Option unifies the platform signatures")]` to the `#[cfg(unix)]` variant of `mode_of` (around line 252). In `crates/promptforge-core/src/execute/scheduler.rs`'s `dispatch_store` (around line 1780), drop the blocking closure's `Arc` clone after the op and its observation and before `tx.send(...)`, with a comment naming the ordering constraint per the repository comment rule; the fix changes when claims release, never whether an operation succeeds. Verification: `rustup target add x86_64-unknown-linux-gnu`, then `cargo clippy -p shared-vfs --target x86_64-unknown-linux-gnu --all-targets -- -D warnings` (check-only, no linker needed); `cargo nextest run -p promptforge-core jump_inside_a_fanout_arm_to_a_silent_chain_returns_empty_text` in a 100-iteration loop, every iteration green. These are the two fixes behind PR #35's `clippy` and `test` job failures. + + + + + +### Step 4: Add the VFS operation-observation hook + +- Component: vfs-observation-hook + +One seam, one commit. In `crates/shared-vfs/`: add `Origin` (a `#[non_exhaustive]` struct: `label`, `file`, `line`, all mandatory - `Origin::new(label)` is `#[track_caller]` and stamps the Rust call site via `Location::caller()`, `Origin::at(label, file, line)` sets an explicit position); `VfsRef::acquire` and `Access::spawn` gain a mandatory caller-supplied `Origin` (claims still key on the internal `ExecId`; the `Origin` is for observability); `VfsRef::builder()` gains the op-sink installation; the handle (`crates/shared-vfs/src/handle.rs`) fires the sink on every admitted operation - after policy and claims pass, before the backend executes - with the op kind, the canonical path, and the origin; fire-and-forget, no outcome, and a policy-denied operation never fires. Reads, writes, and enumeration (list, glob) all fire. The sink is `Send + Sync` and must be cheap: store ops fire it from the blocking pool. Update every call site: the executor and agent call `Origin::at` with the section name and the prompt's source position (all inputs already exist - the section VM is tagged with the section name, every compiled chunk carries its absolute `source_line`, sections carry spans, and the prompt carries its name); host code like the mount probe and tests call `Origin::new` and get the Rust call site for free. Module docs document the seam and name the deferred consumers (the bounded event log, the Lua pull query, enrichment policies); doc comments on `Origin::new` and `Origin::at` carry the most-specific-label guidance at the point of use (a section name for a chain, a tool id for a tool, a fixture name for a test - never a generic label when a specific one exists), and `crates/shared-vfs/AGENTS.md` carries it as a rule so agents working in the crate get it as instructions, not just docs. Verification: new shared-vfs tests - a sink receives events in order with op, path, and label; `Origin::new` stamps the caller's file and line; no sink means no events; a policy-denied operation never fires the sink - plus the shared-vfs and promptforge-vfs suites green and the workspace building with the new `acquire` signature. + + + +Deferred and out of scope: historical `vibe/*.md` references to `shared-protocol` (dated records); the PR #35 runner configuration repair (operator's machine, already applied and proven green). Deferred above the VFS observation hook (which lands in this plan): the bounded run-scoped event log, the pull-based Lua query (the agent asks; the host never pushes), and every enrichment policy - deferred because context enrichment is per-host policy (a UI coding agent wants open windows and touched files; a headless agent has no windows; a report run wants nothing). + + diff --git a/vibe/ACTIVE b/vibe/ACTIVE new file mode 100644 index 000000000..3203e3f20 --- /dev/null +++ b/vibe/ACTIVE @@ -0,0 +1 @@ +vibe/2026-09-12-2-dependency-rules-vfs-hook.md \ No newline at end of file From c8947b933f56a57008fb8cd983e47ea25b133f35 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 07:31:47 -0700 Subject: [PATCH 21/26] Land dependency-rule revisions and the design move Reworks the repository's contributor policy. The four cross-product dependency rules leave the engineering list for a new structure section that names each crate family, states what it may not depend on, adds the shared and build crate kinds, and binds normal, development, build, and target-specific dependencies alike. The do-more-with-less bullet grows into a principles section with an explicit reuse-then-improve-then-add priority order, and a roles section states what each of the three products is. Two design records for the finished gateway speech work leave the tree because they now live in the design repository. - `AGENTS.md` gains Principles, Roles, and Structure sections. The Structure section restates the dependency rules per crate family with naming conventions, adds the shared-* and build-* crate kinds, and extends the rules to normal, dev, build, and target-specific dependencies. - `AGENTS.md` drops the four-rule dependency bullet and the do-more-with-less bullet from Engineering; the new sections subsume both. - `design/design-gateway-tts-phase-1.md` and `design/note-gateway-tts-phase-1-verification.md` are deleted; both records moved to the design repository. Plan: vibe/2026-09-12-2-dependency-rules-vfs-hook.md --- AGENTS.md | 29 ++++++++- design/design-gateway-tts-phase-1.md | 60 ------------------- .../note-gateway-tts-phase-1-verification.md | 41 ------------- .../2026-09-12-2-dependency-rules-vfs-hook.md | 2 +- 4 files changed, 28 insertions(+), 104 deletions(-) delete mode 100644 design/design-gateway-tts-phase-1.md delete mode 100644 design/note-gateway-tts-phase-1-verification.md diff --git a/AGENTS.md b/AGENTS.md index 9c5721f2e..54b235da6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,11 +2,36 @@ Multi-crate Rust workspace for the PromptForge pipeline runtime, inference gateway, and Workshop desktop product. +## Principles + +- Do more with less. Prefer simple, foundational primitives over specific solutions: a primitive that naturally enables today's functionality and also generalizes beats a custom mechanism specified as a laundry list of requirements. Generality is the payoff, not a goal. +- When evaluating how to implement a capability, check whether the existing facilities subsume the work before building new machinery. Prioritize in this order: + 1. Reuse an existing facility + 2. Make the smallest improvement to an existing facility which enables the capability. + 3. Add a new facility. New machinery must have a material benefit beyond tidiness. +- When improving an existing facility, prefer an improvement that serves a problem class beyond the current case over one that solves only the case at hand, when the general shape costs no more. + +## Roles + +- Workshop is a user-facing agentic development environment: a Tauri desktop application with an HTML/CSS/TypeScript UI +- PromptForge is the runtime execution engine for the PromptForge Prompting Language: structured Markdown files with live Lua code fences +- Gateway is an independent service that proxies local and remote inference through one OpenAI-compatible HTTP and WebSocket endpoint + +## Structure + +- The three main products are PromptForge, Gateway, and Workshop +- Workshop crates are named workshop-* and must not depend on gateway crates +- Gateway crates are named gateway-* and must not depend on promptforge or workshop crates +- PromptForge crates are named promptforge-* and must not depend on gateway or workshop crates +- Shared crates are named shared-*, contain the public API surface across products and downstream crates, and must not depend on any product crates +- Crates named build-* are for building specific outputs +- Dependency rules bind all kinds: normal, dev, build, and target-specific dependencies + +## Engineering + - Prefer types and compiler checks, then behavior tests and deterministic fault injection. Add a structural check only with explicit user approval for a stable product or security boundary that has no ordinary equivalent. - Repository policy binds plans. A plan cannot introduce a source parser, snapshot, allowlist, count, ceiling, topology check, import walker, or other structural enforcement unless the user explicitly approves that exception. - Behavior changes ship with tests in the same change. Preserve product and behavior tests during refactors. Structural tests that an approved plan identifies as unsupported may be removed without replacement by another structural proxy. -- Keep four cross-product dependency rules: Gateway product crates cannot depend on Workshop product crates; PromptForge product crates cannot depend on Gateway or Workshop product crates; Gateway product crates cannot depend on PromptForge product crates; Workshop product crates cannot depend on Gateway product crates. -- Do more with less. Before adding a frontmatter field, configuration key, public type, or resolution path, determine whether sandboxed Lua, the run-scoped store, or the catalog already carries the work. New machinery must have a material benefit beyond tidiness. - A Cargo feature gates a real constraint such as a toolchain requirement or heavy native build. It does not describe product shape. Feature-disabled builds must not leak optional types into core paths. - Runtime and serve paths never compile native dependencies or invoke build tools. Library and serve paths return failures instead of exiting the process or installing process-global state. - Long-running work reports through `shared-progress`. Producers report operation state, hosts forward it, and renderers format it. diff --git a/design/design-gateway-tts-phase-1.md b/design/design-gateway-tts-phase-1.md deleted file mode 100644 index 0f44759dc..000000000 --- a/design/design-gateway-tts-phase-1.md +++ /dev/null @@ -1,60 +0,0 @@ -# Gateway Speech Synthesis, Phase 1, As Built: A Routed Speech Kind with Remote Passthrough - -## Executive summary - -The PromptForge gateway synthesizes speech. A catalog model declared with `kind = "speech"` is routed like any chat, embedding, or classifier model, and a client with a bearer token calls `POST /v1/audio/speech` with an OpenAI-shaped request (`model`, `input`, `voice`, optional `response_format`, `speed`, `instructions`, `stream_format`) and receives the provider's audio as a chunked binary stream. A companion route, `GET /v1/audio/voices`, answers the deduplicated union of every speech model's configured voices so clients can discover what they may ask for before synthesizing. Phase 1 is remote passthrough: the gateway holds the provider credential, substitutes the upstream model name, forwards the request verbatim, and streams the audio bytes back unread. Together AI serving Orpheus 3B is the live-probe backend; any OpenAI-shaped speech provider works with no further code. The local Orpheus engine is deliberately not part of this phase and is gated on an engine spike. - -The build landed as nine commits on branch `add-tts-phase-1`, executing the plan in `vibe/2026-09-07-2-gateway-tts-phase-1.md`. The pre-build specification lived in `design/report-gateway-tts-endpoint.md`; that report was removed from this repository when design records moved out (`cc7faa6a`), and this as-built supersedes it. The eight preceding commits are `d092527b`, `e10cc8dd`, `36065e56`, `37e4b166`, `2ed256f7`, `5cba784e`, `cdd6e8cd`, and `d3cf7b60`; this document lands as the ninth. Live-provider verification is deferred to the Phase 3 rerun. `design/note-gateway-tts-phase-1-verification.md` records the probe contract now and will record that run's provenance (commit and tree hash) when the rerun happens. - -## Key design choices - -**1. Speech is a fourth routed model kind, not an STT-style engine slot.** `ModelKind::Speech` (serde and `Display` spelling `"speech"`) joined the kind enum in `crates/gateway-config/src/config.rs`, so a speech model is an ordinary catalog entry: routable by name, visible in `GET /v1/models`, admitted through the dominion queue, and guarded per route by `require_kind`. The report's Pattern B alternative, an engine-slot `[[tts_model]]` catalog mirroring STT, was rejected because STT's defining constraint (audio never leaves the machine) does not apply to synthesis, and the slot pattern has no endpoint, credential, or upstream concept and no routing-table visibility. The Decision Record carries this as the plan's second decision, and the finished work matches it: speech models flow through the same routing, queue, and catalog machinery as every other kind. - -**2. The feature was rebuilt fresh; the earlier branch was discarded, not ported.** Branch `gate-way-tts-phase-1` already carried a complete phase-1 implementation (~2,750 lines) built around a `gateway-tts` service crate when the plan was written. The operator chose to discard it and rebuild on what became `add-tts-phase-1`; the branch remains in the repository untouched as a reference, and none of its code was ported. The Decision Record's first entry documents the fork choice and the later base move from master to `add-tts-phase-1` (which carries PR #18's STT rework); the discard decision survived the rebase unchanged. - -**3. The handlers live inline in the gateway crate; no service crate exists.** `audio_speech` and `audio_voices` sit in `crates/gateway/src/lib.rs` beside the embeddings and rerank handlers, registered unconditionally in `build_router`. The branch's `gateway-tts` crate split was rejected with the rebuild: the embeddings handler is the template, no module ceiling governs lib.rs (only workshop-server carries one), and a crate split adds machinery without a boundary to enforce. The revisit condition stands as recorded: if lib.rs gains a module ceiling or phase 2 gives speech a lifecycle to isolate, the split can be reconsidered. - -**4. Authentication runs before body extraction.** The report asked for the transcription handler's auth-first ordering, but PR #18 had moved the STT routes into gateway-stt behind the `authorize_stt_route` middleware, so there was no in-crate transcription ordering left to match. The built mechanism is the one the Decision Record names: `audio_speech` takes the ungated `Caller` parts-extractor (`crates/gateway/src/auth.rs`) and a raw `Request`, runs `check_auth`, and only then extracts `Json` by hand, mapping the rejection to `malformed_request`. An unauthorized caller never makes the gateway parse a body, and the `speech_auth_tests` module pins the 401-before-400 order through the real router. The extraction re-added an ungated `FromRequest` import, acceptable because axum is an unconditional dependency and the `cargo check -p gateway --no-default-features` headless gate stays green. - -**5. Voice validation precedes queue admission.** The report sequenced the voice check after queue admission; the built handler validates the requested voice against the model's catalog `voices` list before touching the dominion queue, because a 400 must not burn a queue slot. The Decision Record records this as an explicit deviation, and the integration suite pins it: a voice rejection under a full pool never reaches the backend. An empty or absent `voices` list stays valid and skips the check, so providers with no fixed voice catalog keep working. - -**6. The `voices` list is speech-only and content-validated at load.** `validate_kind_scope` (`crates/gateway-config/src/config/validate.rs`) rejects a non-empty `voices` list on any non-speech kind with an error naming the field, symmetric with the chat-only-field discipline, so a stale or misplaced list fails loudly at load instead of applying silently to the wrong kind. A list that passes is content-validated in `validate_capabilities` beside the `effort_levels` checks: no empty entries, no duplicates. The Decision Record carries this as its own entry, and the finished work matches it: `rejects_voices_on_non_speech_models`, `rejects_empty_voice_entries`, and `rejects_duplicate_voices` pin the three rejections, while `accepts_speech_model_with_empty_or_absent_voices` pins the empty list that stays valid and skips the route voice check. - -**7. The wire type pins the contract structurally.** `SpeechRequest` in `crates/shared-protocol/src/wire.rs` names seven fields and flattens every unnamed field into a `rest` map that rides to the provider verbatim, with a `RESERVED` list keeping the seven known keys out of the passthrough. Three sub-decisions, each in the Decision Record, landed as types rather than conventions. `voice` is an untagged string-or-`{"id"}` enum (`SpeechVoice`), so OpenAI's object form stays representable while membership validation stays at the route against per-model catalog data, because voice sets are per-checkpoint and never a shared constant. `response_format` is a closed enum whose `#[serde(default)]` resolves an omitted field to mp3 at deserialization: OpenAI defaults to mp3 while Together defaults to wav, so an unpinned default would silently change the wire for a Together-backed model, and pinning in the type means no route can forget it. Together-only `raw` and `mulaw` spellings fail deserialization and stay unrepresentable until the enum is deliberately widened. `validate()` returns `&'static str`, mirroring `ChatRequest::validate` exactly; it rejects an empty model, empty or over-cap input (4096 characters, matching OpenAI's limit), out-of-range speed (0.25-4.0), and reserved keys smuggled into `rest`. The report's one hard rule held throughout: nothing on the path sanitizes angle-bracket content. The offline suite pins `` passthrough, and the live probe asserts it through the gateway at the Phase 3 rerun. - -**8. The reply is a byte passthrough with header mapping, owned by a bounded background relay.** Speech is the one departure from the gateway's typed-relay norm: audio frames are opaque bytes that cannot be re-validated per chunk, so `relay_audio` re-emits the upstream stream unread. The response forwards the upstream `Content-Type` when present and otherwise falls back through `speech_fallback_mime`, which takes the framing selector first (`text/event-stream` when `stream_format` is `sse`) and only then the requested format's MIME mapping (`mp3` to `audio/mpeg`, `wav` to `audio/wav`, `pcm` to `audio/pcm`, `opus` to `audio/ogg`, `flac` to `audio/flac`, `aac` to `audio/aac`). Together SSE is unrequestable: that dialect needs `response_format = "raw"`, which the wire enum rejects, so phase-1 Together speech is non-streaming and `stream_format` forwarding is forward-looking for SSE-capable OpenAI-compatible providers. `Content-Length` is never set, so hyper emits `Transfer-Encoding: chunked`. - -The permit's lifetime does not ride the HTTP body's Drop chain. A spawned Tokio task (`relay_speech_stream`) owns the upstream body, the dominion permit, and the `InFlightGuard` cancellation guard, and feeds a bounded channel to the HTTP body: `SPEECH_RELAY_CHANNEL_CAPACITY` data slots plus one reserved terminal-error slot, reserved up front so delivering the error never waits on a stalled downstream. Four named static constants bound the stream, cfg(test)-scaled so the boundary tests run in milliseconds: `SPEECH_RELAY_TOTAL_LIFETIME` (60 min), `SPEECH_RELAY_BYTE_CEILING` (1 GiB), `SPEECH_RELAY_UPSTREAM_IDLE` (30 s, after headers; time-to-headers is the upstream layer's first-response budget), and `SPEECH_RELAY_DOWNSTREAM_BLOCKED` (60 s). Every terminal path (a bound tripped, an upstream body error, a profile-switch cancellation, the downstream gone) emits exactly one `Err` item through the reserved slot, then drops the body, the permit, and the guard together. A clean upstream end inside every bound is the one exit with no error item. Profile-switch cancellation synthesizes a body error (`relay_terminal("request cancelled for profile switch")`); it never surfaces as a clean EOF, the same fail-rather-than-truncate trade `relay_sse` makes with its `RequestCancelled` envelope. Over the wire a body-stream error aborts the response, so the item's message is server-side diagnostics and the client observes a failed read. The handler's doc comment carries the standing warning that the route must never sit under a `CompressionLayer` or whole-request `TimeoutLayer`. - -**9. Audio streams get their own HTTP client, and both deadlines live in `send_speech`.** The shared `streaming_client()` is connect-timeout-only because it also serves chat SSE, where a per-read timeout could kill long thinking pauses. Speech instead got `audio_streaming_client()` in `crates/shared-protocol/src/http_util.rs`: connect timeout 10 s and 60 s TCP keepalive, and deliberately no `read_timeout`. A 4,096-character batch generation can legitimately exceed 30 s to first headers, and reqwest 0.12 arms a client-level `read_timeout` during the header wait (`PendingRequest::poll`), which would cap time-to-headers at the body-idle budget. Both budgets therefore live in `OpenAiUpstream::send_speech`: `FIRST_RESPONSE_TIMEOUT` (~120 s, cfg(test)-scaled) wraps the `post` that waits for headers, and `AUDIO_READ_TIMEOUT` (30 s, cfg(test)-scaled) wraps each read of the opened body. `OpenAiUpstream` carries the client as a third field (`http_audio`) beside the chat and SSE clients, and the chat SSE client is untouched. - -**10. Upstream 429 and 503 map to distinct envelopes on the speech path only.** `ProtocolError::classify` renders an upstream 429 as `upstream_client_error` and a 503 as a 502 `upstream_error`, and it and its table test stayed frozen, so chat, embedding, and rerank envelopes are bit-identical. The seam lives in the gateway's own error type: `GatewayError::UpstreamRateLimited` (429 `rate_limit_error`/`upstream_rate_limited`) and `GatewayError::UpstreamUnavailable` (503 `server_error`/`upstream_unavailable`) sit beside `QueueRejected` in `crates/gateway/src/error.rs`, the gateway's exhaustive `classify()` makes the new arms compiler-forced, and the `audio_speech` handler matches `ProtocolError::UpstreamStatus` into them while forwarding every other error through the unchanged `Protocol` arm. New `ProtocolError` variants were rejected because shared-protocol's exhaustive `classify()` cannot compile them without editing the frozen function, and they would offer the codes to every route. A no-leak integration test pins the negative half: an upstream error body carrying provider internals never reaches the client. - -**11. The voices union route reads the live routing table and leads with the id.** `GET /v1/audio/voices` authenticates through `check_auth`, holds the publication lock while reading live routing, and collects every speech model's configured voices into a `BTreeSet`, so the answer is deduplicated and sorted and never calls an upstream. OpenAI has no voice-list route, but the OpenAI-compatible ecosystem (Kokoro-FastAPI, vLLM-Omni, Fish Audio) converged on one whose entries lead with the voice identifier, so each entry is an id-first `{"id", "name"}` object with `name` mirroring `id`, the catalog configuring voices as bare strings with no separate display name. The entry shape is a compatibility surface and is pinned on the raw response body; tolerant clients also accept plain strings, which the guide documents. The live probe asserts the union shape through the gateway at the Phase 3 rerun. - -**12. Building launch options is fallible, unknown kinds are refused, and the kind check runs before any provisioning side effect.** `launch_options` in `crates/gateway-local/src/runtime.rs` previously mapped serve mode through a `_ => ServeMode::Chat` catch-all, which would have compiled clean and launched a `kind = "speech"` local model as a chat server. The kind mapping is a side-effect-free `serve_mode_for(kind) -> Result`; `launch_options` calls it and returns `Result`. The speech arm and the retained wildcard both fail with `LocalError::UnsupportedKind`, which carries the offending kind and renders "local {kind} models are not yet supported". The wildcard stays because `ModelKind` is `#[non_exhaustive]`: a kind added later fails loudly instead of inheriting the chat default. `start_impl` and `provision_artifacts_impl` run `serve_mode_for` before any server, model, companion, or cache side effect (A6), so an all-speech profile touches neither the server provisioner nor the model store; a mixed profile provisions only supported models and records a per-model failure for each unsupported kind. No `ServeMode::Speech` arm exists, because no local speech runtime exists yet; the error is the entire launch behavior for the kind, and the guide's local-models chapter says so. The `tool_dialect` wildcard stays with its deliberate chat default, harmless for a kind that never launches. - -**13. Live verification is a repeatable gateway-only Node probe, not a one-shot manual check.** `tools/gateway-tts-live.mjs` is a zero-dependency Node script (built-in `fetch`, paired `tools/gateway-tts-live.test.mjs` covering startup failure, readiness timeout, request failure, assertion failure, process cleanup, key-absent skip, and secret-free output). It always runs `cargo build -p gateway` first so a stale binary can never be tested against a recorded current hash, then boots the fresh binary on an ephemeral loopback port with a throwaway Together-backed profile and asserts the speech and voices surfaces through the gateway's own responses. The vendor key comes from the process environment only (no dotenv parsing) and is ferried only to the gateway subprocess; the throwaway config carries the `api_key = "${TOGETHER_API_KEY}"` interpolation and no secret material. The script never calls a vendor directly (A19) and never prints or persists the key. With no key in the environment it prints a skip and exits 0 without building or touching the network. It never runs in CI. It deliberately never provokes the 429/503 envelopes on a paid provider, exercises no voice rejection or kind mismatch, and reads every response to completion; those behaviors stay with the Rust integration suite. The live run's provenance is recorded in `design/note-gateway-tts-phase-1-verification.md` at the Phase 3 rerun. - -**14. Config examples follow the shipped schema's credential spelling.** The report's example entries used `secret = "env:TOGETHER_API_KEY"`, which predates the config schema. The built examples, in `gateway.local.example.toml`, the gateway README, the guide, and the live probe's throwaway config, all use `api_key = "${TOGETHER_API_KEY}"` with the schema's `${VAR}` interpolation. The Decision Record names this a conformance fix, not a design change. - -**15. The config UI speaks speech; the voice picker does not.** `speech` joined the Kind dropdown options in the config UI (`models-view.ts`), and `voices` got a chips editor in the settings registry that renders only when the model kind is speech, the same control class as `effort_levels`; the round-trip into the PUT body is test-pinned. A request-time voice picker stayed a non-goal, and Discover's hardcoded `kind: "chat"` filter is a named gap left for the discover flow's own pass, exactly as the Decision Record scoped it. - -**16. Documentation landed with the feature, generated artifacts included.** The guide gained `guide/src/gateway/06-speech-synthesis.md`, inserted at number 06 with the five following chapters moved one number up as pure renames and `SUMMARY.md` regenerated by the `build-user-guide` assembler, never hand-edited. The gateway README gained a "Speech synthesis models" section documenting the route dialect; the kind lists in the remote and local model chapters include `speech`; the example configuration carries a commented speech `[[model]]` block; and the compiled single-file guide was regenerated in the same change so it mirrors the source verbatim. - -## Deferred to phase 2 - -The plan explicitly defers these, and they remain open: - -- The local speech engine: a spike choosing between a managed CrispASR child and llama-server plus in-process SNAC decode, scored on time-to-first-audio, real-time factor under concurrent load, and an ASR-roundtrip quality gate. Phase 2's engine follows the gateway-stt crate trio as its structural template, and `ServeMode::Speech` lands with the winner. -- Encoder and WAV-header policy for locally generated audio. -- Speech-model sampling-default pins (the report's Risks recipe targets Orpheus generation): no remote speech dialect carries sampling fields, so the pin lands with the local engine that actually generates. -- Together SSE reframing: Together's streaming mode needs `response_format = "raw"`, which the wire enum rejects, so Together SSE is unrequestable in phase 1 and `stream_format` forwarding is forward-looking for SSE-capable OpenAI-compatible providers. -- Workshop playback of synthesized audio. -- The config-UI voice picker. -- Technical-text conditioning before synthesis. -- ElevenLabs and Baseten adapters, both structurally divergent from the OpenAI shape. - ---- - -*2026-09-10 - Cursor Grok 4.6 (Cursor agent)* diff --git a/design/note-gateway-tts-phase-1-verification.md b/design/note-gateway-tts-phase-1-verification.md deleted file mode 100644 index 3f27b4ba2..000000000 --- a/design/note-gateway-tts-phase-1-verification.md +++ /dev/null @@ -1,41 +0,0 @@ -# Gateway TTS phase 1: live-provider verification note - -## Status - -Live verification is deferred to the Phase 3 rerun; no run has been recorded yet. The live probe is `tools/gateway-tts-live.mjs`, a dev-only Node script (zero dependencies, built-in `fetch`, never in CI) that builds the gateway fresh, boots it with a throwaway Together-backed profile, and asserts the speech and voices surfaces through the gateway's own responses. This note records the probe's contract now and its findings after the rerun. Behavior and wire shapes only: no credential material appears in this note, and none is written to disk by a run. - -## Provenance - -Recorded at the Phase 3 live rerun: - -- Commit: -- Tree hash: - -## Run boundary - -- Command: `node tools/gateway-tts-live.mjs`. Exit 0 with `LIVE OK` when every assertion passes; with no `TOGETHER_API_KEY` in the process environment the script prints a skip and exits 0 without building or touching the network. -- The script always runs `cargo build -p gateway` first, so the probed binary always matches the recorded commit; a stale binary can never be tested against a current hash. It then boots the gateway on an ephemeral loopback port with a throwaway config in a temp directory: one `[[endpoint]]` for `https://api.together.xyz/v1` with `api_key = "${TOGETHER_API_KEY}"`, one `kind = "speech"` model named `orpheus` upstreaming to `canopylabs/orpheus-3b-0.1-ft` with the eight Orpheus voices configured, and a throwaway `tts-live` profile selected via `--profile`. -- Credential invariant A19 (`vibe/archdoc.md`: keep vendor and remote-service credentials inside the gateway): the script takes the vendor key from the process environment only (no dotenv parsing, no `.env` reading) and ferries it only to the gateway subprocess environment, where the config's `${TOGETHER_API_KEY}` interpolation resolves it. The script never calls a vendor directly and never prints or persists the key. -- Calls, all through the gateway: `POST /v1/audio/speech` with default format, with `wav`, and with an emotion-tag input; `GET /v1/audio/voices`; three field-tolerance probes. -- Probe input: a single English sentence (~90 characters). - -## Assertions (each fails the run) - -- Default-format speech call returns `Content-Type: audio/mpeg` with a byte-nonempty mp3 body (ID3 tag or frame-sync magic). Together's own documented default is wav, so an mp3 answer through the gateway proves the structural `response_format = "mp3"` pin reached the provider. -- The response is streamed: no `Content-Length`, matching the chunked relay contract. -- `response_format = "wav"` returns `Content-Type: audio/wav` with a byte-nonempty RIFF/WAVE body. -- An input carrying `` returns 200 with audio: angle-bracket emotion tags pass through the gateway untouched. -- `GET /v1/audio/voices` returns 200 with `{"voices": [...]}` holding the eight configured voices as sorted `{"id", "name"}` objects with `name` mirroring `id`. -- The named optional `instructions` field is forwarded and tolerated with a 2xx, and fields outside the gateway's named wire set (`sample_rate`, a bogus `promptforge_probe`) ride the verbatim passthrough and come back 2xx. - -## Observed dialect (filled in at the Phase 3 live rerun) - -- Default format, framing, and byte counts: -- Emotion-tag handling: -- Rejected or ignored fields: -- 429/503 envelopes: not provoked unless the rerun finds a cost-free trigger; provoking a rate limit on a paid provider for observation is not worth the cost. The gateway's distinct `UpstreamRateLimited`/`UpstreamUnavailable` mappings stay covered by the Rust integration suite regardless. - -## Not exercised live - -- Voice rejection (a voice outside the catalog's `voices` list earns a 400 naming the valid set) and kind mismatch are covered by the gateway integration suite, not this probe. -- Mid-stream disconnect cancellation is covered by the integration suite; the probe reads every response to completion. diff --git a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md index 3155e2561..a2cb80f86 100644 --- a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md +++ b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md @@ -124,7 +124,7 @@ Components in dependency order: rule-landing first (the initial-commit constrain -### Step 1: Land the dependency-rule revisions +### Step 1: Land the dependency-rule revisions [completed] - Component: rule-landing From 919f85e2ac8babd69c6a286f7aa3631f289842f2 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 07:39:26 -0700 Subject: [PATCH 22/26] Rename shared-protocol to gateway-protocol, add Shared rule The wire protocol crate leaves the shared product family and joins the gateway family, since every crate that consumes it is a gateway crate. The rename is mechanical across manifests, imports, the lockfile, and docs; no wire type, trait, or helper changes behavior. The dependency boundary check gains a fifth rule forbidding any shared crate from depending on a gateway, promptforge, or workshop crate, and workshop membership becomes prefix-based so a prefixed package can no longer slip past the workshop rule. - `crates/gateway-protocol/Cargo.toml` declares the crate under its new name with the product qualifier dropped from the description and `publish = false` kept. - `crates/gateway-protocol/AGENTS.md` restates the crate as a gateway crate and replaces the no-dependency-back-into-Gateway rule with a single shared error surface for every gateway consumer. - `PackageSet` gains `Shared` (prefix `shared-`) and `AnyProduct` variants, and `Workshop` now matches any `workshop-` prefix instead of only `workshop` and `workshop-server`. - `PRODUCT_DEPENDENCY_RULES` grows from four rows to five with a Shared-cannot-depend-on-AnyProduct row, so the workspace check rejects a shared crate reaching into any product. - `adversarial_metadata_triggers_each_product_dependency_rule` adds `workshop-shell` and `shared-source` fixture violators and expects seven violations instead of five. - `crates/gateway-protocol/src/` moves `error.rs`, `http_util.rs`, `lib.rs`, `upstream.rs`, and `wire.rs` without a single content change, so the rename carries no behavior risk inside the crate. Design: new shotgun-surgery @ crates/gateway-protocol/Cargo.toml Repairs: Workshop boundary membership @ crates/gateway-stt/tests/it/architecture.rs::PackageSet - a workshop-prefixed package depending on a gateway crate passed the product dependency check Plan: vibe/2026-09-12-2-dependency-rules-vfs-hook.md --- Cargo.lock | 42 +++++++++---------- Cargo.toml | 2 +- crates/gateway-local/Cargo.toml | 2 +- crates/gateway-local/src/dialect.rs | 2 +- crates/gateway-local/src/lib.rs | 2 +- crates/gateway-local/src/runtime.rs | 6 +-- crates/gateway-local/src/server/support.rs | 2 +- crates/gateway-local/src/server/tests.rs | 30 ++++++------- crates/gateway-local/src/upstream.rs | 24 +++++------ crates/gateway-protocol/AGENTS.md | 6 +++ .../Cargo.toml | 4 +- .../README.md | 2 +- .../src/error.rs | 0 .../src/http_util.rs | 0 .../src/lib.rs | 0 .../src/upstream.rs | 0 .../src/wire.rs | 0 crates/gateway-routing/Cargo.toml | 2 +- crates/gateway-routing/src/model.rs | 2 +- crates/gateway-stt/tests/it/architecture.rs | 37 +++++++++++++--- crates/gateway-web-search/Cargo.toml | 2 +- crates/gateway-web-search/src/brave.rs | 4 +- crates/gateway-web-search/src/error.rs | 2 +- crates/gateway-web-search/src/service.rs | 6 +-- crates/gateway/Cargo.toml | 2 +- crates/gateway/src/error.rs | 2 +- crates/gateway/src/hf.rs | 4 +- crates/gateway/src/lib.rs | 4 +- crates/gateway/src/profile_switch.rs | 2 +- crates/shared-protocol/AGENTS.md | 6 --- tools/document.md | 2 +- .../2026-09-12-2-dependency-rules-vfs-hook.md | 2 +- 32 files changed, 114 insertions(+), 89 deletions(-) create mode 100644 crates/gateway-protocol/AGENTS.md rename crates/{shared-protocol => gateway-protocol}/Cargo.toml (83%) rename crates/{shared-protocol => gateway-protocol}/README.md (96%) rename crates/{shared-protocol => gateway-protocol}/src/error.rs (100%) rename crates/{shared-protocol => gateway-protocol}/src/http_util.rs (100%) rename crates/{shared-protocol => gateway-protocol}/src/lib.rs (100%) rename crates/{shared-protocol => gateway-protocol}/src/upstream.rs (100%) rename crates/{shared-protocol => gateway-protocol}/src/wire.rs (100%) delete mode 100644 crates/shared-protocol/AGENTS.md diff --git a/Cargo.lock b/Cargo.lock index 5e6bfa885..49fa1c8dd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2022,6 +2022,7 @@ dependencies = [ "gateway-config-ui", "gateway-local", "gateway-logging", + "gateway-protocol", "gateway-routing", "gateway-stt", "gateway-stt-engine", @@ -2042,7 +2043,6 @@ dependencies = [ "sha2 0.11.0", "shared-loopback", "shared-progress", - "shared-protocol", "shared-sidecar", "subtle", "sysinfo", @@ -2096,6 +2096,7 @@ dependencies = [ "async-trait", "flate2", "gateway-config", + "gateway-protocol", "gateway-routing", "minijinja", "minijinja-contrib", @@ -2105,7 +2106,6 @@ dependencies = [ "serde_json", "sha2 0.11.0", "shared-progress", - "shared-protocol", "tar", "tempfile", "thiserror 2.0.19", @@ -2124,12 +2124,29 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "gateway-protocol" +version = "0.3.0" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "gateway-config", + "reqwest 0.12.28", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "gateway-routing" version = "0.3.0" dependencies = [ "gateway-config", - "shared-protocol", + "gateway-protocol", "thiserror 2.0.19", "tokio", ] @@ -2188,10 +2205,10 @@ name = "gateway-web-search" version = "0.3.0" dependencies = [ "gateway-config", + "gateway-protocol", "reqwest 0.12.28", "serde", "serde_json", - "shared-protocol", "thiserror 2.0.19", "tokio", "url", @@ -6086,23 +6103,6 @@ dependencies = [ "tracing", ] -[[package]] -name = "shared-protocol" -version = "0.3.0" -dependencies = [ - "async-trait", - "bytes", - "futures-util", - "gateway-config", - "reqwest 0.12.28", - "serde", - "serde_json", - "thiserror 2.0.19", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "shared-sidecar" version = "0.3.0" diff --git a/Cargo.toml b/Cargo.toml index 31c68c4d3..593c656c5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,7 @@ gateway-config-ui = { path = "crates/gateway-config-ui", version = "0.3.0" } gateway-local = { path = "crates/gateway-local", version = "0.3.0" } gateway-logging = { path = "crates/gateway-logging", version = "0.3.0" } shared-loopback = { path = "crates/shared-loopback", version = "0.3.0" } -shared-protocol = { path = "crates/shared-protocol", version = "0.3.0" } +gateway-protocol = { path = "crates/gateway-protocol", version = "0.3.0" } shared-sidecar = { path = "crates/shared-sidecar", version = "0.3.0" } shared-vfs = { path = "crates/shared-vfs", version = "0.3.0" } gateway-routing = { path = "crates/gateway-routing", version = "0.3.0" } diff --git a/crates/gateway-local/Cargo.toml b/crates/gateway-local/Cargo.toml index faecf06d5..7aaa63462 100644 --- a/crates/gateway-local/Cargo.toml +++ b/crates/gateway-local/Cargo.toml @@ -16,7 +16,7 @@ documentation = "https://cppalliance.github.io/promptforge/" async-trait.workspace = true flate2.workspace = true gateway-config.workspace = true -shared-protocol.workspace = true +gateway-protocol.workspace = true gateway-routing.workspace = true shared-progress.workspace = true rand.workspace = true diff --git a/crates/gateway-local/src/dialect.rs b/crates/gateway-local/src/dialect.rs index 53fc73ad2..2b1708db0 100644 --- a/crates/gateway-local/src/dialect.rs +++ b/crates/gateway-local/src/dialect.rs @@ -19,7 +19,7 @@ use crate::error::LocalError; const PROPS_TIMEOUT: Duration = Duration::from_secs(5); /// Byte ceiling for a dialect-probe JSON body (HYGIENE-BOUNDS-001). -const MAX_PROBE_BODY: u64 = shared_protocol::http_util::MAX_JSON_BODY as u64; +const MAX_PROBE_BODY: u64 = gateway_protocol::http_util::MAX_JSON_BODY as u64; /// Evidence from a local child's `/props`, `/v1/models`, and sidecar metadata /// used to select a tool-calling dialect. diff --git a/crates/gateway-local/src/lib.rs b/crates/gateway-local/src/lib.rs index d7383331d..fb1a7fe6c 100644 --- a/crates/gateway-local/src/lib.rs +++ b/crates/gateway-local/src/lib.rs @@ -12,7 +12,7 @@ //! header inspection in [`gguf`]. //! //! Failures are reported as [`LocalError`]; an explicit teardown failure is -//! reported as [`ShutdownError`](shared_protocol::ShutdownError). +//! reported as [`ShutdownError`](gateway_protocol::ShutdownError). //! The crate contains no HTTP routing and no error envelopes; those live in //! the gateway crate. diff --git a/crates/gateway-local/src/runtime.rs b/crates/gateway-local/src/runtime.rs index a95b3e368..53f43d710 100644 --- a/crates/gateway-local/src/runtime.rs +++ b/crates/gateway-local/src/runtime.rs @@ -14,10 +14,10 @@ use std::thread; use std::time::Duration; use gateway_config::{Config, LocalModelConfig, ModelKind, QueuePolicy, ThinkingMode}; +use gateway_protocol::ShutdownError; use gateway_routing::queue::DominionQueue; use gateway_routing::{Endpoint, Model, dominion_queues}; use shared_progress::ProgressHandle; -use shared_protocol::ShutdownError; use tokio_util::sync::CancellationToken; use crate::artifacts::{self, ArtifactStore, ProvisionedServer, ServerSelection}; @@ -658,7 +658,7 @@ impl LocalRuntime { /// Removes one started model and its upstream from the runtime, returning /// the model so the caller can tear the child down through the - /// [`Upstream`](shared_protocol::upstream::Upstream) seam (which disables + /// [`Upstream`](gateway_protocol::upstream::Upstream) seam (which disables /// respawn before killing the process). Returns `None` when no started /// model carries `name`. /// @@ -682,7 +682,7 @@ impl LocalRuntime { /// Dropping the runtime does not guarantee child termination, because the /// routing table holds `Arc` clones of these same models, so /// the runtime is not the sole owner (PFGL-MOD-001). This drives an explicit - /// teardown through the [`Upstream`](shared_protocol::upstream::Upstream) seam so a + /// teardown through the [`Upstream`](gateway_protocol::upstream::Upstream) seam so a /// profile switch frees the old children's VRAM deterministically before the /// replacement profile's children start. Every child is torn down even if an /// earlier one fails, so one stuck child never strands the rest. diff --git a/crates/gateway-local/src/server/support.rs b/crates/gateway-local/src/server/support.rs index 4eb89960d..0c94ee705 100644 --- a/crates/gateway-local/src/server/support.rs +++ b/crates/gateway-local/src/server/support.rs @@ -18,7 +18,7 @@ use super::{ SpawnRequest, }; use crate::error::LocalError; -use shared_protocol::http_util::MAX_JSON_BODY; +use gateway_protocol::http_util::MAX_JSON_BODY; /// A spawn callback: builds a child from a [`SpawnRequest`]. pub(super) type SpawnFn = Box) -> Result + Send>; diff --git a/crates/gateway-local/src/server/tests.rs b/crates/gateway-local/src/server/tests.rs index df7a82e02..bff35aa65 100644 --- a/crates/gateway-local/src/server/tests.rs +++ b/crates/gateway-local/src/server/tests.rs @@ -817,9 +817,9 @@ fn respawn_reuses_port_and_identity_after_child_death() { #[test] fn local_upstream_send_respawns_dead_child_once() { use crate::upstream::LocalUpstream; + use gateway_protocol::upstream::Upstream; + use gateway_protocol::wire::ChatRequest; use serde_json::Map; - use shared_protocol::upstream::Upstream; - use shared_protocol::wire::ChatRequest; let port = free_port().expect("select free port"); let mut ports = VecDeque::from([port]); @@ -912,9 +912,9 @@ fn local_upstream_send_embeddings_routes_through_child() { // An embeddings request forwards to the child's `/v1/embeddings` and the // response restores the caller's model name, same contract as chat. use crate::upstream::LocalUpstream; + use gateway_protocol::upstream::Upstream; + use gateway_protocol::wire::{EmbeddingInput, EmbeddingRequest}; use serde_json::Map; - use shared_protocol::upstream::Upstream; - use shared_protocol::wire::{EmbeddingInput, EmbeddingRequest}; let port = free_port().expect("select free port"); let mut ports = VecDeque::from([port]); @@ -980,9 +980,9 @@ fn local_upstream_send_rerank_routes_through_child() { // A rerank request forwards to the child's `/v1/rerank` and the response // restores the caller's model name, same contract as chat. use crate::upstream::LocalUpstream; + use gateway_protocol::upstream::Upstream; + use gateway_protocol::wire::RerankRequest; use serde_json::Map; - use shared_protocol::upstream::Upstream; - use shared_protocol::wire::RerankRequest; let port = free_port().expect("select free port"); let mut ports = VecDeque::from([port]); @@ -1049,10 +1049,10 @@ fn local_upstream_send_honors_cooldown_after_failed_respawn() { // UPSTREAM-005: a failed respawn records the attempt time; an immediate // second failure is short-circuited by the cooldown (no respawn storm). use crate::upstream::LocalUpstream; + use gateway_protocol::ProtocolError; + use gateway_protocol::upstream::Upstream; + use gateway_protocol::wire::ChatRequest; use serde_json::Map; - use shared_protocol::ProtocolError; - use shared_protocol::upstream::Upstream; - use shared_protocol::wire::ChatRequest; let port = free_port().expect("select free port"); let mut ports = VecDeque::from([port]); @@ -1150,9 +1150,9 @@ fn local_upstream_concurrent_sends_respawn_child_at_most_once() { // UPSTREAM-005: two concurrent transport failures on a dead child serialize // through the guard mutex, so recovery respawns the child exactly once. use crate::upstream::LocalUpstream; + use gateway_protocol::upstream::Upstream; + use gateway_protocol::wire::ChatRequest; use serde_json::Map; - use shared_protocol::upstream::Upstream; - use shared_protocol::wire::ChatRequest; let port = free_port().expect("select free port"); let mut ports = VecDeque::from([port]); @@ -1293,9 +1293,9 @@ fn local_upstream_shutdown_kills_child_and_disables_respawn() { // PFGL-MOD-001/PF-GW-SERVER-004: an explicit shutdown terminates the child // and prevents any later transport failure from respawning it. use crate::upstream::LocalUpstream; + use gateway_protocol::upstream::Upstream; + use gateway_protocol::wire::ChatRequest; use serde_json::Map; - use shared_protocol::upstream::Upstream; - use shared_protocol::wire::ChatRequest; let port = free_port().expect("select free port"); let mut ports = VecDeque::from([port]); @@ -1422,9 +1422,9 @@ fn switch_shutdown_terminates_an_in_flight_respawned_child() { // recovery/respawn must cancel the respawn and terminate the freshly spawned // child, so no old child can outlive a profile switch. use crate::upstream::LocalUpstream; + use gateway_protocol::upstream::Upstream; + use gateway_protocol::wire::ChatRequest; use serde_json::Map; - use shared_protocol::upstream::Upstream; - use shared_protocol::wire::ChatRequest; let port = free_port().expect("select free port"); let mut ports = VecDeque::from([port]); diff --git a/crates/gateway-local/src/upstream.rs b/crates/gateway-local/src/upstream.rs index ed30592c1..77e66cfc7 100644 --- a/crates/gateway-local/src/upstream.rs +++ b/crates/gateway-local/src/upstream.rs @@ -6,11 +6,11 @@ use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; use async_trait::async_trait; -use shared_protocol::upstream::Upstream; -use shared_protocol::wire::{ +use gateway_protocol::upstream::Upstream; +use gateway_protocol::wire::{ ChatRequest, ChatResponse, EmbeddingRequest, EmbeddingResponse, RerankRequest, RerankResponse, }; -use shared_protocol::{ProtocolError, ShutdownError}; +use gateway_protocol::{ProtocolError, ShutdownError}; use crate::error::LocalError; use crate::server::{LaunchOptions, ServerGuard}; @@ -77,8 +77,8 @@ impl LocalUpstream { last_respawn: Mutex::new(None), shut_down: AtomicBool::new(false), }), - http: shared_protocol::http_util::bounded_client(), - http_stream: shared_protocol::http_util::streaming_client(), + http: gateway_protocol::http_util::bounded_client(), + http_stream: gateway_protocol::http_util::streaming_client(), } } @@ -243,9 +243,9 @@ impl LocalUpstream { let status = response.status(); if !status.is_success() { - let body = shared_protocol::http_util::read_body_capped( + let body = gateway_protocol::http_util::read_body_capped( response, - shared_protocol::http_util::MAX_ERROR_BODY, + gateway_protocol::http_util::MAX_ERROR_BODY, ) .await; let body: String = body.chars().take(2000).collect(); @@ -272,9 +272,9 @@ impl LocalUpstream { body: &impl serde::Serialize, ) -> Result, ProtocolError> { let response = self.post(&self.http, path, body).await?; - shared_protocol::http_util::read_bytes_capped( + gateway_protocol::http_util::read_bytes_capped( response, - shared_protocol::http_util::MAX_JSON_BODY, + gateway_protocol::http_util::MAX_JSON_BODY, ) .await .map_err(ProtocolError::upstream_transport) @@ -323,13 +323,13 @@ impl LocalUpstream { &self, mut req: ChatRequest, upstream_model: &str, - ) -> Result { + ) -> Result { let requested = std::mem::replace(&mut req.model, upstream_model.to_string()); req.stream = true; let response = self .post(&self.http_stream, "chat/completions", &req) .await?; - Ok(shared_protocol::upstream::sse_chunks(response, requested)) + Ok(gateway_protocol::upstream::sse_chunks(response, requested)) } /// Run the dead-child recovery after a transport failure. @@ -414,7 +414,7 @@ impl Upstream for LocalUpstream { &self, req: ChatRequest, upstream_model: &str, - ) -> Result { + ) -> Result { // Recovery applies only to a pre-stream transport failure: once the // chunk stream is open, a mid-stream death surfaces as an `Err` item // rather than triggering a respawn under a live response. diff --git a/crates/gateway-protocol/AGENTS.md b/crates/gateway-protocol/AGENTS.md new file mode 100644 index 000000000..1b0d45b04 --- /dev/null +++ b/crates/gateway-protocol/AGENTS.md @@ -0,0 +1,6 @@ +# gateway-protocol + +This gateway crate owns the OpenAI wire protocol, bounded client behavior, and the upstream abstraction. + +- Local inference, routing, and HTTP handlers stay in their owning crates. +- Protocol errors do not name local-inference concepts. Upstream shutdown uses this crate's error vocabulary so every gateway consumer shares one error surface. diff --git a/crates/shared-protocol/Cargo.toml b/crates/gateway-protocol/Cargo.toml similarity index 83% rename from crates/shared-protocol/Cargo.toml rename to crates/gateway-protocol/Cargo.toml index 4b45c9aa5..48f186c12 100644 --- a/crates/shared-protocol/Cargo.toml +++ b/crates/gateway-protocol/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "shared-protocol" +name = "gateway-protocol" version.workspace = true edition.workspace = true license.workspace = true repository.workspace = true publish = false -description = "PromptForge gateway protocol: OpenAI wire types, validation, and the upstream abstraction" +description = "Gateway protocol: OpenAI wire types, validation, and the upstream abstraction" readme = "README.md" keywords = ["llm", "gateway", "openai", "proxy"] categories = ["web-programming::http-client"] diff --git a/crates/shared-protocol/README.md b/crates/gateway-protocol/README.md similarity index 96% rename from crates/shared-protocol/README.md rename to crates/gateway-protocol/README.md index 1d57a5370..162005fec 100644 --- a/crates/shared-protocol/README.md +++ b/crates/gateway-protocol/README.md @@ -1,4 +1,4 @@ -# shared-protocol +# gateway-protocol The OpenAI wire protocol and upstream abstraction for the PromptForge inference gateway: request/response wire types with trust-boundary diff --git a/crates/shared-protocol/src/error.rs b/crates/gateway-protocol/src/error.rs similarity index 100% rename from crates/shared-protocol/src/error.rs rename to crates/gateway-protocol/src/error.rs diff --git a/crates/shared-protocol/src/http_util.rs b/crates/gateway-protocol/src/http_util.rs similarity index 100% rename from crates/shared-protocol/src/http_util.rs rename to crates/gateway-protocol/src/http_util.rs diff --git a/crates/shared-protocol/src/lib.rs b/crates/gateway-protocol/src/lib.rs similarity index 100% rename from crates/shared-protocol/src/lib.rs rename to crates/gateway-protocol/src/lib.rs diff --git a/crates/shared-protocol/src/upstream.rs b/crates/gateway-protocol/src/upstream.rs similarity index 100% rename from crates/shared-protocol/src/upstream.rs rename to crates/gateway-protocol/src/upstream.rs diff --git a/crates/shared-protocol/src/wire.rs b/crates/gateway-protocol/src/wire.rs similarity index 100% rename from crates/shared-protocol/src/wire.rs rename to crates/gateway-protocol/src/wire.rs diff --git a/crates/gateway-routing/Cargo.toml b/crates/gateway-routing/Cargo.toml index 907c2be33..b151fff75 100644 --- a/crates/gateway-routing/Cargo.toml +++ b/crates/gateway-routing/Cargo.toml @@ -14,7 +14,7 @@ documentation = "https://cppalliance.github.io/promptforge/" [dependencies] gateway-config.workspace = true -shared-protocol.workspace = true +gateway-protocol.workspace = true thiserror.workspace = true tokio.workspace = true diff --git a/crates/gateway-routing/src/model.rs b/crates/gateway-routing/src/model.rs index 8dccb0e2d..2eeeda93f 100644 --- a/crates/gateway-routing/src/model.rs +++ b/crates/gateway-routing/src/model.rs @@ -4,7 +4,7 @@ use std::sync::Arc; use gateway_config::{Capabilities, ModelKind, ThinkingMode}; -use shared_protocol::upstream::Upstream; +use gateway_protocol::upstream::Upstream; use crate::queue::DominionQueue; diff --git a/crates/gateway-stt/tests/it/architecture.rs b/crates/gateway-stt/tests/it/architecture.rs index f864ea83b..0f76831f2 100644 --- a/crates/gateway-stt/tests/it/architecture.rs +++ b/crates/gateway-stt/tests/it/architecture.rs @@ -1,4 +1,4 @@ -//! Cargo metadata checks for the four approved product dependency boundaries. +//! Cargo metadata checks for the five approved product dependency boundaries. use std::collections::{BTreeMap, BTreeSet}; use std::path::{Path, PathBuf}; @@ -29,7 +29,9 @@ enum PackageSet { Gateway, PromptForge, Workshop, + Shared, GatewayOrWorkshop, + AnyProduct, } impl PackageSet { @@ -37,10 +39,16 @@ impl PackageSet { match self { Self::Gateway => package == "gateway" || package.starts_with("gateway-"), Self::PromptForge => package == "promptforge" || package.starts_with("promptforge-"), - Self::Workshop => matches!(package, "workshop" | "workshop-server"), + Self::Workshop => package == "workshop" || package.starts_with("workshop-"), + Self::Shared => package.starts_with("shared-"), Self::GatewayOrWorkshop => { Self::Gateway.contains(package) || Self::Workshop.contains(package) } + Self::AnyProduct => { + Self::Gateway.contains(package) + || Self::PromptForge.contains(package) + || Self::Workshop.contains(package) + } } } } @@ -51,7 +59,7 @@ struct DependencyRule { description: &'static str, } -const PRODUCT_DEPENDENCY_RULES: [DependencyRule; 4] = [ +const PRODUCT_DEPENDENCY_RULES: [DependencyRule; 5] = [ DependencyRule { dependent: PackageSet::Gateway, forbidden: PackageSet::Workshop, @@ -72,6 +80,11 @@ const PRODUCT_DEPENDENCY_RULES: [DependencyRule; 4] = [ forbidden: PackageSet::Gateway, description: "Workshop cannot depend on Gateway", }, + DependencyRule { + dependent: PackageSet::Shared, + forbidden: PackageSet::AnyProduct, + description: "Shared cannot depend on any product", + }, ]; fn workspace_root() -> PathBuf { @@ -161,7 +174,7 @@ fn dependency_violations(metadata: &CargoMetadata) -> Vec { } #[test] -fn workspace_obeys_the_four_product_dependency_rules() { +fn workspace_obeys_the_five_product_dependency_rules() { let violations = dependency_violations(workspace_metadata()); assert!( violations.is_empty(), @@ -229,6 +242,7 @@ fn adversarial_metadata_triggers_each_product_dependency_rule() { { "workspace_members": [ "gateway-source", "promptforge-source", "workshop-source", + "workshop-shell-source", "shared-source", "gateway-target", "promptforge-target", "workshop-target" ], "packages": [ @@ -253,6 +267,16 @@ fn adversarial_metadata_triggers_each_product_dependency_rule() { "manifest_path": "C:/workspace/workshop-source/Cargo.toml", "dependencies": [{"path": "C:/workspace/gateway-target"}] }, + { + "name": "workshop-shell", "id": "workshop-shell-source", + "manifest_path": "C:/workspace/workshop-shell-source/Cargo.toml", + "dependencies": [{"path": "C:/workspace/gateway-target"}] + }, + { + "name": "shared-source", "id": "shared-source", + "manifest_path": "C:/workspace/shared-source/Cargo.toml", + "dependencies": [{"path": "C:/workspace/promptforge-target"}] + }, { "name": "gateway-target", "id": "gateway-target", "manifest_path": "C:/workspace/gateway-target/Cargo.toml", "dependencies": [] @@ -282,7 +306,8 @@ fn adversarial_metadata_triggers_each_product_dependency_rule() { } assert_eq!( violations.len(), - 5, - "PromptForge's combined rule rejects both forbidden product families" + 7, + "PromptForge's combined rule rejects both forbidden product families, \ + the Workshop rule is prefix-based, and Shared rejects every product" ); } diff --git a/crates/gateway-web-search/Cargo.toml b/crates/gateway-web-search/Cargo.toml index abbfe8455..f885891c3 100644 --- a/crates/gateway-web-search/Cargo.toml +++ b/crates/gateway-web-search/Cargo.toml @@ -14,7 +14,7 @@ documentation = "https://cppalliance.github.io/promptforge/" [dependencies] gateway-config.workspace = true -shared-protocol.workspace = true +gateway-protocol.workspace = true reqwest.workspace = true serde.workspace = true serde_json.workspace = true diff --git a/crates/gateway-web-search/src/brave.rs b/crates/gateway-web-search/src/brave.rs index 1dc8f9da7..5eaab655d 100644 --- a/crates/gateway-web-search/src/brave.rs +++ b/crates/gateway-web-search/src/brave.rs @@ -5,9 +5,9 @@ //! [`BraveSearchParams`] and calls [`brave_search`]; everything Brave-specific //! (query pairs, over-fetch policy, JSON shape, error prefixing) lives here. +use gateway_protocol::ProtocolError; +use gateway_protocol::http_util; use serde::Deserialize; -use shared_protocol::ProtocolError; -use shared_protocol::http_util; use crate::service::SearchResult; diff --git a/crates/gateway-web-search/src/error.rs b/crates/gateway-web-search/src/error.rs index 60caf1751..b862e5cf0 100644 --- a/crates/gateway-web-search/src/error.rs +++ b/crates/gateway-web-search/src/error.rs @@ -4,7 +4,7 @@ //! gateway adapts it into its own route-level error type so the envelope and //! status mapping stay in one place. -use shared_protocol::ProtocolError; +use gateway_protocol::ProtocolError; /// A request-time failure of the web-search service. #[derive(Debug, thiserror::Error)] diff --git a/crates/gateway-web-search/src/service.rs b/crates/gateway-web-search/src/service.rs index 157152c80..16414a08a 100644 --- a/crates/gateway-web-search/src/service.rs +++ b/crates/gateway-web-search/src/service.rs @@ -74,7 +74,7 @@ impl WebSearchState { api_key: cfg.api_key().clone(), base_url: cfg.base_url().trim_end_matches('/').to_string(), settings: WebSearchSettings::from_config(cfg), - http: shared_protocol::http_util::bounded_client(), + http: gateway_protocol::http_util::bounded_client(), } } } @@ -507,12 +507,12 @@ mod tests { #[test] fn prefix_web_search_upstream_prefixes_status_body() { - let err = prefix_web_search_upstream(shared_protocol::ProtocolError::upstream_status( + let err = prefix_web_search_upstream(gateway_protocol::ProtocolError::upstream_status( 429, "rate limited".to_string(), )); match err { - shared_protocol::ProtocolError::UpstreamStatus { body, .. } => { + gateway_protocol::ProtocolError::UpstreamStatus { body, .. } => { assert_eq!(body, "web_search: rate limited"); } other => panic!("expected UpstreamStatus, got {other:?}"), diff --git a/crates/gateway/Cargo.toml b/crates/gateway/Cargo.toml index 8d9ee46cf..562827f1a 100644 --- a/crates/gateway/Cargo.toml +++ b/crates/gateway/Cargo.toml @@ -51,7 +51,7 @@ rand.workspace = true # The shared loopback wall for the admin config endpoints; always on, # because those endpoints hold secrets in every build. shared-loopback.workspace = true -shared-protocol.workspace = true +gateway-protocol.workspace = true # The gateway-discovery-file seam: gateway.json is written after every successful # bind, in every build, so the workshop can discover a running gateway. shared-sidecar.workspace = true diff --git a/crates/gateway/src/error.rs b/crates/gateway/src/error.rs index 146715f03..57cc5e96e 100644 --- a/crates/gateway/src/error.rs +++ b/crates/gateway/src/error.rs @@ -4,7 +4,7 @@ use axum::Json; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use gateway_config::ModelKind; -use shared_protocol::ProtocolError; +use gateway_protocol::ProtocolError; /// A request-time failure, rendered to the client as an OpenAI error envelope. #[derive(Debug, thiserror::Error)] diff --git a/crates/gateway/src/hf.rs b/crates/gateway/src/hf.rs index 60da954cf..2a2f6b388 100644 --- a/crates/gateway/src/hf.rs +++ b/crates/gateway/src/hf.rs @@ -16,8 +16,8 @@ use axum::http::HeaderValue; use axum::http::header::CONTENT_TYPE; use axum::response::Response; use gateway_config::Secret; -use shared_protocol::ProtocolError; -use shared_protocol::http_util::{self, MAX_ERROR_BODY, read_body_capped}; +use gateway_protocol::ProtocolError; +use gateway_protocol::http_util::{self, MAX_ERROR_BODY, read_body_capped}; use crate::auth::Caller; use crate::error::GatewayError; diff --git a/crates/gateway/src/lib.rs b/crates/gateway/src/lib.rs index 7783cd9d5..7b623d8c0 100644 --- a/crates/gateway/src/lib.rs +++ b/crates/gateway/src/lib.rs @@ -113,7 +113,7 @@ mod tray; // The wire protocol and upstream abstraction live in the protocol crate; // these re-exports keep every `crate::wire::*` and `crate::upstream::*` // path resolving unchanged. -pub(crate) use shared_protocol::{upstream, wire}; +pub(crate) use gateway_protocol::{upstream, wire}; // The dominion admission queues live in the routing crate; this re-export // keeps every `crate::queue::*` path resolving unchanged. pub(crate) use gateway_routing::queue; @@ -169,12 +169,12 @@ use crate::wire::{ use gateway_config::ModelKind; #[cfg(feature = "web-search")] use gateway_config::WebSearchConfig; +use gateway_protocol::ProtocolError; #[cfg(feature = "stt")] use gateway_stt::SpeechService; #[cfg(feature = "web-search")] use gateway_web_search::{WebSearchRequest, WebSearchResponse, WebSearchState}; use shared_progress::{EventState, OperationId, ProgressEvent, ProgressHub, ProgressTree}; -use shared_protocol::ProtocolError; /// Mutable live configuration held behind a lock so profile switches can swap /// routing and local children without rebuilding the axum router. diff --git a/crates/gateway/src/profile_switch.rs b/crates/gateway/src/profile_switch.rs index 9ceaeaabd..5130b672c 100644 --- a/crates/gateway/src/profile_switch.rs +++ b/crates/gateway/src/profile_switch.rs @@ -1266,7 +1266,7 @@ struct OldRuntimes { } impl OldRuntimes { - fn shutdown(self) -> Result<(), shared_protocol::ShutdownError> { + fn shutdown(self) -> Result<(), gateway_protocol::ShutdownError> { #[cfg(feature = "local")] let result = self.local.shutdown(); #[cfg(not(feature = "local"))] diff --git a/crates/shared-protocol/AGENTS.md b/crates/shared-protocol/AGENTS.md deleted file mode 100644 index f0ab3b1d6..000000000 --- a/crates/shared-protocol/AGENTS.md +++ /dev/null @@ -1,6 +0,0 @@ -# shared-protocol - -This crate owns the OpenAI wire protocol, bounded client behavior, and the upstream abstraction. - -- Local inference, routing, and HTTP handlers stay in their owning crates. -- Shared protocol errors do not name Gateway-local concepts. Upstream shutdown uses this crate's error vocabulary so no dependency points back into Gateway code. diff --git a/tools/document.md b/tools/document.md index e1833fa68..2734f2730 100644 --- a/tools/document.md +++ b/tools/document.md @@ -111,7 +111,7 @@ Template: the Tour. Dependency order. Each chapter builds on the last. Audience: the gateway operator. -Targets: `crates/gateway/`, `crates/gateway-config/`, `crates/gateway-config-ui/`, `crates/gateway-local/`, `crates/shared-loopback/`, `crates/shared-protocol/`, `crates/gateway-routing/`, `crates/gateway-stt/`, `crates/gateway-stt-engine/`, `crates/gateway-web-search/`, `crates/gateway-whisper-ffi/`, `gateway.local.example.toml`. +Targets: `crates/gateway/`, `crates/gateway-config/`, `crates/gateway-config-ui/`, `crates/gateway-local/`, `crates/shared-loopback/`, `crates/gateway-protocol/`, `crates/gateway-routing/`, `crates/gateway-stt/`, `crates/gateway-stt-engine/`, `crates/gateway-web-search/`, `crates/gateway-whisper-ffi/`, `gateway.local.example.toml`. Extract: what the operator configures and observes. Every configuration key and what it does. Profiles. The configuration UI. The HTTP endpoints. Startup and provisioning behavior. Profile switching. Health and logs. Noise: internal machinery as features (wire types, transport internals, test infrastructure) and the Rust public API. Most files yield zero or one operator-facing features. That is expected. The empty extractions are the proof. Output: `guide/src/gateway/`. diff --git a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md index a2cb80f86..cdac96f06 100644 --- a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md +++ b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md @@ -134,7 +134,7 @@ Commit exactly the working tree's `AGENTS.md` revisions together with the two `d -### Step 2: Rename shared-protocol to gateway-protocol and enforce the Shared rule +### Step 2: Rename shared-protocol to gateway-protocol and enforce the Shared rule [completed] - Component: rename-and-enforcement From a90bed3e23d4badd14b178e6683c7c3ecb394358 Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 07:52:00 -0700 Subject: [PATCH 23/26] Fix claims-release ordering and unix mode lint A store operation's access capability now drops after the operation and its observation but before the answer posts, so the claims it holds release before a resumed chain can acquire overlapping claims. Only the timing of the release changes, never whether an operation succeeds. The Unix file-mode helper also gains a lint expectation recording that its optional return exists to unify the platform signatures, since the non-Unix variant has no mode bits to return. - `drop(access);` places the capability clone's destruction after the operation and its observation and before the answer posts, and the accompanying comment names the claims-release ordering constraint the placement enforces. - `clippy::unnecessary_wraps` is now expected on the Unix `mode_of`, whose `Option` return unifies the platform signatures because the non-Unix variant returns `None`. - Neither `mode_of` nor the `drop(access);` ordering gains a regression test in this change. Design: new temporal-coupling @ crates/promptforge-core/src/execute/scheduler.rs::dispatch_store Plan: vibe/2026-09-12-2-dependency-rules-vfs-hook.md --- crates/promptforge-core/src/execute/scheduler.rs | 6 ++++++ crates/shared-vfs/src/host.rs | 4 ++++ vibe/2026-09-12-2-dependency-rules-vfs-hook.md | 2 +- 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/crates/promptforge-core/src/execute/scheduler.rs b/crates/promptforge-core/src/execute/scheduler.rs index bdca7175c..c93cf2391 100644 --- a/crates/promptforge-core/src/execute/scheduler.rs +++ b/crates/promptforge-core/src/execute/scheduler.rs @@ -1786,6 +1786,12 @@ impl<'a> Scheduler<'a> { if result.is_ok() { succeeded } else { failed }, ); } + // Claims-release ordering constraint: the access clone must + // drop after the op and its observation and before the answer + // posts, so the claims it holds release before a resumed chain + // can acquire overlapping claims; the fix changes when claims + // release, never whether an operation succeeds. + drop(access); // A send fails only when the driver is gone (a cancelled run); // the answer is then moot. let _ = tx.send(( diff --git a/crates/shared-vfs/src/host.rs b/crates/shared-vfs/src/host.rs index 185670298..27af5aace 100644 --- a/crates/shared-vfs/src/host.rs +++ b/crates/shared-vfs/src/host.rs @@ -249,6 +249,10 @@ fn file_type_of(file_type: fs::FileType) -> FileType { /// POSIX mode bits where the host tracks them. #[cfg(unix)] +#[expect( + clippy::unnecessary_wraps, + reason = "the not(unix) variant returns None; the Option unifies the platform signatures" +)] fn mode_of(metadata: &fs::Metadata) -> Option { use std::os::unix::fs::PermissionsExt; Some(metadata.permissions().mode()) diff --git a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md index cdac96f06..8cda511e8 100644 --- a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md +++ b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md @@ -144,7 +144,7 @@ One coupled commit - the new Shared rule fails against the pre-rename workspace, -### Step 3: Fix the two PR #35 code failures +### Step 3: Fix the two PR #35 code failures [completed] - Component: ci-code-fixes From 51755f087e281912014980ce8796ca4576e09f3c Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 08:16:35 -0700 Subject: [PATCH 24/26] Add the VFS operation-observation hook Every filesystem capability now carries a caller-supplied origin, a label plus the most precise source position the caller knows, and a handle with an installed operation sink fires it on every admitted operation, after policy and claims pass and before the backend executes. The origin is pure observability: claims still key on the internal identity, a policy-denied operation never fires, and a mounted-handle forward fires nothing rather than doubling the event with a less precise label. The executor and the agent stamp the prompt's position explicitly, while host code, probes, and tests get the Rust call site stamped for free. - `Origin` is a non-exhaustive struct with mandatory label, file, and line; `Origin::new` stamps the Rust call site through `#[track_caller]`, and `Origin::at` sets an explicit position so the executor and agent can substitute the prompt's position. - `on_op` installs the operation sink, which must be cheap because store operations fire it inline from the blocking pool; `policy` installs the handle's policy the same way, defaulting to `AllowAll`. - `fire` runs after policy and claims pass and before the backend executes; rename and copy fire one event per canonical path, and the crate-private mount forward carries no origin so the outer handle's event is never doubled. - `crates/shared-vfs/AGENTS.md` gains the most-specific-label rule - a section name for a chain, a tool id for a tool, a fixture name for a test - so the guidance lands as instructions, not only doc comments. - `acquire` and `spawn` now take a mandatory origin, and every call site in this change passes one: the scheduler derives prompt positions through `prompt_origin` and `first_chunk_line`, while fixtures and probes label themselves. - `a_sink_receives_events_in_order_with_op_path_and_label` proves ordering, op, path, and label through a recording sink, and a policy-denied operation is proven never to fire; the only sink installations in this change are those tests, with the bounded event log, the Lua pull query, and enrichment policies named in the module docs as later subscribers. Design: new event-hook @ crates/shared-vfs/src/router.rs::VfsRefBuilder::on_op boundary: pub Design: new surface-growth @ crates/shared-vfs/src/handle.rs::VfsRef::acquire boundary: pub Design: new pure-function @ crates/promptforge-core/src/execute/scheduler.rs::first_chunk_line deps: Block Design: new pure-function @ crates/promptforge-core/src/execute/scheduler.rs::prompt_origin deps: Block,Prompt,str Plan: vibe/2026-09-12-2-dependency-rules-vfs-hook.md --- crates/promptforge-agent/src/agent.rs | 19 +- crates/promptforge-agent/src/tests.rs | 7 +- crates/promptforge-core/src/execute.rs | 5 +- .../promptforge-core/src/execute/scheduler.rs | 47 ++- .../src/execute/tests/exec_flow.rs | 2 +- .../promptforge-core/src/execute/tests/mod.rs | 12 +- crates/promptforge-core/src/lua/coro_tests.rs | 2 +- .../promptforge-core/src/model/tests/mod.rs | 2 +- .../promptforge-core/tests/suite/support.rs | 6 +- crates/promptforge-core/tests/suite/vfs.rs | 18 +- crates/promptforge-lua/benches/surface.rs | 2 +- crates/promptforge-lua/src/messages/tests.rs | 2 +- crates/promptforge-lua/src/models/tests.rs | 2 +- crates/promptforge-lua/src/tests.rs | 6 +- crates/promptforge-lua/src/tools/tests.rs | 2 +- crates/promptforge-lua/src/vm.rs | 10 +- crates/promptforge-store/src/error.rs | 12 +- crates/promptforge-store/src/lib.rs | 44 ++- crates/promptforge-store/src/tests.rs | 22 +- crates/promptforge-vfs/src/lib.rs | 15 +- crates/shared-vfs/AGENTS.md | 1 + crates/shared-vfs/src/handle.rs | 313 +++++++++++++----- crates/shared-vfs/src/lib.rs | 5 +- crates/shared-vfs/src/memory.rs | 4 +- crates/shared-vfs/src/observe.rs | 125 +++++++ crates/shared-vfs/src/router.rs | 68 +++- .../2026-09-12-2-dependency-rules-vfs-hook.md | 2 +- 27 files changed, 598 insertions(+), 157 deletions(-) create mode 100644 crates/shared-vfs/src/observe.rs diff --git a/crates/promptforge-agent/src/agent.rs b/crates/promptforge-agent/src/agent.rs index e78994854..b69c903df 100644 --- a/crates/promptforge-agent/src/agent.rs +++ b/crates/promptforge-agent/src/agent.rs @@ -45,7 +45,7 @@ use promptforge_model_client::model::{ }; use promptforge_store::Access; use promptforge_tools::ToolCatalog; -use shared_vfs::VfsRef; +use shared_vfs::{Origin, VfsRef}; use crate::config::AgentConfig; @@ -230,11 +230,16 @@ async fn drive( // exists - the section drivers' contract. vm.apply_lua_limits(limits.lua_memory_bytes, limits.lua_log_events)?; // The agent is one serial thread of execution: one capability for the - // whole run, released when it drops at the run's end. - let access = Arc::new(vfs.acquire().map_err(|error| AgentError::Program { - message: format!("the store capability acquisition failed: {error}"), - source: Some(Box::new(error)), - })?); + // whole run, released when it drops at the run's end. Its origin is + // the agent's own: the program is one chunk starting at its first + // line, and the agent's name is its source's name. + let access = Arc::new( + vfs.acquire(Origin::at(name.as_str(), name.as_str(), 1)) + .map_err(|error| AgentError::Program { + message: format!("the store capability acquisition failed: {error}"), + source: Some(Box::new(error)), + })?, + ); let (counts, events) = match setup_agent_vm(&mut vm, &access, &observer, &name, &tool_set, event_log, ui) { Ok(installed) => installed, @@ -910,7 +915,7 @@ mod tests { path: &str, ) -> std::result::Result { let access = vfs - .acquire() + .acquire(Origin::new("read_store")) .map_err(promptforge_store::StoreError::backend)?; promptforge_store::StoreExt::store(vfs, &access).read(path) } diff --git a/crates/promptforge-agent/src/tests.rs b/crates/promptforge-agent/src/tests.rs index 4d32ea9e6..e78c1b812 100644 --- a/crates/promptforge-agent/src/tests.rs +++ b/crates/promptforge-agent/src/tests.rs @@ -28,7 +28,7 @@ use promptforge_model_client::client::{GatewayClient, GatewayEndpoint, SecretStr use promptforge_model_client::model::{ModelCatalog, ModelDescriptor, ModelId, ThinkingMode}; use promptforge_store::StoreExt; use promptforge_tools::{Tool, ToolCatalog, ToolError, ToolId, ToolOutput}; -use shared_vfs::VfsRef; +use shared_vfs::{Origin, VfsRef}; use crate::agent::run_agent_with_client; use crate::{AgentConfig, AgentError, AgentLimits, run_agent}; @@ -562,7 +562,10 @@ impl FixtureRun { /// immediately dropped access: the run's identity dropped with it, so /// nothing it wrote conflicts with the extraction. fn read(&self, path: &str) -> String { - let access = self.vfs.acquire().expect("the stock backend acquires"); + let access = self + .vfs + .acquire(Origin::new("FixtureRun::read")) + .expect("the stock backend acquires"); self.vfs .store(&access) .read(path) diff --git a/crates/promptforge-core/src/execute.rs b/crates/promptforge-core/src/execute.rs index 2a419866f..a025b74b9 100644 --- a/crates/promptforge-core/src/execute.rs +++ b/crates/promptforge-core/src/execute.rs @@ -272,7 +272,10 @@ pub async fn run( /// throwaway overlay. The probe's identity and claim release with the /// access. fn store_mount_present(vfs: &VfsRef) -> std::result::Result { - match vfs.acquire()?.stat(promptforge_vfs::STORE_MOUNT) { + match vfs + .acquire(shared_vfs::Origin::new("store mount probe"))? + .stat(promptforge_vfs::STORE_MOUNT) + { Ok(_) => Ok(true), Err(shared_vfs::VfsError::NotFound(_)) => Ok(false), Err(error) => Err(error), diff --git a/crates/promptforge-core/src/execute/scheduler.rs b/crates/promptforge-core/src/execute/scheduler.rs index c93cf2391..b00b91b77 100644 --- a/crates/promptforge-core/src/execute/scheduler.rs +++ b/crates/promptforge-core/src/execute/scheduler.rs @@ -60,6 +60,7 @@ use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; use mlua::{RegistryKey, Thread}; +use shared_vfs::Origin; use tokio::sync::mpsc; use tokio::task::AbortHandle; @@ -75,7 +76,7 @@ use crate::lua::{ }; use crate::model::ModelBinding; use crate::observe::{Observation, Observer, detail}; -use crate::parser::{Block, Section}; +use crate::parser::{Block, Prompt, Section}; use crate::resolve::RuntimeResolution; use crate::store::{Access, Store, StoreError}; use crate::tools::{Tool, ToolId}; @@ -87,6 +88,26 @@ use super::engine::{ }; use super::gateway::{GatewaySource, ResolutionContext}; use super::protocol::{Answer, Request, StoreOp, ToolCallOutcome, YieldParse}; + +/// The most precise prompt-source line known for `blocks`: the first +/// compiled chunk's absolute source line, else the prompt's opening line. +fn first_chunk_line(blocks: &[Block]) -> u32 { + blocks + .iter() + .find_map(|block| match block { + Block::Lua(program) => Some(program.source_line().get()), + _ => None, + }) + .unwrap_or(1) +} + +/// The observability origin for a capability the run acquires: `label` is +/// the section or pass the capability serves, and the prompt's title +/// stands in for a file name - a prompt's name is its title, since the +/// source may never have lived on disk. +fn prompt_origin(prompt: &Prompt, label: &str, blocks: &[Block]) -> Origin { + Origin::at(label, prompt.title(), first_chunk_line(blocks)) +} use super::scope::prepare_effective_scope; use super::section_context::SectionContext; use super::support::{GENERIC_COMPLETION, MAX_CALL_DEPTH, next_id, now_rfc3339_checked}; @@ -770,7 +791,15 @@ impl<'a> Scheduler<'a> { /// # Errors /// Returns [`Error::Store`] when the backend refuses acquisition. fn install_root_slots(&mut self, root: ChainId) -> Result<()> { - let access = self.ctx.vfs().acquire().map_err(Error::Store)?; + // The walk capability serves every section in turn, so its label + // is the prompt's own; the line is where the walk starts. + let prompt = self.ctx.prompt(); + let blocks: &[Block] = prompt + .sections() + .first() + .map_or(&[], |section| section.blocks()); + let origin = prompt_origin(prompt, prompt.title(), blocks); + let access = self.ctx.vfs().acquire(origin).map_err(Error::Store)?; self.chains[root.index()].access = Some(Arc::new(access)); self.chains[root.index()].client = self.client.ready().cloned(); Ok(()) @@ -791,7 +820,14 @@ impl<'a> Scheduler<'a> { // The pass owns its client slot, seeded from the run's configured // client, exactly as the legacy pass seeds its own. let client = self.client.ready().cloned(); - let access = self.ctx.vfs().acquire().map_err(Error::Store)?; + // The live H1 pass runs under the prompt's title, from its first + // compiled H1 chunk. + let origin = prompt_origin( + self.ctx.prompt(), + self.ctx.prompt().title(), + self.ctx.prompt().h1_blocks(), + ); + let access = self.ctx.vfs().acquire(origin).map_err(Error::Store)?; self.chains.push(Chain { ctx: self.ctx.clone(), access: Some(Arc::new(access)), @@ -2310,8 +2346,9 @@ impl<'a> Scheduler<'a> { // capability spawns from the fanout caller's, retiring the // caller's claims (the happens-before edge), and drops with // the chain so a finished arm's claims never linger into the - // join's merge. - let access = template.access.spawn().map_err(Error::Store)?; + // join's merge. The arm's origin is the worker section's. + let origin = prompt_origin(template.ctx.prompt(), worker.name(), worker.blocks()); + let access = template.access.spawn(origin).map_err(Error::Store)?; self.chains[chain.index()].access = Some(Arc::new(access)); // The arm inherits the caller's client slot: an // already-resolved client is shared, an unresolved one stays diff --git a/crates/promptforge-core/src/execute/tests/exec_flow.rs b/crates/promptforge-core/src/execute/tests/exec_flow.rs index 5f9c2568f..b69351119 100644 --- a/crates/promptforge-core/src/execute/tests/exec_flow.rs +++ b/crates/promptforge-core/src/execute/tests/exec_flow.rs @@ -2318,7 +2318,7 @@ async fn a_mount_less_handle_runs_on_the_defensive_store_overlay() { // or the run's writes. assert!( matches!( - vfs.acquire() + vfs.acquire(shared_vfs::Origin::new("overlay absence probe")) .expect("the stock backend acquires") .stat(promptforge_vfs::STORE_MOUNT), Err(shared_vfs::VfsError::NotFound(_)) diff --git a/crates/promptforge-core/src/execute/tests/mod.rs b/crates/promptforge-core/src/execute/tests/mod.rs index 6ebfa0a6a..d7bca0447 100644 --- a/crates/promptforge-core/src/execute/tests/mod.rs +++ b/crates/promptforge-core/src/execute/tests/mod.rs @@ -38,7 +38,7 @@ use crate::untrusted::GuardNonce; fn fresh_access() -> Arc { Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(shared_vfs::Origin::new("execute test fixture")) .expect("the stock backend acquires"), ) } @@ -231,12 +231,18 @@ impl TestStore { } fn read(&self, path: &str) -> std::result::Result { - let access = self.0.acquire().map_err(StoreError::backend)?; + let access = self + .0 + .acquire(shared_vfs::Origin::new("TestStore::read")) + .map_err(StoreError::backend)?; self.0.store(&access).read(path) } fn glob(&self, pattern: &str) -> std::result::Result, StoreError> { - let access = self.0.acquire().map_err(StoreError::backend)?; + let access = self + .0 + .acquire(shared_vfs::Origin::new("TestStore::glob")) + .map_err(StoreError::backend)?; self.0.store(&access).glob(pattern) } } diff --git a/crates/promptforge-core/src/lua/coro_tests.rs b/crates/promptforge-core/src/lua/coro_tests.rs index 7963eda7c..57bef14d1 100644 --- a/crates/promptforge-core/src/lua/coro_tests.rs +++ b/crates/promptforge-core/src/lua/coro_tests.rs @@ -116,7 +116,7 @@ fn scheduler_vm_with_tools( let sys = json!({}); let access = Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(shared_vfs::Origin::new("coroutine test fixture")) .expect("the stock backend acquires"), ); let setup = SectionVmSetup { diff --git a/crates/promptforge-core/src/model/tests/mod.rs b/crates/promptforge-core/src/model/tests/mod.rs index 8f51332a1..9c9a60e82 100644 --- a/crates/promptforge-core/src/model/tests/mod.rs +++ b/crates/promptforge-core/src/model/tests/mod.rs @@ -23,7 +23,7 @@ const EXECUTION: &str = "model-bind-test"; fn fresh_access() -> Arc { Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(shared_vfs::Origin::new("model test fixture")) .expect("the stock backend acquires"), ) } diff --git a/crates/promptforge-core/tests/suite/support.rs b/crates/promptforge-core/tests/suite/support.rs index b5dae12e3..a8bd2ffe3 100644 --- a/crates/promptforge-core/tests/suite/support.rs +++ b/crates/promptforge-core/tests/suite/support.rs @@ -12,6 +12,7 @@ use promptforge_core::parser::Prompt; use promptforge_core::store::{StoreError, StoreExt, VfsRef}; use promptforge_tool_picker::{Catalog, Config, ToolPicker}; use promptforge_tools::{Tool, ToolCatalog}; +use shared_vfs::Origin; /// One correlated observation: which execution and section emitted it, plus the /// rendered event detail the fixtures assert on. @@ -111,7 +112,10 @@ pub(super) struct FixtureStore(VfsRef); impl FixtureStore { /// Reads a store path through a fresh, immediately dropped access. pub(super) fn read(&self, path: &str) -> Result { - let access = self.0.acquire().map_err(StoreError::backend)?; + let access = self + .0 + .acquire(Origin::new("FixtureStore::read")) + .map_err(StoreError::backend)?; self.0.store(&access).read(path) } } diff --git a/crates/promptforge-core/tests/suite/vfs.rs b/crates/promptforge-core/tests/suite/vfs.rs index e74cf08ca..56659f264 100644 --- a/crates/promptforge-core/tests/suite/vfs.rs +++ b/crates/promptforge-core/tests/suite/vfs.rs @@ -6,7 +6,7 @@ use promptforge_core::parser::Prompt; use promptforge_core::store::{Store, StoreError, StoreExt}; -use shared_vfs::{HostBackend, VfsRef}; +use shared_vfs::{HostBackend, Origin, VfsRef}; use super::support::{RunOptions, parse_execution_fixture, run, run_fixture}; use crate::support::Recorder; @@ -79,7 +79,9 @@ fn seed_declared_input(vfs: &VfsRef, prompt: &Prompt, contents: &str) { .frontmatter() .input() .expect("the fixture declares an input"); - let access = vfs.acquire().expect("the stock backend acquires"); + let access = vfs + .acquire(Origin::new("seed_declared_input")) + .expect("the stock backend acquires"); vfs.store(&access) .write(input.path(), contents) .expect("the declared input seeds"); @@ -141,7 +143,9 @@ return store.read('handoff.txt')\n\ assert_eq!(result, "across the reset"); // Extraction after the run takes a fresh access: the run's identities // dropped with it, so nothing the run touched can conflict here. - let access = vfs.acquire().expect("the stock backend acquires"); + let access = vfs + .acquire(Origin::new("stock handle extraction")) + .expect("the stock backend acquires"); assert_eq!( vfs.store(&access) .read("handoff.txt") @@ -165,7 +169,9 @@ async fn a_host_seeds_and_extracts_through_the_stock_handle_with_no_real_files() .await .expect("the seeded run executes offline"); assert_eq!(result, "done"); - let access = vfs.acquire().expect("the stock backend acquires"); + let access = vfs + .acquire(Origin::new("round-trip extraction")) + .expect("the stock backend acquires"); let report = extract_declared_output(&vfs.store(&access), &prompt) .expect("the run left its promised output"); assert_eq!(report, "report on: the paper body"); @@ -188,7 +194,9 @@ async fn a_missing_declared_output_is_a_contract_error_naming_the_prompts_promis .await .expect("the run itself succeeds"); assert_eq!(result, "read: the paper body"); - let access = vfs.acquire().expect("the stock backend acquires"); + let access = vfs + .acquire(Origin::new("missing-output extraction")) + .expect("the stock backend acquires"); let error = extract_declared_output(&vfs.store(&access), &prompt) .expect_err("the missing output is a contract error"); assert!( diff --git a/crates/promptforge-lua/benches/surface.rs b/crates/promptforge-lua/benches/surface.rs index 6583438f8..3ceb59430 100644 --- a/crates/promptforge-lua/benches/surface.rs +++ b/crates/promptforge-lua/benches/surface.rs @@ -46,7 +46,7 @@ fn builder_vm() -> SectionVm { &json!({}), &std::sync::Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(shared_vfs::Origin::new("surface bench")) .expect("the stock backend acquires"), ), ) diff --git a/crates/promptforge-lua/src/messages/tests.rs b/crates/promptforge-lua/src/messages/tests.rs index 99744669d..88313499c 100644 --- a/crates/promptforge-lua/src/messages/tests.rs +++ b/crates/promptforge-lua/src/messages/tests.rs @@ -11,7 +11,7 @@ use crate::{Error, SectionVm}; fn fresh_access() -> std::sync::Arc { std::sync::Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(shared_vfs::Origin::new("messages test fixture")) .expect("the stock backend acquires"), ) } diff --git a/crates/promptforge-lua/src/models/tests.rs b/crates/promptforge-lua/src/models/tests.rs index e2d133a44..7bd7c30d3 100644 --- a/crates/promptforge-lua/src/models/tests.rs +++ b/crates/promptforge-lua/src/models/tests.rs @@ -337,7 +337,7 @@ fn h2_vm(raw_ids: bool) -> crate::SectionVm { &serde_json::json!({}), &std::sync::Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(shared_vfs::Origin::new("models test fixture")) .expect("the stock backend acquires"), ), ) diff --git a/crates/promptforge-lua/src/tests.rs b/crates/promptforge-lua/src/tests.rs index a87601bf3..deb20564d 100644 --- a/crates/promptforge-lua/src/tests.rs +++ b/crates/promptforge-lua/src/tests.rs @@ -7,7 +7,7 @@ use promptforge_core_support::observe::{NullObserver, Observation}; use promptforge_store::Store; use promptforge_tools::{Tool, ToolError, ToolOutput}; use serde_json::json; -use shared_vfs::{ExecId, Vfs, VfsAccess, VfsError, VfsPath, VfsRef}; +use shared_vfs::{ExecId, Origin, Vfs, VfsAccess, VfsError, VfsPath, VfsRef}; const EXECUTION: &str = "lua-test"; @@ -17,7 +17,7 @@ const EXECUTION: &str = "lua-test"; fn fresh_access() -> Arc { Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(Origin::new("lua test fixture")) .expect("the stock backend acquires"), ) } @@ -140,7 +140,7 @@ impl VfsAccess for FailingAccess { fn failing_access() -> Arc { Arc::new( VfsRef::new(FailingBackend) - .acquire() + .acquire(Origin::new("failing backend test")) .expect("the failing backend still acquires"), ) } diff --git a/crates/promptforge-lua/src/tools/tests.rs b/crates/promptforge-lua/src/tools/tests.rs index e3bdb46e2..e92d319fc 100644 --- a/crates/promptforge-lua/src/tools/tests.rs +++ b/crates/promptforge-lua/src/tools/tests.rs @@ -16,7 +16,7 @@ use std::sync::{Arc, Mutex}; fn fresh_access() -> Arc { Arc::new( promptforge_vfs::empty() - .acquire() + .acquire(shared_vfs::Origin::new("tool test fixture")) .expect("the stock backend acquires"), ) } diff --git a/crates/promptforge-lua/src/vm.rs b/crates/promptforge-lua/src/vm.rs index 3ee80cf0c..1a534dbb7 100644 --- a/crates/promptforge-lua/src/vm.rs +++ b/crates/promptforge-lua/src/vm.rs @@ -410,7 +410,10 @@ impl SectionVm { /// /// let nonce = GuardNonce::fresh(); /// let vfs = promptforge_vfs::empty(); - /// let access = std::sync::Arc::new(vfs.acquire().expect("the stock backend acquires")); + /// let access = std::sync::Arc::new( + /// vfs.acquire(shared_vfs::Origin::new("vm example")) + /// .expect("the stock backend acquires"), + /// ); /// let mut vm = SectionVm::new(&nonce, "example-run", &NullObserver::default(), "Example")?; /// vm.inject_host("input", &serde_json::json!({ "id": 1 }), &access)?; /// vm.teardown(&NullObserver::default(), "Example"); @@ -783,7 +786,10 @@ impl SectionVm { /// /// let nonce = GuardNonce::fresh(); /// let vfs = promptforge_vfs::empty(); - /// let access = std::sync::Arc::new(vfs.acquire().expect("the stock backend acquires")); + /// let access = std::sync::Arc::new( + /// vfs.acquire(shared_vfs::Origin::new("vm example")) + /// .expect("the stock backend acquires"), + /// ); /// let mut vm = SectionVm::new(&nonce, "example-run", &NullObserver::default(), "Example")?; /// vm.inject_host("", &serde_json::json!({}), &access)?; /// assert_eq!(vm.var()?, serde_json::json!({})); diff --git a/crates/promptforge-store/src/error.rs b/crates/promptforge-store/src/error.rs index 6e914390f..e83c8aec4 100644 --- a/crates/promptforge-store/src/error.rs +++ b/crates/promptforge-store/src/error.rs @@ -189,7 +189,9 @@ impl StoreError { /// use promptforge_store::{StoreErrorKind, StoreExt}; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().expect("the stock backend acquires"); + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store error example")) + /// .expect("the stock backend acquires"); /// let store = vfs.store(&access); /// let err = store.read("missing.txt").unwrap_err(); /// assert_eq!(err.kind(), StoreErrorKind::NotFound); @@ -217,7 +219,9 @@ impl StoreError { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().expect("the stock backend acquires"); + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store error example")) + /// .expect("the stock backend acquires"); /// let store = vfs.store(&access); /// let err = store.read("missing.txt").unwrap_err(); /// assert!(err.is_not_found()); @@ -234,7 +238,9 @@ impl StoreError { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().expect("the stock backend acquires"); + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store error example")) + /// .expect("the stock backend acquires"); /// let store = vfs.store(&access); /// let err = store.read("missing.txt").unwrap_err(); /// assert_eq!(err.path(), Some("missing.txt")); diff --git a/crates/promptforge-store/src/lib.rs b/crates/promptforge-store/src/lib.rs index b24a99bd3..1cac59cbb 100644 --- a/crates/promptforge-store/src/lib.rs +++ b/crates/promptforge-store/src/lib.rs @@ -52,7 +52,9 @@ pub(crate) const MAX_GLOB_PATTERN_BYTES: usize = 1024; /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); -/// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; +/// let access = vfs +/// .acquire(shared_vfs::Origin::new("store example")) +/// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("shared.txt", "state")?; /// assert_eq!(store.read("shared.txt")?, "state"); @@ -90,7 +92,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "hi")?; /// # Ok::<(), promptforge_store::StoreError>(()) @@ -114,7 +118,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.append("a.txt", "hi")?; /// # Ok::<(), promptforge_store::StoreError>(()) @@ -140,7 +146,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "hi\n")?; /// assert_eq!(store.read("a.txt")?, "hi\n"); @@ -171,7 +179,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "one\ntwo\nthree\n")?; /// assert_eq!(store.read_range("a.txt", 2, None)?, "two\nthree"); @@ -210,7 +220,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "one\ntwo\nthree\n")?; /// assert_eq!( @@ -264,7 +276,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "one two")?; /// store.str_replace("a.txt", "two", "three")?; @@ -314,7 +328,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "hi")?; /// store.delete("a.txt")?; @@ -350,7 +366,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("a.txt", "")?; /// store.write("b.md", "")?; @@ -427,7 +445,9 @@ impl Store<'_> { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// assert!(!store.exists("a.txt")?); /// store.write("a.txt", "hi")?; @@ -457,7 +477,9 @@ pub trait StoreExt { /// use promptforge_store::StoreExt; /// /// let vfs = promptforge_vfs::empty(); - /// let access = vfs.acquire().map_err(promptforge_store::StoreError::backend)?; + /// let access = vfs + /// .acquire(shared_vfs::Origin::new("store example")) + /// .map_err(promptforge_store::StoreError::backend)?; /// let store = vfs.store(&access); /// store.write("seeded.txt", "input")?; /// # Ok::<(), promptforge_store::StoreError>(()) diff --git a/crates/promptforge-store/src/tests.rs b/crates/promptforge-store/src/tests.rs index 585bb2ab2..345171c88 100644 --- a/crates/promptforge-store/src/tests.rs +++ b/crates/promptforge-store/src/tests.rs @@ -7,7 +7,7 @@ use promptforge_vfs::STORE_MOUNT; use shared_vfs::{ - Access, Entry, ExecId, MemoryBackend, Stat, Vfs, VfsAccess, VfsError, VfsPath, VfsRef, + Access, Entry, ExecId, MemoryBackend, Origin, Stat, Vfs, VfsAccess, VfsError, VfsPath, VfsRef, }; use super::path::MAX_STORE_PATH_BYTES; @@ -17,7 +17,9 @@ use super::{MAX_GLOB_PATTERN_BYTES, PathReason, Store, StoreError, StoreErrorKin /// single-identity test starts from. fn stock() -> (VfsRef, Access) { let vfs = promptforge_vfs::empty(); - let access = vfs.acquire().expect("the stock backend acquires"); + let access = vfs + .acquire(Origin::new("store parity test")) + .expect("the stock backend acquires"); (vfs, access) } @@ -80,7 +82,9 @@ fn a_second_identitys_write_to_a_claimed_path_races() { store .write("a.txt", "uno") .expect("one identity may rewrite its own path"); - let second = vfs.acquire().expect("the stock backend acquires"); + let second = vfs + .acquire(Origin::new("store parity test")) + .expect("the stock backend acquires"); let contender = vfs.store(&second); let err = contender .write("a.txt", "two") @@ -115,7 +119,9 @@ fn a_glob_over_a_claimed_path_races_like_a_read() { let (vfs, first) = stock(); let store = vfs.store(&first); store.write("a.txt", "one").expect("first write"); - let second = vfs.acquire().expect("the stock backend acquires"); + let second = vfs + .acquire(Origin::new("store parity test")) + .expect("the stock backend acquires"); let contender = vfs.store(&second); let err = contender .glob("*.txt") @@ -147,7 +153,9 @@ fn identities_share_backing_state_once_claims_are_released() { .write("shared.txt", "written by the first") .expect("write"); drop(first); - let second = vfs.acquire().expect("the stock backend acquires"); + let second = vfs + .acquire(Origin::new("store parity test")) + .expect("the stock backend acquires"); let reader = vfs.store(&second); assert_eq!( reader.read("shared.txt").expect("read"), @@ -777,7 +785,9 @@ fn a_panicking_operation_does_not_wedge_the_store() { let vfs = VfsRef::builder() .mount(STORE_MOUNT, PanicBackend(MemoryBackend::new())) .build(); - let access = vfs.acquire().expect("the stock backend acquires"); + let access = vfs + .acquire(Origin::new("store parity test")) + .expect("the stock backend acquires"); let store = vfs.store(&access); let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| store.write("a.txt", "x"))); diff --git a/crates/promptforge-vfs/src/lib.rs b/crates/promptforge-vfs/src/lib.rs index 77995d522..8c74208db 100644 --- a/crates/promptforge-vfs/src/lib.rs +++ b/crates/promptforge-vfs/src/lib.rs @@ -140,14 +140,14 @@ impl Policy for ModePolicy { #[cfg(test)] mod tests { - use shared_vfs::{VfsError, VfsRef}; + use shared_vfs::{Origin, VfsError, VfsRef}; use super::{Mode, ModePolicy, STORE_MOUNT, empty}; #[test] fn empty_carries_the_store_mount() -> Result<(), VfsError> { let vfs = empty(); - let access = vfs.acquire()?; + let access = vfs.acquire(Origin::new("empty store mount test"))?; let path = format!("{STORE_MOUNT}/paper.md"); access.write(&path, b"# draft")?; assert_eq!(access.read(&path)?, b"# draft"); @@ -159,7 +159,7 @@ mod tests { #[test] fn empty_serves_nothing_outside_the_store_mount() -> Result<(), VfsError> { let vfs = empty(); - let access = vfs.acquire()?; + let access = vfs.acquire(Origin::new("empty namespace test"))?; assert!(matches!( access.read("/elsewhere.txt"), Err(VfsError::NotFound(_)) @@ -173,7 +173,7 @@ mod tests { let policy = ModePolicy::new(Mode::Ask); let handle = policy.handle(); let vfs = VfsRef::with_policy(empty(), policy); - let access = vfs.acquire()?; + let access = vfs.acquire(Origin::new("mode flip test"))?; let path = format!("{STORE_MOUNT}/notes.md"); match access.write(&path, b"x") { Err(VfsError::PermissionDenied(reason)) => { @@ -196,7 +196,7 @@ mod tests { fn plan_mode_allows_mutations_only_to_markdown_paths() -> Result<(), VfsError> { let policy = ModePolicy::new(Mode::Plan); let vfs = VfsRef::with_policy(empty(), policy); - let access = vfs.acquire()?; + let access = vfs.acquire(Origin::new("plan mode test"))?; let markdown = format!("{STORE_MOUNT}/notes.md"); let binary = format!("{STORE_MOUNT}/data.bin"); access.write(&markdown, b"# ok")?; @@ -220,10 +220,11 @@ mod tests { let handle = policy.handle(); let vfs = VfsRef::with_policy(empty(), policy); let path = format!("{STORE_MOUNT}/paper.md"); - vfs.acquire()?.write(&path, b"text")?; + vfs.acquire(Origin::new("read gate test"))? + .write(&path, b"text")?; // Even in Ask, the strictest mode, reads flow. handle.set(Mode::Ask); - let access = vfs.acquire()?; + let access = vfs.acquire(Origin::new("read gate test"))?; assert_eq!(access.read(&path)?, b"text"); assert!(access.exists(&path)?); Ok(()) diff --git a/crates/shared-vfs/AGENTS.md b/crates/shared-vfs/AGENTS.md index c35ff313d..9771f091d 100644 --- a/crates/shared-vfs/AGENTS.md +++ b/crates/shared-vfs/AGENTS.md @@ -5,3 +5,4 @@ Generic virtual filesystem machinery: the permanent bottom of the dependency sta - std only. No dependencies, workspace or external. The manifest test enforces this; never weaken it. - No promptforge policy: no /_promptforge paths, no Store, no run concepts. - The public surface is load-bearing: add defaulted methods, never change existing signatures. Every edit rebuilds the whole stack. +- Origin labels are most-specific: a section name for a chain, a tool id for a tool, a fixture name for a test - never a generic label when a specific one exists. diff --git a/crates/shared-vfs/src/handle.rs b/crates/shared-vfs/src/handle.rs index 0e4c73208..50d199f02 100644 --- a/crates/shared-vfs/src/handle.rs +++ b/crates/shared-vfs/src/handle.rs @@ -7,13 +7,16 @@ //! so a denied operation never registers a claim, registers claims, and //! locks the backend's access object per call. Dropping an [`Access`] //! releases the identity and its claims, so cancellation, panics, and -//! early returns cannot leak claims. +//! early returns cannot leak claims. A handle with an installed op sink +//! fires it on every admitted operation - after policy and claims pass, +//! before the backend executes; see [`crate::observe`]. use std::collections::{HashMap, HashSet}; use std::fmt; use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use crate::error::VfsError; +use crate::observe::{OpEvent, OpSink, Origin}; use crate::path::{VfsPath, canonicalize}; use crate::router::{Mounts, Router, VfsRefBuilder}; use crate::traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; @@ -154,6 +157,7 @@ fn conflict( struct Volume { backend: Arc>>, claims: Arc, + sink: Option, } /// The cloneable handle over one backend and its claims ledger. @@ -190,6 +194,7 @@ impl VfsRef { volume: Arc::new(Volume { backend: Arc::new(Mutex::new(Box::new(backend))), claims: Arc::new(Claims::new()), + sink: None, }), policy: Arc::new(policy), } @@ -232,6 +237,9 @@ impl VfsRef { volume: Arc::new(Volume { backend: Arc::new(Mutex::new(Box::new(Router::new(mounts)))), claims: Arc::clone(&self.volume.claims), + // One sink observes both views of the same storage, fired + // by the outer capability with the caller's origin. + sink: self.volume.sink.clone(), }), policy: Arc::clone(&self.policy), } @@ -239,26 +247,36 @@ impl VfsRef { /// Acquires the capability for a new serial thread of execution. /// This is the only way in: every acquire vends a fresh [`ExecId`]. + /// `origin` is pure observability: it labels every operation event + /// this capability fires and never gates anything. /// /// # Errors /// Returns an error when the backend refuses to acquire the identity. - pub fn acquire(&self) -> Result { - self.acquire_with(ExecId::vend()) + pub fn acquire(&self, origin: Origin) -> Result { + self.acquire_with(ExecId::vend(), Some(origin)) } /// Acquires the capability under a given identity: how a mounted /// handle forwards the caller's attribution. The identity registers /// as live in this handle's claims table, so conflicts are detected - /// across both views of the same storage. + /// across both views of the same storage. A `None` origin is the + /// mount forward: the outer handle already fired the caller's origin, + /// so the forward fires nothing rather than double the event with a + /// fabricated, less precise one. /// /// # Errors /// Returns an error when the backend refuses to acquire the identity; /// see [`VfsRef::acquire`]. - pub(crate) fn acquire_with(&self, id: ExecId) -> Result { + pub(crate) fn acquire_with( + &self, + id: ExecId, + origin: Option, + ) -> Result { let inner = self.backend().acquire(id)?; self.volume.claims.register_live(id); Ok(Access { id, + origin, volume: self.volume.clone(), policy: self.policy.clone(), inner: Mutex::new(inner), @@ -266,14 +284,19 @@ impl VfsRef { } /// Builds a handle over a router with a fresh claims table and the - /// [`AllowAll`] policy: the builder's exit. - pub(crate) fn from_router(router: Router) -> VfsRef { + /// installed policy and op sink: the builder's exit. + pub(crate) fn from_router( + router: Router, + policy: Arc, + sink: Option, + ) -> VfsRef { VfsRef { volume: Arc::new(Volume { backend: Arc::new(Mutex::new(Box::new(router))), claims: Arc::new(Claims::new()), + sink, }), - policy: Arc::new(AllowAll), + policy, } } @@ -288,10 +311,14 @@ impl VfsRef { /// The public capability. Holds an [`ExecId`] and the backend's access /// object; every operation canonicalizes the path, checks the policy, -/// checks the claims tables, then locks the backend per call. +/// checks the claims tables, fires the op sink, then locks the backend +/// per call. #[must_use = "an acquire dropped immediately is a bug: the capability carries the identity's claims"] pub struct Access { id: ExecId, + /// The caller-supplied observability origin; `None` only on the + /// crate-private mount forward, which never fires. + origin: Option, volume: Arc, policy: Arc, inner: Mutex>, @@ -303,18 +330,21 @@ impl Access { /// are deleted from the tables: they predate the child by /// construction, so a retired claim can never conflict again. The /// spawn IS the happens-before edge - no fence call, no epochs. + /// `origin` labels the child's operation events, exactly as in + /// [`VfsRef::acquire`]. /// /// # Errors /// Returns an error when the backend refuses to acquire the child's /// identity; see [`VfsRef::acquire`]. A failed spawn leaves this /// capability's claims untouched. - pub fn spawn(&self) -> Result { + pub fn spawn(&self, origin: Origin) -> Result { let id = ExecId::vend(); let inner = self.backend().acquire(id)?; self.volume.claims.retire(self.id); self.volume.claims.register_live(id); Ok(Access { id, + origin: Some(origin), volume: self.volume.clone(), policy: self.policy.clone(), inner: Mutex::new(inner), @@ -334,6 +364,7 @@ impl Access { /// fails. pub fn read(&self, path: &str) -> Result, VfsError> { let path = self.gate(Op::Read, path, ClaimKind::Read)?; + self.fire(Op::Read, &path); self.inner().read(&path) } @@ -396,6 +427,7 @@ impl Access { /// live identity holds a claim on `path`, or when the backend fails. pub fn write(&self, path: &str, contents: &[u8]) -> Result<(), VfsError> { let path = self.gate(Op::Write, path, ClaimKind::Write)?; + self.fire(Op::Write, &path); self.inner().write(&path, contents) } @@ -406,6 +438,7 @@ impl Access { /// live identity holds a claim on `path`, or when the backend fails. pub fn append(&self, path: &str, contents: &[u8]) -> Result<(), VfsError> { let path = self.gate(Op::Append, path, ClaimKind::Write)?; + self.fire(Op::Append, &path); self.inner().append(&path, contents) } @@ -418,6 +451,7 @@ impl Access { /// exactly one, or when the backend fails. pub fn str_replace(&self, path: &str, old: &str, new: &str) -> Result<(), VfsError> { let path = self.gate(Op::Write, path, ClaimKind::Write)?; + self.fire(Op::Write, &path); self.inner().str_replace(&path, old, new) } @@ -428,6 +462,7 @@ impl Access { /// live identity holds a claim on `path`, or when the backend fails. pub fn remove(&self, path: &str, recursive: bool) -> Result<(), VfsError> { let path = self.gate(Op::Delete, path, ClaimKind::Write)?; + self.fire(Op::Delete, &path); self.inner().remove(&path, recursive) } @@ -439,6 +474,7 @@ impl Access { /// fails. pub fn exists(&self, path: &str) -> Result { let path = self.gate(Op::Exists, path, ClaimKind::Read)?; + self.fire(Op::Exists, &path); self.inner().exists(&path) } @@ -450,7 +486,8 @@ impl Access { pub fn glob(&self, pattern: &str) -> Result, VfsError> { // The claim key is the canonicalized pattern; the backend // receives the pattern verbatim. - let _claimed = self.gate(Op::Glob, pattern, ClaimKind::Read)?; + let claimed = self.gate(Op::Glob, pattern, ClaimKind::Read)?; + self.fire(Op::Glob, &claimed); self.inner().glob(pattern) } @@ -462,6 +499,7 @@ impl Access { /// fails. pub fn list(&self, path: &str) -> Result, VfsError> { let path = self.gate(Op::List, path, ClaimKind::Read)?; + self.fire(Op::List, &path); self.inner().list(&path) } @@ -473,6 +511,7 @@ impl Access { /// fails. pub fn stat(&self, path: &str) -> Result { let path = self.gate(Op::Stat, path, ClaimKind::Read)?; + self.fire(Op::Stat, &path); self.inner().stat(&path) } @@ -483,6 +522,7 @@ impl Access { /// live identity holds a claim on `path`, or when the backend fails. pub fn mkdir(&self, path: &str, recursive: bool) -> Result<(), VfsError> { let path = self.gate(Op::Mkdir, path, ClaimKind::Write)?; + self.fire(Op::Mkdir, &path); self.inner().mkdir(&path, recursive) } @@ -496,6 +536,10 @@ impl Access { pub fn rename(&self, from: &str, to: &str) -> Result<(), VfsError> { let from = self.gate(Op::Rename, from, ClaimKind::Write)?; let to = self.gate(Op::Rename, to, ClaimKind::Write)?; + // Both paths gated, so the operation is admitted: one event per + // canonical path. + self.fire(Op::Rename, &from); + self.fire(Op::Rename, &to); self.inner().rename(&from, &to) } @@ -509,6 +553,8 @@ impl Access { pub fn copy(&self, from: &str, to: &str) -> Result<(), VfsError> { let from = self.gate(Op::Copy, from, ClaimKind::Read)?; let to = self.gate(Op::Copy, to, ClaimKind::Write)?; + self.fire(Op::Copy, &from); + self.fire(Op::Copy, &to); self.inner().copy(&from, &to) } @@ -521,7 +567,10 @@ impl Access { pub fn grep(&self, query: &GrepQuery) -> Result { let root = canonicalize(query.root.as_str())?; self.check_policy(Op::Grep, &root)?; - self.volume.claims.claim(root, self.id, ClaimKind::Read)?; + self.volume + .claims + .claim(root.clone(), self.id, ClaimKind::Read)?; + self.fire(Op::Grep, &root); self.inner().grep(query) } @@ -547,6 +596,18 @@ impl Access { } } + /// Fires the installed sink for one admitted path of `op` - + /// fire-and-forget, after policy and claims pass, before the backend + /// executes. A handle without a sink, or a capability vended to a + /// mounted-handle forward (the outer handle already fired with the + /// caller's origin), fires nothing. + fn fire(&self, op: Op, path: &VfsPath) { + let (Some(sink), Some(origin)) = (&self.volume.sink, &self.origin) else { + return; + }; + sink(OpEvent { op, path, origin }); + } + /// Reads the file and resolves one line range while its contents /// remain live. fn with_line_range( @@ -613,7 +674,10 @@ impl Drop for Access { /// base's policy and claims under the caller's identity. impl Vfs for VfsRef { fn acquire(&mut self, id: ExecId) -> Result, VfsError> { - Ok(Box::new(HandleAccess(self.acquire_with(id)?))) + // The mount forward carries no origin: the outer handle already + // fired the caller's, and a fabricated one here would double the + // event with a less precise label. + Ok(Box::new(HandleAccess(self.acquire_with(id, None)?))) } fn release(&mut self, id: ExecId) -> Result<(), VfsError> { @@ -698,6 +762,7 @@ mod tests { use super::{Access, VfsRef}; use crate::error::VfsError; + use crate::observe::{OpEvent, Origin}; use crate::path::VfsPath; use crate::traits::{ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; use crate::types::{Entry, Stat}; @@ -842,6 +907,27 @@ mod tests { VfsRef::new(stub.clone()) } + /// The claims tests never observe origins, so they acquire under one + /// blanket label; the observability tests label precisely. + fn test_origin() -> Origin { + Origin::new("handle test") + } + + /// A policy whose verdict flips through shared state mid-run. + struct FlipPolicy { + verdict: Arc>, + } + + impl Policy for FlipPolicy { + fn check(&self, op: Op, path: &VfsPath) -> Verdict { + let _ = (op, path); + self.verdict + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() + } + } + /// Extracts the Conflict message or fails the test. fn conflict_message(result: Result<(), VfsError>) -> String { match result { @@ -853,7 +939,7 @@ mod tests { #[test] fn an_access_reads_and_writes_through_the_handle() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; access.write("/notes/a.txt", b"hello")?; assert_eq!(access.read("/notes/a.txt")?, b"hello"); assert!(access.exists("/notes/a.txt")?); @@ -863,8 +949,8 @@ mod tests { #[test] fn every_acquire_vends_a_process_unique_identity() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let first = vfs.acquire()?; - let second = vfs.acquire()?; + let first = vfs.acquire(test_origin())?; + let second = vfs.acquire(test_origin())?; assert_ne!(first.id(), second.id()); Ok(()) } @@ -875,7 +961,7 @@ mod tests { // access, so sequential ops on one path by one identity stay // legal - no new identity, no false conflict. let vfs = handle(&StubFs::default()); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; access.write("/f.txt", b"one")?; access.write("/f.txt", b"two")?; access.append("/f.txt", b"!")?; @@ -886,8 +972,8 @@ mod tests { #[test] fn a_write_conflicts_with_another_identitys_read_claim() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); - let reader = vfs.acquire()?; - let writer = vfs.acquire()?; + let reader = vfs.acquire(test_origin())?; + let writer = vfs.acquire(test_origin())?; reader.read("/f.txt")?; let message = conflict_message(writer.write("/f.txt", b"new")); assert!(message.contains("/f.txt"), "names the path: {message}"); @@ -913,9 +999,9 @@ mod tests { #[test] fn a_read_conflicts_with_another_identitys_write_claim() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let writer = vfs.acquire()?; + let writer = vfs.acquire(test_origin())?; writer.write("/f.txt", b"x")?; - let reader = vfs.acquire()?; + let reader = vfs.acquire(test_origin())?; match reader.read("/f.txt") { Err(VfsError::Conflict(_)) => {} other => panic!("expected a conflict, got {other:?}"), @@ -926,9 +1012,9 @@ mod tests { #[test] fn two_writes_by_two_identities_conflict() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let first = vfs.acquire()?; + let first = vfs.acquire(test_origin())?; first.write("/f.txt", b"1")?; - let second = vfs.acquire()?; + let second = vfs.acquire(test_origin())?; let message = conflict_message(second.write("/f.txt", b"2")); assert!(message.contains("write claim"), "{message}"); Ok(()) @@ -937,8 +1023,8 @@ mod tests { #[test] fn reads_by_two_identities_never_conflict() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); - let first = vfs.acquire()?; - let second = vfs.acquire()?; + let first = vfs.acquire(test_origin())?; + let second = vfs.acquire(test_origin())?; first.read("/f.txt")?; assert_eq!(second.read("/f.txt")?, b"data"); Ok(()) @@ -949,9 +1035,9 @@ mod tests { // Copy claims the source as a read, and a read booms on another // live identity's write claim. let vfs = handle(&StubFs::default()); - let writer = vfs.acquire()?; + let writer = vfs.acquire(test_origin())?; writer.write("/src.txt", b"data")?; - let copier = vfs.acquire()?; + let copier = vfs.acquire(test_origin())?; let message = conflict_message(copier.copy("/src.txt", "/dst.txt")); assert!(message.contains("/src.txt"), "names the source: {message}"); Ok(()) @@ -963,9 +1049,9 @@ mod tests { // read claim on the source must not block the copy. Were the // source claimed as a write, this copy would conflict. let vfs = handle(&StubFs::seeded(&[("/src.txt", "data")])); - let reader = vfs.acquire()?; + let reader = vfs.acquire(test_origin())?; reader.read("/src.txt")?; - let copier = vfs.acquire()?; + let copier = vfs.acquire(test_origin())?; copier.copy("/src.txt", "/dst.txt")?; assert_eq!(copier.read("/dst.txt")?, b"data"); Ok(()) @@ -979,9 +1065,9 @@ mod tests { ("/src.txt", "data"), ("/dst.txt", "old"), ])); - let reader = vfs.acquire()?; + let reader = vfs.acquire(test_origin())?; reader.read("/dst.txt")?; - let copier = vfs.acquire()?; + let copier = vfs.acquire(test_origin())?; let message = conflict_message(copier.copy("/src.txt", "/dst.txt")); assert!( message.contains("/dst.txt"), @@ -993,9 +1079,9 @@ mod tests { #[test] fn a_rename_conflicts_with_a_claim_on_the_source_path() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/from.txt", "data")])); - let reader = vfs.acquire()?; + let reader = vfs.acquire(test_origin())?; reader.read("/from.txt")?; - let renamer = vfs.acquire()?; + let renamer = vfs.acquire(test_origin())?; let message = conflict_message(renamer.rename("/from.txt", "/to.txt")); assert!(message.contains("/from.txt"), "names the source: {message}"); Ok(()) @@ -1009,9 +1095,9 @@ mod tests { ("/from.txt", "data"), ("/to.txt", "old"), ])); - let reader = vfs.acquire()?; + let reader = vfs.acquire(test_origin())?; reader.read("/to.txt")?; - let renamer = vfs.acquire()?; + let renamer = vfs.acquire(test_origin())?; let message = conflict_message(renamer.rename("/from.txt", "/to.txt")); assert!( message.contains("/to.txt"), @@ -1024,12 +1110,12 @@ mod tests { fn dropping_an_access_releases_its_identity_and_claims() -> Result<(), VfsError> { let stub = StubFs::default(); let vfs = handle(&stub); - let first = vfs.acquire()?; + let first = vfs.acquire(test_origin())?; let first_id = first.id(); first.write("/f.txt", b"1")?; drop(first); assert!(stub.released().contains(&first_id)); - let second = vfs.acquire()?; + let second = vfs.acquire(test_origin())?; second.write("/f.txt", b"2")?; assert_eq!(second.read("/f.txt")?, b"2"); Ok(()) @@ -1038,9 +1124,9 @@ mod tests { #[test] fn spawn_deletes_the_parents_claims() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let parent = vfs.acquire()?; + let parent = vfs.acquire(test_origin())?; parent.write("/f.txt", b"1")?; - let child = parent.spawn()?; + let child = parent.spawn(test_origin())?; assert_ne!(parent.id(), child.id()); // The parent's pre-spawn write claim is retired: the child can // touch the same path without a false conflict. @@ -1055,11 +1141,11 @@ mod tests { // arm in turn; a dropped arm releases its claims, so the next arm // can merge onto the same path. let vfs = handle(&StubFs::default()); - let parent = vfs.acquire()?; - let arm_one = parent.spawn()?; + let parent = vfs.acquire(test_origin())?; + let arm_one = parent.spawn(test_origin())?; arm_one.write("/evidence.md", b"one\n")?; drop(arm_one); - let arm_two = parent.spawn()?; + let arm_two = parent.spawn(test_origin())?; arm_two.append("/evidence.md", b"two\n")?; assert_eq!(arm_two.read("/evidence.md")?, b"one\ntwo\n"); Ok(()) @@ -1068,12 +1154,12 @@ mod tests { #[test] fn transfer_of_control_moves_the_claims_with_the_access() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "data")])); - let original = vfs.acquire()?; + let original = vfs.acquire(test_origin())?; original.read("/f.txt")?; // Transfer of control moves the access object; the identity and // its claims move with it. let moved = original; - let other = vfs.acquire()?; + let other = vfs.acquire(test_origin())?; let message = conflict_message(other.write("/f.txt", b"new")); assert!(message.contains(&format!("{:?}", moved.id()))); assert_eq!(moved.read("/f.txt")?, b"data"); @@ -1083,9 +1169,9 @@ mod tests { #[test] fn alias_spellings_of_one_file_land_on_one_claim_key() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/a/b.txt", "x")])); - let reader = vfs.acquire()?; + let reader = vfs.acquire(test_origin())?; reader.read("/a/./b.txt")?; - let writer = vfs.acquire()?; + let writer = vfs.acquire(test_origin())?; let message = conflict_message(writer.write("/a//b.txt", b"y")); assert!(message.contains("/a/b.txt"), "the canonical key: {message}"); Ok(()) @@ -1095,9 +1181,9 @@ mod tests { fn claims_are_shared_across_handle_clones() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); let clone = vfs.clone(); - let first = vfs.acquire()?; + let first = vfs.acquire(test_origin())?; first.write("/f.txt", b"1")?; - let second = clone.acquire()?; + let second = clone.acquire(test_origin())?; let message = conflict_message(second.write("/f.txt", b"2")); assert!(message.contains("/f.txt"), "{message}"); Ok(()) @@ -1105,21 +1191,6 @@ mod tests { #[test] fn a_denied_operation_never_registers_a_claim() -> Result<(), VfsError> { - /// A policy whose verdict flips through shared state mid-run. - struct FlipPolicy { - verdict: Arc>, - } - - impl Policy for FlipPolicy { - fn check(&self, op: Op, path: &VfsPath) -> Verdict { - let _ = (op, path); - self.verdict - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone() - } - } - let verdict = Arc::new(Mutex::new(Verdict::Deny("writes are sealed".to_owned()))); let vfs = VfsRef::with_policy( StubFs::default(), @@ -1127,7 +1198,7 @@ mod tests { verdict: Arc::clone(&verdict), }, ); - let denied = vfs.acquire()?; + let denied = vfs.acquire(test_origin())?; match denied.write("/f.txt", b"x") { Err(VfsError::PermissionDenied(reason)) => { assert_eq!(reason, "writes are sealed"); @@ -1136,7 +1207,7 @@ mod tests { } // The host flips the policy mid-run through shared state. *verdict.lock().unwrap_or_else(PoisonError::into_inner) = Verdict::Allow; - let allowed = vfs.acquire()?; + let allowed = vfs.acquire(test_origin())?; // Had the denied attempt registered a write claim, this write // would conflict with it. allowed.write("/f.txt", b"x")?; @@ -1147,7 +1218,7 @@ mod tests { #[test] fn read_range_slices_lines_one_based_and_inclusive() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\nthree\n")])); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; assert_eq!(access.read_range("/f.txt", 2, None)?, "two\nthree"); assert_eq!(access.read_range("/f.txt", 2, Some(99))?, "two\nthree"); assert_eq!(access.read_range("/f.txt", 99, None)?, ""); @@ -1158,7 +1229,7 @@ mod tests { #[test] fn read_range_rejects_invalid_bounds() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\n")])); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; assert!(access.read_range("/f.txt", 0, None).is_err()); assert!(access.read_range("/f.txt", 2, Some(1)).is_err()); Ok(()) @@ -1167,7 +1238,7 @@ mod tests { #[test] fn read_range_numbered_numbers_absolutely_from_start() -> Result<(), VfsError> { let vfs = handle(&StubFs::seeded(&[("/f.txt", "one\ntwo\nthree\n")])); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; assert_eq!( access.read_range_numbered("/f.txt", 1, None)?, "1| one\n2| two\n3| three" @@ -1185,7 +1256,7 @@ mod tests { let lines: Vec = (1..=10).map(|n| format!("line{n}")).collect(); let text = lines.join("\n"); let vfs = handle(&StubFs::seeded(&[("/f.txt", &text)])); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; assert_eq!( access.read_range_numbered("/f.txt", 9, Some(10))?, " 9| line9\n10| line10" @@ -1196,7 +1267,7 @@ mod tests { #[test] fn read_string_rejects_non_utf8() -> Result<(), VfsError> { let vfs = handle(&StubFs::default()); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; access.write("/bin.dat", &[0xff, 0xfe])?; match access.read_string("/bin.dat") { Err(VfsError::Backend(_)) => {} @@ -1234,7 +1305,7 @@ mod tests { #[test] fn a_backend_refusal_fails_acquire_with_an_error_instead_of_panicking() { let vfs = VfsRef::new(RefusingFs); - match vfs.acquire() { + match vfs.acquire(test_origin()) { Err(VfsError::Backend(message)) => { assert_eq!(message, "the backend refuses acquisition"); } @@ -1274,9 +1345,9 @@ mod tests { let vfs = VfsRef::new(RefuseSecond { vended: Arc::new(Mutex::new(0)), }); - let parent = vfs.acquire()?; + let parent = vfs.acquire(test_origin())?; parent.write("/f.txt", b"1")?; - match parent.spawn() { + match parent.spawn(test_origin()) { Err(VfsError::Backend(message)) => { assert_eq!(message, "the backend refuses acquisition"); } @@ -1284,9 +1355,103 @@ mod tests { } // The failed spawn did not retire the parent's claims: a second // identity still conflicts with the parent's write. - let other = vfs.acquire()?; + let other = vfs.acquire(test_origin())?; let message = conflict_message(other.write("/f.txt", b"2")); assert!(message.contains("/f.txt"), "{message}"); Ok(()) } + + #[test] + fn a_sink_receives_events_in_order_with_op_path_and_label() -> Result<(), VfsError> { + /// One recorded event: op, canonical path, label, and line. + type Recorded = (Op, String, String, u32); + + let events: Arc>> = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&events); + let vfs = VfsRef::builder() + .mount("/", StubFs::seeded(&[("/a.txt", "x")])) + .on_op(move |event: OpEvent<'_>| { + recorded + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(( + event.op(), + event.path().to_string(), + event.origin().label.clone(), + event.origin().line, + )); + }) + .build(); + let access = vfs.acquire(Origin::at("the section", "the prompt", 7))?; + access.read("/a.txt")?; + access.write("/b.txt", b"y")?; + access.glob("/*.txt")?; + // A spawned child's events carry the child's own origin. + let child = access.spawn(Origin::at("the arm", "the prompt", 9))?; + child.append("/b.txt", b"!")?; + let events = events.lock().unwrap_or_else(PoisonError::into_inner); + assert_eq!( + *events, + vec![ + (Op::Read, "/a.txt".to_owned(), "the section".to_owned(), 7), + (Op::Write, "/b.txt".to_owned(), "the section".to_owned(), 7), + (Op::Glob, "/*.txt".to_owned(), "the section".to_owned(), 7), + (Op::Append, "/b.txt".to_owned(), "the arm".to_owned(), 9), + ] + ); + Ok(()) + } + + #[test] + fn a_handle_without_a_sink_serves_operations_without_firing() -> Result<(), VfsError> { + // The None sink branch: no `on_op`, and operations behave + // exactly as before - not firing is a no-op, never a panic. + let vfs = VfsRef::builder().mount("/", StubFs::default()).build(); + let access = vfs.acquire(test_origin())?; + access.write("/f.txt", b"x")?; + assert_eq!(access.read("/f.txt")?, b"x"); + Ok(()) + } + + #[test] + fn a_policy_denied_operation_never_fires_the_sink() -> Result<(), VfsError> { + let verdict = Arc::new(Mutex::new(Verdict::Deny("writes are sealed".to_owned()))); + let events: Arc>> = Arc::new(Mutex::new(Vec::new())); + let recorded = Arc::clone(&events); + let vfs = VfsRef::builder() + .mount("/", StubFs::default()) + .policy(FlipPolicy { + verdict: Arc::clone(&verdict), + }) + .on_op(move |event: OpEvent<'_>| { + recorded + .lock() + .unwrap_or_else(PoisonError::into_inner) + .push(event.op()); + }) + .build(); + let access = vfs.acquire(test_origin())?; + assert!(matches!( + access.write("/f.txt", b"x"), + Err(VfsError::PermissionDenied(_)) + )); + assert!( + events + .lock() + .unwrap_or_else(PoisonError::into_inner) + .is_empty(), + "a denied operation fired the sink" + ); + // The host flips the policy mid-run; the admitted write fires. + *verdict.lock().unwrap_or_else(PoisonError::into_inner) = Verdict::Allow; + access.write("/f.txt", b"x")?; + assert_eq!( + events + .lock() + .unwrap_or_else(PoisonError::into_inner) + .as_slice(), + &[Op::Write] + ); + Ok(()) + } } diff --git a/crates/shared-vfs/src/lib.rs b/crates/shared-vfs/src/lib.rs index 9f17c1791..afeb977b7 100644 --- a/crates/shared-vfs/src/lib.rs +++ b/crates/shared-vfs/src/lib.rs @@ -1,5 +1,6 @@ //! Generic virtual filesystem machinery: canonical interned paths, the -//! claims model, the mount router, and backends. +//! claims model, the mount router, the operation-observation seam, and +//! backends. //! //! This crate is the permanent bottom of the dependency stack: std only, //! no workspace or external crates, and no promptforge policy (no @@ -10,6 +11,7 @@ mod glob; mod handle; mod host; mod memory; +mod observe; mod path; mod router; mod traits; @@ -19,6 +21,7 @@ pub use error::VfsError; pub use handle::{Access, VfsRef}; pub use host::HostBackend; pub use memory::MemoryBackend; +pub use observe::{OpEvent, OpSink, Origin}; pub use path::{VfsPath, VfsPathBuf}; pub use router::VfsRefBuilder; pub use traits::{AllowAll, ExecId, Op, Policy, Verdict, Vfs, VfsAccess}; diff --git a/crates/shared-vfs/src/memory.rs b/crates/shared-vfs/src/memory.rs index 903d49446..7a080a391 100644 --- a/crates/shared-vfs/src/memory.rs +++ b/crates/shared-vfs/src/memory.rs @@ -143,10 +143,10 @@ impl Tree { /// /// # Examples /// ``` -/// use shared_vfs::{MemoryBackend, VfsRef}; +/// use shared_vfs::{MemoryBackend, Origin, VfsRef}; /// /// let vfs = VfsRef::new(MemoryBackend::new()); -/// let access = vfs.acquire()?; +/// let access = vfs.acquire(Origin::new("memory backend example"))?; /// access.write("/notes.md", b"todo")?; /// assert_eq!(access.read("/notes.md")?, b"todo"); /// # Ok::<(), shared_vfs::VfsError>(()) diff --git a/crates/shared-vfs/src/observe.rs b/crates/shared-vfs/src/observe.rs new file mode 100644 index 000000000..3be90c232 --- /dev/null +++ b/crates/shared-vfs/src/observe.rs @@ -0,0 +1,125 @@ +//! The operation-observation seam: [`Origin`], [`OpEvent`], and the op +//! sink. +//! +//! A handle with an installed sink fires it on every admitted operation - +//! after policy and claims pass, before the backend executes - with the +//! op kind, the canonical path, and the caller-supplied [`Origin`]. The +//! seam is fire-and-forget: no outcome flows back, and a policy-denied +//! operation never fires. Claims still key on the internal +//! [`ExecId`](crate::ExecId); the origin is observability, never +//! identity. The deferred consumers - the bounded event log, the Lua +//! pull query, and enrichment policies - subscribe through this seam in +//! later steps. + +use std::panic::Location; +use std::sync::Arc; + +use crate::path::VfsPath; +use crate::traits::Op; + +/// Who asked for an operation: a label and the most precise source +/// position the caller knows. Pure observability - an origin never gates +/// an operation and never appears in a claim. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Origin { + /// The most specific label the caller has: a section name for a + /// chain, a tool id for a tool, a fixture name for a test. + pub label: String, + /// The source file or document `line` refers to: a Rust source file + /// for [`Origin::new`], the prompt's name for [`Origin::at`]. + pub file: String, + /// The 1-based line within `file`. + pub line: u32, +} + +impl Origin { + /// Stamps the Rust call site via [`Location::caller`]: host code and + /// tests get their position for free. Use the most specific label + /// available - a section name for a chain, a tool id for a tool, a + /// fixture name for a test - never a generic label when a specific + /// one exists. + #[must_use] + #[track_caller] + pub fn new(label: impl Into) -> Origin { + let caller = Location::caller(); + Origin { + label: label.into(), + file: caller.file().to_owned(), + line: caller.line(), + } + } + + /// Sets an explicit position: the executor and the agent substitute + /// the prompt's position for the Rust one, so every event's position + /// is the most precise thing the caller knows. The label guidance of + /// [`Origin::new`] applies unchanged. + #[must_use] + pub fn at(label: impl Into, file: impl Into, line: u32) -> Origin { + Origin { + label: label.into(), + file: file.into(), + line, + } + } +} + +/// One admitted operation, handed to the installed sink. Borrows the +/// capability's own values, so firing allocates nothing; a sink that +/// retains events clones out of the views. +#[derive(Debug)] +pub struct OpEvent<'a> { + pub(crate) op: Op, + pub(crate) path: &'a VfsPath, + pub(crate) origin: &'a Origin, +} + +impl<'a> OpEvent<'a> { + /// The operation kind. + #[must_use] + pub fn op(&self) -> Op { + self.op + } + + /// The canonical path the operation acts on. Two-path operations + /// (rename, copy) fire one event per path. + #[must_use] + pub fn path(&self) -> &'a VfsPath { + self.path + } + + /// The origin of the capability that admitted the operation. + #[must_use] + pub fn origin(&self) -> &'a Origin { + self.origin + } +} + +/// The installed operation sink. Must be cheap: store operations fire it +/// from the blocking pool, inline with the operation. +pub type OpSink = Arc) + Send + Sync>; + +#[cfg(test)] +mod tests { + use super::Origin; + + #[test] + fn origin_new_stamps_the_callers_file_and_line() { + let origin = Origin::new("the fixture"); + assert_eq!(origin.line, line!() - 1, "the call site's line"); + assert!( + origin.file.ends_with("observe.rs"), + "the call site's file: {}", + origin.file + ); + assert_eq!(origin.label, "the fixture"); + } + + #[test] + fn origin_at_carries_the_explicit_position() { + let origin = Origin::at("the section", "the prompt", 42); + assert_eq!(origin.label, "the section"); + assert_eq!(origin.file, "the prompt"); + assert_eq!(origin.line, 42); + } +} diff --git a/crates/shared-vfs/src/router.rs b/crates/shared-vfs/src/router.rs index 336830b5d..95c5086e8 100644 --- a/crates/shared-vfs/src/router.rs +++ b/crates/shared-vfs/src/router.rs @@ -15,8 +15,9 @@ use std::sync::{Arc, Mutex, MutexGuard, PoisonError}; use crate::error::VfsError; use crate::handle::VfsRef; +use crate::observe::{OpEvent, OpSink}; use crate::path::{VfsPath, VfsPathBuf, canonicalize}; -use crate::traits::{ExecId, Vfs, VfsAccess}; +use crate::traits::{AllowAll, ExecId, Policy, Vfs, VfsAccess}; use crate::types::{Entry, GrepQuery, GrepResults, Stat}; /// One mounted backend behind a shared lock. @@ -329,6 +330,8 @@ impl Drop for RoutingAccess { /// `Arc`-share thereafter. pub struct VfsRefBuilder { mounts: Mounts, + policy: Option>, + sink: Option, } impl fmt::Debug for VfsRefBuilder { @@ -341,6 +344,8 @@ impl VfsRefBuilder { pub(crate) fn new() -> VfsRefBuilder { VfsRefBuilder { mounts: BTreeMap::new(), + policy: None, + sink: None, } } /// Mounts `backend` at `prefix`, consuming and returning the @@ -364,13 +369,37 @@ impl VfsRefBuilder { self } - /// Freezes the mount table into a handle with the [`AllowAll`] - /// policy. + /// Installs the policy consulted on every operation, consuming and + /// returning the builder. The default is [`AllowAll`]. + /// + /// [`AllowAll`]: crate::AllowAll + #[must_use] + pub fn policy(mut self, policy: impl Policy + Sync + 'static) -> VfsRefBuilder { + self.policy = Some(Arc::new(policy)); + self + } + + /// Installs the operation sink, consuming and returning the builder. + /// The sink fires on every admitted operation - after policy and + /// claims pass, before the backend executes - with the op kind, the + /// canonical path, and the caller's origin. Fire-and-forget: no + /// outcome flows back, and a policy-denied operation never fires. + /// The sink must be cheap: store operations fire it from the + /// blocking pool. + #[must_use] + pub fn on_op(mut self, sink: impl Fn(OpEvent<'_>) + Send + Sync + 'static) -> VfsRefBuilder { + self.sink = Some(Arc::new(sink)); + self + } + + /// Freezes the mount table into a handle with the installed policy + /// and op sink (the [`AllowAll`] policy and no sink by default). /// /// [`AllowAll`]: crate::AllowAll #[must_use] pub fn build(self) -> VfsRef { - VfsRef::from_router(Router::new(self.mounts)) + let policy = self.policy.unwrap_or_else(|| Arc::new(AllowAll)); + VfsRef::from_router(Router::new(self.mounts), policy, self.sink) } } @@ -381,6 +410,7 @@ mod tests { use crate::error::VfsError; use crate::handle::VfsRef; + use crate::observe::Origin; use crate::path::VfsPath; use crate::traits::{ExecId, Vfs, VfsAccess}; use crate::types::{Entry, Stat}; @@ -555,6 +585,12 @@ mod tests { } } + /// The routing tests never observe origins, so they acquire under + /// one blanket label. + fn test_origin() -> Origin { + Origin::new("router test") + } + #[test] fn the_longest_prefix_mount_wins_and_acquires_lazily() -> Result<(), VfsError> { let outer = StubFs::default(); @@ -565,7 +601,7 @@ mod tests { .mount("/a/b", inner.clone()) .mount("/elsewhere", untouched.clone()) .build(); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; access.write("/a/b/f.txt", b"inner")?; access.write("/a/f.txt", b"outer")?; // Each backend keyed the file by its mount-relative path. @@ -588,7 +624,7 @@ mod tests { .mount("/", base.clone()) .mount("/mnt", shadow.clone()) .build(); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; // The shadow mount owns everything under /mnt. assert_eq!(access.read("/mnt/f.txt")?, b"shadow-mnt"); // The base still owns the rest of the namespace. @@ -612,14 +648,14 @@ mod tests { .mount("/base", base.clone()) .mount("/local", local.clone()) .build(); - let writer = child.acquire()?; + let writer = child.acquire(test_origin())?; writer.write("/base/f.txt", b"nested")?; // The child router stripped its mount prefix: the base backend // keyed the file at its own root. assert!(base_storage.files().contains_key("/f.txt")); // A second child identity conflicts on the same path: the claim // registered through the mounted handle is visible. - let reader = child.acquire()?; + let reader = child.acquire(test_origin())?; match reader.read("/base/f.txt") { Err(VfsError::Conflict(_)) => {} other => panic!("expected a conflict, got {other:?}"), @@ -639,7 +675,7 @@ mod tests { .mount("/", rw.clone()) .mount("/ro", ro.clone()) .build(); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; // Reads are not gated. assert_eq!(access.read("/ro/a.txt")?, b"keep"); // A write is denied with a clear read-only error. @@ -673,7 +709,7 @@ mod tests { #[test] fn traversal_that_escapes_the_namespace_root_is_rejected() -> Result<(), VfsError> { let vfs = VfsRef::builder().mount("/mnt", StubFs::default()).build(); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; assert!(matches!( access.read("/mnt/../../etc/passwd"), Err(VfsError::InvalidPath(_)) @@ -690,7 +726,7 @@ mod tests { // No root mount: the only storage lives at /mnt. let storage = StubFs::seeded(&[("/f.txt", "inside")]); let vfs = VfsRef::builder().mount("/mnt", storage.clone()).build(); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; // Dot segments within the mount resolve within the mount: the // backend sees the clean mount-relative path. assert_eq!(access.read("/mnt/sub/../f.txt")?, b"inside"); @@ -717,7 +753,7 @@ mod tests { .mount("/a", StubFs::default()) .mount("/a/b", inner.clone()) .build(); - let access = vfs.acquire()?; + let access = vfs.acquire(test_origin())?; let matches = access.glob("/a/b/*.txt")?; assert_eq!(matches, vec!["/a/b/x.txt".to_owned()]); Ok(()) @@ -734,7 +770,7 @@ mod tests { .build(); let overlay = base.overlay("/overlay", extra.clone()); - let writer = overlay.acquire()?; + let writer = overlay.acquire(test_origin())?; writer.write("/store/doc.md", b"store")?; writer.write("/scratch/tmp.txt", b"scratch")?; writer.write("/overlay/x.txt", b"overlay")?; @@ -742,7 +778,7 @@ mod tests { drop(writer); // The base handle serves its own mounts from the same storage. - let reader = base.acquire()?; + let reader = base.acquire(test_origin())?; assert_eq!(reader.read("/store/doc.md")?, b"store"); assert_eq!(reader.read("/scratch/tmp.txt")?, b"scratch"); // The overlay mount exists only in the overlay's view. @@ -757,11 +793,11 @@ mod tests { fn an_overlay_shares_the_bases_claims_table() -> Result<(), VfsError> { let base = VfsRef::builder().mount("/store", StubFs::default()).build(); let overlay = base.overlay("/overlay", StubFs::default()); - let first = base.acquire()?; + let first = base.acquire(test_origin())?; first.write("/store/shared.txt", b"1")?; // A write claim registered through the base conflicts with a // write attempted through the overlay: one claims table. - let second = overlay.acquire()?; + let second = overlay.acquire(test_origin())?; match second.write("/store/shared.txt", b"2") { Err(VfsError::Conflict(message)) => { assert!(message.contains("/store/shared.txt"), "{message}"); diff --git a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md index 8cda511e8..35d9d4457 100644 --- a/vibe/2026-09-12-2-dependency-rules-vfs-hook.md +++ b/vibe/2026-09-12-2-dependency-rules-vfs-hook.md @@ -154,7 +154,7 @@ Both fixes are tiny, share the PR #35 CI provenance (the GitHub-hosted `ubuntu-l -### Step 4: Add the VFS operation-observation hook +### Step 4: Add the VFS operation-observation hook [completed] - Component: vfs-observation-hook From e75a5f121337de0b48442689b54290f33eed164f Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 08:31:52 -0700 Subject: [PATCH 25/26] Close plan: dependency-rules-vfs-hook Plan: vibe/2026-09-12-2-dependency-rules-vfs-hook.md --- vibe/ACTIVE | 1 - 1 file changed, 1 deletion(-) delete mode 100644 vibe/ACTIVE diff --git a/vibe/ACTIVE b/vibe/ACTIVE deleted file mode 100644 index 3203e3f20..000000000 --- a/vibe/ACTIVE +++ /dev/null @@ -1 +0,0 @@ -vibe/2026-09-12-2-dependency-rules-vfs-hook.md \ No newline at end of file From 5b79db3d8bdee83e67755468302416657b56d42a Mon Sep 17 00:00:00 2001 From: Vinnie Falco Date: Sat, 12 Sep 2026 08:59:37 -0700 Subject: [PATCH 26/26] Drain in-flight I/O tasks before delivering the run result A run that ended while a leaf op was still in flight could deliver its result before the op's access clone was gone, so a fresh access acquired after the run could meet a lingering claim. `drive` now funnels every terminal outcome through `drain_io_tasks`, which aborts each recorded task and then awaits it; aborting a blocking-pool op detaches rather than interrupts, so only the join bounds the release of the claims the op's access clone holds. - `io_tasks` stores `JoinHandle<()>` instead of `AbortHandle` so a terminal outcome can await completion, not only abort. - `abort_subtree` keeps the aborted handle in `io_tasks` instead of removing it, so the run-end drain also covers a fanout-aborted sibling op; a late answer that arrives first still takes the handle in the answer loop. - `a_terminal_failure_releases_an_in_flight_arms_claims_before_returning` gates a winning arm's append behind `AppendGate` in a `GatedStore` backend and fails without the drain with the same WriteRace signature CI showed on Linux. --- .../promptforge-core/src/execute/scheduler.rs | 53 ++++- crates/promptforge-core/tests/suite/fanout.rs | 196 ++++++++++++++++++ 2 files changed, 239 insertions(+), 10 deletions(-) diff --git a/crates/promptforge-core/src/execute/scheduler.rs b/crates/promptforge-core/src/execute/scheduler.rs index b00b91b77..b43c0f8b3 100644 --- a/crates/promptforge-core/src/execute/scheduler.rs +++ b/crates/promptforge-core/src/execute/scheduler.rs @@ -62,7 +62,7 @@ use std::sync::atomic::{AtomicU32, Ordering}; use mlua::{RegistryKey, Thread}; use shared_vfs::Origin; use tokio::sync::mpsc; -use tokio::task::AbortHandle; +use tokio::task::JoinHandle; use crate::client::GatewayClient; use crate::fanout; @@ -510,11 +510,14 @@ pub(crate) struct Scheduler<'a> { answer_tx: mpsc::UnboundedSender<(RequestId, Answer)>, /// The receive half the driver awaits when no chain is ready. answers: mpsc::UnboundedReceiver<(RequestId, Answer)>, - /// Abort handles of the in-flight leaf I/O tasks, keyed by request so + /// Join handles of the in-flight leaf I/O tasks, keyed by request so /// a fatal fanout arm can abort a sibling arm's own in-flight round; /// every handle is aborted on cancellation, and aborting a completed - /// task is a no-op. - io_tasks: HashMap, + /// task is a no-op. The handles are kept joinable (not bare abort + /// handles) so a terminal run outcome can drain them: a store op + /// runs on the blocking pool, where abort detaches rather than + /// interrupts, and only the op's completion drops its access clone. + io_tasks: HashMap>, /// The request ids whose in-flight tasks an abort discarded: a task /// that posted its answer before the abort landed delivers it late, /// and the driver discards exactly those answers. An unknown id that @@ -619,7 +622,31 @@ impl<'a> Scheduler<'a> { /// Returns [`Error::Interrupted`] when the run's cancellation handle is /// signaled while chains are running or suspended. pub(crate) async fn drive(&mut self) -> Result { - self.drive_inner().await + let result = self.drive_inner().await; + self.drain_io_tasks().await; + result + } + + /// Claims-release ordering constraint: the run's result - success, + /// determinism failure, or cancellation alike - must not be delivered + /// while an in-flight leaf op still holds its access clone. A store + /// op runs on the blocking pool, where aborting the task detaches + /// rather than interrupts, so an abandoned op would release its + /// identity's claims only when the closure finishes - past the run's + /// end, where a fresh access could meet the lingering claim. Abort + /// every task still recorded (prompt for an async task, a no-op for + /// a blocking op already running, which runs to completion), then + /// await each handle: the join resolves only once the op's access + /// clone - and with it the identity's claims - is gone. This changes + /// when claims release, never what an operation does. + async fn drain_io_tasks(&mut self) { + let tasks = std::mem::take(&mut self.io_tasks); + for task in tasks.values() { + task.abort(); + } + for (_, task) in tasks { + let _ = task.await; + } } async fn drive_inner(&mut self) -> Result { @@ -1552,7 +1579,7 @@ impl<'a> Scheduler<'a> { fn dispatch_infer(&mut self, id: ChainId, prompt: String, binding: Option) { match self.prepare_infer(id, prompt, binding) { Ok((request_id, task)) => { - self.io_tasks.insert(request_id, task.abort_handle()); + self.io_tasks.insert(request_id, task); self.pending.insert(request_id, id); } Err(error) => { @@ -1629,7 +1656,7 @@ impl<'a> Scheduler<'a> { fn dispatch_tool_call(&mut self, id: ChainId, alias: &str, args: serde_json::Value) { match self.prepare_tool_call(id, alias, args) { Ok((request_id, task)) => { - self.io_tasks.insert(request_id, task.abort_handle()); + self.io_tasks.insert(request_id, task); self.pending.insert(request_id, id); } Err(error) => { @@ -1782,7 +1809,7 @@ impl<'a> Scheduler<'a> { // the answer is then moot. let _ = tx.send((request_id, answer)); }); - self.io_tasks.insert(request_id, task.abort_handle()); + self.io_tasks.insert(request_id, task); self.pending.insert(request_id, id); } @@ -1835,7 +1862,7 @@ impl<'a> Scheduler<'a> { Answer::Store(result.map_err(|e| classify_store_failure(&e))), )); }); - self.io_tasks.insert(request_id, task.abort_handle()); + self.io_tasks.insert(request_id, task); self.pending.insert(request_id, id); Ok(()) } @@ -2529,7 +2556,13 @@ impl<'a> Scheduler<'a> { // the driver discards; anything else stays a loud invariant // failure. self.aborted_requests.insert(request); - if let Some(task) = self.io_tasks.remove(&request) { + // The handle stays in `io_tasks`: aborting a blocking-pool op + // detaches rather than interrupts, so the op's access clone - + // and the claims it holds - releases only when the op finishes. + // The run-end drain awaits the handle, keeping claim release + // bounded to the run's lifetime on this path too; if the op's + // late answer arrives first, the answer loop takes the handle. + if let Some(task) = self.io_tasks.get(&request) { task.abort(); } } diff --git a/crates/promptforge-core/tests/suite/fanout.rs b/crates/promptforge-core/tests/suite/fanout.rs index 53442d8e9..eb84a3088 100644 --- a/crates/promptforge-core/tests/suite/fanout.rs +++ b/crates/promptforge-core/tests/suite/fanout.rs @@ -1,9 +1,12 @@ //! Concurrent fanout: per-arm start/terminal accounting, store writes across //! arms, and the propagated arm-failure error contract. +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Condvar, Mutex}; use std::time::Duration; use promptforge_core::execute::RunErrorKind; +use shared_vfs::{Entry, ExecId, MemoryBackend, Stat, Vfs, VfsAccess, VfsError, VfsPath, VfsRef}; use super::support::{Record, run_fixture}; @@ -191,6 +194,199 @@ async fn a_cross_arm_append_terminates_the_run_with_a_determinism_violation() { ); } +/// A one-shot gate for the winning arm's backend append: the first +/// `append` the backend serves signals `started` and parks until `open` +/// releases it, so the run's terminal failure lands while the winning +/// op - its access clone alive, its write claim held - is still in +/// flight on the blocking pool. +#[derive(Default)] +struct AppendGate { + started: tokio::sync::Notify, + released: Mutex, + release: Condvar, + taken: AtomicBool, +} + +impl AppendGate { + /// Parks the first caller until the gate opens; later callers pass. + fn block_first(&self) { + if self.taken.swap(true, Ordering::SeqCst) { + return; + } + self.started.notify_one(); + let mut released = self + .released + .lock() + .expect("the gate mutex is not poisoned"); + while !*released { + released = self + .release + .wait(released) + .expect("the gate mutex is not poisoned"); + } + } + + /// Waits for the first parked append. + async fn await_started(&self) { + self.started.notified().await; + } + + /// Releases the parked append. + fn open(&self) { + let mut released = self + .released + .lock() + .expect("the gate mutex is not poisoned"); + *released = true; + self.release.notify_all(); + } +} + +/// Opens the gate on drop, so a panicking assertion never strands the +/// parked blocking-pool op (the runtime waits for it at shutdown). +struct GateGuard(Arc); + +impl Drop for GateGuard { + fn drop(&mut self) { + self.0.open(); + } +} + +/// A memory backend whose first `append` parks on the gate, standing in +/// for a slow host backend: the winning arm's op stays in flight across +/// the run's terminal failure. +struct GatedStore { + inner: MemoryBackend, + gate: Arc, +} + +impl Vfs for GatedStore { + fn acquire(&mut self, id: ExecId) -> Result, VfsError> { + Ok(Box::new(GatedAccess { + inner: self.inner.acquire(id)?, + gate: Arc::clone(&self.gate), + })) + } + + fn release(&mut self, id: ExecId) -> Result<(), VfsError> { + self.inner.release(id) + } +} + +struct GatedAccess { + inner: Box, + gate: Arc, +} + +impl VfsAccess for GatedAccess { + fn read(&self, path: &VfsPath) -> Result, VfsError> { + self.inner.read(path) + } + + fn write(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.inner.write(path, contents) + } + + fn append(&mut self, path: &VfsPath, contents: &[u8]) -> Result<(), VfsError> { + self.gate.block_first(); + self.inner.append(path, contents) + } + + fn remove(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.inner.remove(path, recursive) + } + + fn exists(&self, path: &VfsPath) -> Result { + self.inner.exists(path) + } + + fn glob(&self, pattern: &str) -> Result, VfsError> { + self.inner.glob(pattern) + } + + fn list(&self, path: &VfsPath) -> Result, VfsError> { + self.inner.list(path) + } + + fn stat(&self, path: &VfsPath) -> Result { + self.inner.stat(path) + } + + fn mkdir(&mut self, path: &VfsPath, recursive: bool) -> Result<(), VfsError> { + self.inner.mkdir(path, recursive) + } + + fn rename(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.inner.rename(from, to) + } + + fn copy(&mut self, from: &VfsPath, to: &VfsPath) -> Result<(), VfsError> { + self.inner.copy(from, to) + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_terminal_failure_releases_an_in_flight_arms_claims_before_returning() { + // The winning arm's append parks inside the backend with its write + // claim held; the sibling's append booms against that claim and the + // run fails fast at the answer boundary. The run's result must not + // be delivered while the parked op's access clone still holds the + // claim: the driver drains (awaits) the in-flight op before + // returning, so the post-run fresh-access read never meets a + // lingering claim. + let gate = Arc::new(AppendGate::default()); + let _guard = GateGuard(Arc::clone(&gate)); + let vfs = VfsRef::builder() + .mount( + promptforge_vfs::STORE_MOUNT, + GatedStore { + inner: MemoryBackend::new(), + gate: Arc::clone(&gate), + }, + ) + .build(); + let mut run_task = tokio::spawn(run_fixture( + FANOUT_CROSS_ARM_APPEND, + "execution/fanout-cross-arm-append.md", + FANOUT_CROSS_ARM_EXECUTION, + "", + Some(vfs), + )); + // The winning arm's op is now parked inside the backend, its claim + // held; the sibling's boom needs no test interaction. + gate.await_started().await; + // Proving the negative - the run did NOT return while the claim is + // held - takes a bounded wait. The fail-fast path is pure in-runtime + // scheduling (no I/O, no timers), so the grace is generous: without + // the drain the run returns within milliseconds. + if let Ok(done) = tokio::time::timeout(Duration::from_secs(5), &mut run_task).await { + let run = done.expect("the run task must not panic"); + let lingering = run.store.read("evidence.md").err(); + panic!( + "the run returned while an in-flight arm op still held its write claim; \ + a fresh post-run access meets the lingering claim: {lingering:?}" + ); + } + gate.open(); + let run = run_task.await.expect("the run task must not panic"); + let error = match run.result { + Ok(value) => panic!("a cross-arm append must terminate the run, got {value:?}"), + Err(error) => error, + }; + assert_eq!( + error.kind(), + RunErrorKind::Determinism, + "a claims conflict classifies as a determinism violation: {error:?}" + ); + // The invariant the drain restores: a fresh post-run access never + // meets a lingering claim, and the winning arm's append landed. + let evidence = run.store.read("evidence.md").expect("one arm appended"); + assert!( + evidence == "alpha\n" || evidence == "beta\n", + "exactly one arm's append may land: {evidence:?}" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn fanout_arm_failure_propagates() { let run = run_fixture(