From 049041225e9522ff990fb4a751726a1f71000194 Mon Sep 17 00:00:00 2001 From: Cole Leavitt Date: Wed, 19 Aug 2026 03:51:42 -0400 Subject: [PATCH 1/7] feat(memory): preserve corroborating episode evidence --- STRUCTURE.md | 4 +- crates/mc-module/src/lib.rs | 65 +- crates/mc-module/src/m0_compose.rs | 1 + crates/mc-module/src/m1_compose.rs | 1 + crates/mc-module/src/memory_tool.rs | 1 + crates/mc-module/src/transform.rs | 1 + crates/mc-store/src/lib.rs | 1477 +++++++++++++---- .../pi-plugin/src/tools/ctx-memory.test.ts | 86 + packages/pi-plugin/src/tools/ctx-memory.ts | 38 +- packages/plugin/docs/MEMORY-DESIGN.md | 9 + .../magic-context/context-authority.test.ts | 50 + .../magic-context/context-authority.ts | 35 + .../dreamer/retrospective-learnings.ts | 16 +- .../magic-context/memory/promotion.ts | 22 +- .../magic-context/memory/relocate-memory.ts | 72 +- .../memory/storage-memory-evidence.test.ts | 240 +++ .../magic-context/memory/storage-memory.ts | 162 +- .../features/magic-context/memory/types.ts | 4 + .../magic-context/migrations-v74.test.ts | 2 +- .../magic-context/migrations-v76.test.ts | 2 +- .../magic-context/migrations-v77.test.ts | 2 +- .../magic-context/migrations-v78.test.ts | 2 +- .../magic-context/migrations-v79.test.ts | 49 + .../src/features/magic-context/migrations.ts | 26 + .../src/features/magic-context/storage-db.ts | 14 +- .../storage-identity-merge.test.ts | 18 + .../magic-context/storage-identity-merge.ts | 18 +- .../v22-deferred-backfill.test.ts | 15 + .../magic-context/v22-deferred-backfill.ts | 18 +- .../hooks/magic-context/module-state-sync.ts | 2 + .../magic-context/rust-mode-transform.ts | 24 +- .../plugin/src/tools/ctx-memory/tools.test.ts | 27 +- packages/plugin/src/tools/ctx-memory/tools.ts | 36 +- 33 files changed, 2095 insertions(+), 444 deletions(-) create mode 100644 packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts create mode 100644 packages/plugin/src/features/magic-context/migrations-v79.test.ts diff --git a/STRUCTURE.md b/STRUCTURE.md index b66703ef5..434d9280f 100644 --- a/STRUCTURE.md +++ b/STRUCTURE.md @@ -173,7 +173,7 @@ Unless specified otherwise, TypeScript paths are relative to `packages/plugin/` - `src/hooks/magic-context/compaction-off-transition.ts`: Reconcile per-session compaction mode records and process off/on mode transitions. - `src/hooks/magic-context/child-session-spawn.ts`: Enforce child session spawn choke point with schema fence validation. - `src/shared/escalation-bands.ts`: Derive context limit escalation bands and threshold bounds. -- `src/features/magic-context/migrations.ts`: Versioned schema migrations v1–v78 (`LATEST_SUPPORTED_VERSION` in `storage-db.ts` must track the highest; `schema-version-fence.test.ts` asserts they stay in lockstep). +- `src/features/magic-context/migrations.ts`: Versioned schema migrations v1–v79 (`LATEST_SUPPORTED_VERSION` in `storage-db.ts` must track the highest; `schema-version-fence.test.ts` asserts they stay in lockstep). v79 adds content-bound memory episode evidence. - `src/features/magic-context/message-index.ts`: FTS-backed raw-message index for `ctx_search`. - `src/features/magic-context/search.ts`: Unified retrieval over memories, raw messages, git commits, and session/smart notes. - `src/features/magic-context/session-project-storage.ts`: Persist session-to-project bindings and repair mis-scoped compartment chunk embeddings. @@ -200,7 +200,7 @@ Unless specified otherwise, TypeScript paths are relative to `packages/plugin/` - `crates/mc-module/src/session_resolver.rs`: Resolves incoming MCP facade requests to their backing project and session. - `crates/mc-module/src/lib.rs`: Route subc client requests, implement MCP tool facade routing (supporting `agent_drops.append` queue drops with server-side range parsing and command-id idempotency checks), serve prompt guidance, manage durable pass tracing for transform passes, orchestrate `session.status`, `session.wrapup`, and `session.delete` operations (utilizing structured status fields, machine-readable dispositions, and process-local per-session latches under a `MAX_WRAPUP_REQUEST_BUDGET` deadline, with `session.delete` atomically removing session-owned rows from SQLite tables), track transform dispatch health metrics and heartbeat reporting, manage LRU-bounded `InFlight` snapshot caching, and coordinate bootstrap state imports using `StateImportCoordinator`. - `crates/mc-module/src/historian_producer.rs`: Implement the Rust subc historian producer client using the wire v2 protocol with `OpenedRoute` targeting (channel and epoch routing). -- `crates/mc-store/src/lib.rs`: Define durable session schemas and migrations (including the `mc_reduce_command_ledger` table in migration 16 for idempotency, `mc_project_mural_artifacts` in migration 49 for project mural artifacts, and `raw_messages_deflate` in migration 50 on `mc_chunk_transcripts` for durable `ctx_expand` recovery), handle metadata, and run CAS transitions. +- `crates/mc-store/src/lib.rs`: Define durable session schemas and migrations (including the `mc_reduce_command_ledger` table in migration 16 for idempotency, `mc_project_mural_artifacts` in migration 49 for project mural artifacts, `raw_messages_deflate` in migration 50 on `mc_chunk_transcripts` for durable `ctx_expand` recovery, and content-bound memory evidence in migration 51), handle metadata, and run CAS transitions. - `crates/mc-module/src/codec/`: Decode harness-specific JSON messages (OpenCode, Pi) into canonical `CkIngressMessage` values and encode them back using harness model codecs. - `crates/mc-module/src/caveman.rs`: Age-tier caveman text compression ported to Rust. - `crates/mc-module/src/divergence.rs`: Per-pass transform output divergence tracking and attribution. diff --git a/crates/mc-module/src/lib.rs b/crates/mc-module/src/lib.rs index 9ac576705..ba85575bb 100644 --- a/crates/mc-module/src/lib.rs +++ b/crates/mc-module/src/lib.rs @@ -66,7 +66,8 @@ use mc_store::TagNumberRow; use mc_store::{ canonical_root, validate_state_import_compartments, AuthoritySeedRow, DeferredExecuteState, FacadeMutationOutcome, HistorianPhase, InsertMemoryInput, MappingUpdate, McStore, McStoreError, - ModuleDropSeedRow, ModuleMemoryMutationRow, ModuleMemoryRow, ModuleStateSyncError, + ModuleDropSeedRow, ModuleMemoryEvidenceRow, ModuleMemoryMutationRow, ModuleMemoryRow, + ModuleStateSyncError, ModuleStateSyncRequest, ModuleStripSeedRow, ModuleWorkspaceMemberRow, ModuleWorkspaceRow, NoteCasOutcome, NoteEvaluationInput, NoteInput, NoteNudgeAnchorSeed, NoteWriteInput, PendingAgentDrop, PendingAgentDropSeedRow, PendingCompactionMarkerState, @@ -1697,6 +1698,18 @@ struct ModuleMemoryWire { mural_cue_at: Option, #[serde(default)] mural_cue_rejection_count: i64, + #[serde(default)] + evidence: Vec, +} + +#[derive(Debug, Clone, Deserialize)] +struct ModuleMemoryEvidenceWire { + content_hash: String, + source_session_id: String, + #[serde(default)] + source_message_id: Option, + source_type: String, + observed_at: i64, } #[derive(Debug, Clone, Deserialize)] @@ -1804,6 +1817,17 @@ impl ModuleMemoryWire { mural_cue_hash: self.mural_cue_hash, mural_cue_at: self.mural_cue_at, mural_cue_rejection_count: self.mural_cue_rejection_count, + evidence: self + .evidence + .into_iter() + .map(|row| ModuleMemoryEvidenceRow { + content_hash: row.content_hash, + source_session_id: row.source_session_id, + source_message_id: row.source_message_id, + source_type: row.source_type, + observed_at: row.observed_at, + }) + .collect(), } } } @@ -10423,6 +10447,7 @@ impl McHandler { category, content, source_session_id: Some(conversation_key), + source_message_id: None, source_type: Some("agent"), importance: Some(50), expires_at: None, @@ -10464,6 +10489,24 @@ impl McHandler { .update_memory_content(memory_project, id, content, now_ms()) .map_err(|error| error.to_string())? .ok_or_else(|| format!("memory {id} was not found"))?; + tx.record_memory_evidence( + memory.id, + InsertMemoryInput { + project_path: memory_project, + route_project_root: Some( + facade_scope.route_project_root.as_str(), + ), + category: &memory.category, + content, + source_session_id: Some(conversation_key), + source_message_id: None, + source_type: Some("agent"), + importance: memory.importance, + expires_at: memory.expires_at, + metadata_json: memory.metadata_json.as_deref(), + now_ms: now_ms(), + }, + )?; facade_text_response( format!( "Updated memory [ID: {}] in {}.", @@ -10545,6 +10588,24 @@ impl McHandler { ) .map_err(|error| error.to_string())? .ok_or_else(|| format!("memory {target_id} was not found"))?; + tx.record_memory_evidence( + memory.id, + InsertMemoryInput { + project_path: memory_project, + route_project_root: Some( + facade_scope.route_project_root.as_str(), + ), + category: &memory.category, + content, + source_session_id: Some(conversation_key), + source_message_id: None, + source_type: Some("agent"), + importance: memory.importance, + expires_at: memory.expires_at, + metadata_json: memory.metadata_json.as_deref(), + now_ms: now_ms(), + }, + )?; facade_text_response( format!( "Merged memories into [ID: {}] in {}; superseded [{}].", @@ -16700,6 +16761,7 @@ mod tests { category, content, source_session_id: Some(project), + source_message_id: None, source_type: Some("test"), importance: Some(50), expires_at: None, @@ -22330,6 +22392,7 @@ mod tests { .any(|memory| memory.content == "second shared fact")); let first = store.get_memory_full(project_rows[0].id).unwrap().unwrap(); assert_eq!(first.source_session_id.as_deref(), Some(key_a)); + assert_eq!(first.source_type.as_deref(), Some("agent")); assert!(store .load_active_memories(key_a, now_ms()) .unwrap() diff --git a/crates/mc-module/src/m0_compose.rs b/crates/mc-module/src/m0_compose.rs index 5cd4e9fbb..a5f33fcc7 100644 --- a/crates/mc-module/src/m0_compose.rs +++ b/crates/mc-module/src/m0_compose.rs @@ -652,6 +652,7 @@ mod tests { category: "CONSTRAINTS", content: "must stay hidden", source_session_id: None, + source_message_id: None, source_type: Some("agent"), importance: Some(50), expires_at: None, diff --git a/crates/mc-module/src/m1_compose.rs b/crates/mc-module/src/m1_compose.rs index e8cc8699e..1b5d53c54 100644 --- a/crates/mc-module/src/m1_compose.rs +++ b/crates/mc-module/src/m1_compose.rs @@ -569,6 +569,7 @@ mod tests { category, content, source_session_id: None, + source_message_id: None, source_type: Some("tool"), importance: Some(70), expires_at: None, diff --git a/crates/mc-module/src/memory_tool.rs b/crates/mc-module/src/memory_tool.rs index 5a4480ecb..e9d1cca38 100644 --- a/crates/mc-module/src/memory_tool.rs +++ b/crates/mc-module/src/memory_tool.rs @@ -549,6 +549,7 @@ mod tests { category, content, source_session_id: None, + source_message_id: None, source_type: Some("tool"), importance: Some(50), expires_at: None, diff --git a/crates/mc-module/src/transform.rs b/crates/mc-module/src/transform.rs index 7bfbf6cb1..6e2429bb6 100644 --- a/crates/mc-module/src/transform.rs +++ b/crates/mc-module/src/transform.rs @@ -13984,6 +13984,7 @@ pub(crate) mod tests { category, content, source_session_id: None, + source_message_id: None, source_type: Some("tool"), importance: Some(70), expires_at: None, diff --git a/crates/mc-store/src/lib.rs b/crates/mc-store/src/lib.rs index 3e12b1492..5b9a5a51a 100644 --- a/crates/mc-store/src/lib.rs +++ b/crates/mc-store/src/lib.rs @@ -13,22 +13,25 @@ #![forbid(unsafe_code)] +use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::io::{Cursor, Error, ErrorKind, Read, Write}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Instant; + use cortexkit_cache_core::{CoreState, DurabilityClass, FrozenUnit}; -use cortexkit_store::{open_sqlite, Migration, SqliteStore, StoreError}; +use cortexkit_store::{Migration, SqliteStore, StoreError, open_sqlite}; use cortexkit_store_types::StorageDescriptor; -use flate2::{read::DeflateDecoder, write::DeflateEncoder, Compression}; -use rusqlite::{functions::FunctionFlags, params, types::Value as SqlValue, OptionalExtension}; +use flate2::Compression; +use flate2::read::DeflateDecoder; +use flate2::write::DeflateEncoder; +use rusqlite::functions::FunctionFlags; +use rusqlite::types::Value as SqlValue; +use rusqlite::{OptionalExtension, params}; use serde::{Deserialize, Deserializer, Serialize, Serializer}; use serde_json::Value; use sha2::{Digest, Sha256}; -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; -use std::io::{Cursor, Error, ErrorKind, Read, Write}; -use std::path::{Path, PathBuf}; -use std::sync::{ - atomic::{AtomicU64, Ordering}, - Arc, Mutex, -}; -use std::time::Instant; pub type ProviderExtras = BTreeMap>; @@ -2401,6 +2404,28 @@ const MIGRATIONS: &[Migration] = &[ ALTER TABLE mc_chunk_transcripts ADD COLUMN raw_messages_deflate BLOB NULL; ", }, + Migration { + version: 51, + statements: " + CREATE TABLE IF NOT EXISTS mc_memory_evidence ( + memory_id INTEGER NOT NULL REFERENCES mc_memories(id) ON DELETE CASCADE, + content_hash TEXT NOT NULL, + source_session_id TEXT NOT NULL, + source_message_id TEXT, + source_type TEXT NOT NULL, + observed_at INTEGER NOT NULL, + PRIMARY KEY (memory_id, content_hash, source_session_id) + ); + CREATE INDEX IF NOT EXISTS idx_mc_memory_evidence_session + ON mc_memory_evidence(source_session_id, memory_id); + INSERT OR IGNORE INTO mc_memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT id, normalized_hash, source_session_id, NULL, COALESCE(source_type, 'historian'), first_seen_at + FROM mc_memories + WHERE source_session_id IS NOT NULL; + ", + }, ]; /// The highest `mc_cache` schema migration this binary ships. @@ -2432,6 +2457,34 @@ fn normalize_authority_route_tx( project: &str, route_project_root: &str, ) -> rusqlite::Result<()> { + let collisions = { + let mut statement = tx.prepare( + "SELECT source.id, canonical.id + FROM mc_memories source + JOIN mc_memories canonical + ON canonical.project_path = ?2 + AND canonical.category = source.category + AND canonical.normalized_hash = source.normalized_hash + WHERE source.project_path = ?3 + AND EXISTS ( + SELECT 1 FROM mc_authority + WHERE context_store_uuid = ?1 + AND project = ?2 + AND domain = 'memories' + AND state = 'MODULE' + )", + )?; + let rows = statement + .query_map( + params![context_store_uuid, project, route_project_root], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), + )? + .collect::, _>>()?; + rows + }; + for (source_id, canonical_id) in collisions { + merge_memory_evidence_tx(tx, canonical_id, &[source_id])?; + } tx.execute( "DELETE FROM mc_memories WHERE project_path = ?3 @@ -2450,6 +2503,15 @@ fn normalize_authority_route_tx( )", params![context_store_uuid, project, route_project_root], )?; + tx.execute( + "UPDATE mc_memories + SET seen_count = MAX( + seen_count, + (SELECT COUNT(DISTINCT source_session_id) FROM mc_memory_evidence WHERE memory_id = mc_memories.id) + ) + WHERE project_path = ?1", + [project], + )?; tx.execute( "UPDATE mc_memories SET project_path = ?2 @@ -3032,7 +3094,10 @@ impl std::fmt::Display for HistorianPublishError { "publish CAS conflict: expected {expected:?}, found {found}: {reason}" ) } else { - write!(f, "publish CAS conflict: expected {expected:?}, found {found}") + write!( + f, + "publish CAS conflict: expected {expected:?}, found {found}" + ) } } HistorianPublishError::StateMismatch { expected, found } => write!( @@ -4108,6 +4173,9 @@ pub struct InsertMemoryInput<'a> { pub category: &'a str, pub content: &'a str, pub source_session_id: Option<&'a str>, + /// Host-native owner message id, not a tool command id. Leave absent when the + /// runtime cannot prove that association. + pub source_message_id: Option<&'a str>, pub source_type: Option<&'a str>, pub importance: Option, pub expires_at: Option, @@ -4501,6 +4569,16 @@ pub struct ModuleMemoryRow { pub mural_cue_hash: Option, pub mural_cue_at: Option, pub mural_cue_rejection_count: i64, + pub evidence: Vec, +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct ModuleMemoryEvidenceRow { + pub content_hash: String, + pub source_session_id: String, + pub source_message_id: Option, + pub source_type: String, + pub observed_at: i64, } #[derive(Debug, Clone, Default, PartialEq, Eq)] @@ -5319,6 +5397,77 @@ pub enum FacadeMutationOutcome { Duplicate(Vec), } +fn record_memory_evidence_tx( + tx: &rusqlite::Transaction<'_>, + memory_id: i64, + input: InsertMemoryInput<'_>, +) -> rusqlite::Result { + let Some(source_session_id) = input.source_session_id else { + return Ok(false); + }; + let content_hash: String = tx.query_row( + "SELECT normalized_hash FROM mc_memories WHERE id = ?1", + [memory_id], + |row| row.get(0), + )?; + let feed_seq_before = tx.query_row( + "SELECT COALESCE(MAX(feed_seq), 0) FROM mc_changefeed", + [], + |row| row.get::<_, i64>(0), + )?; + let inserted = tx.execute( + "INSERT OR IGNORE INTO mc_memory_evidence + (memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + memory_id, + content_hash, + source_session_id, + input.source_message_id, + input.source_type.unwrap_or("historian"), + input.now_ms, + ], + )? > 0; + if inserted { + tx.execute( + "UPDATE mc_memories + SET seen_count = MAX( + COALESCE(seen_count, 1), + (SELECT COUNT(DISTINCT source_session_id) FROM mc_memory_evidence WHERE memory_id = ?1) + ), + last_seen_at = ?2, + updated_at = ?2 + WHERE id = ?1", + params![memory_id, input.now_ms], + )?; + if let Some(memory) = load_memory_full_tx(tx, memory_id)? { + emit_verification_memory_snapshot_tx(tx, &memory, feed_seq_before)?; + } + } + Ok(inserted) +} + +fn merge_memory_evidence_tx( + tx: &rusqlite::Transaction<'_>, + target_id: i64, + source_ids: &[i64], +) -> rusqlite::Result { + for source_id in source_ids { + tx.execute( + "INSERT OR IGNORE INTO mc_memory_evidence + (memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at) + SELECT ?1, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM mc_memory_evidence WHERE memory_id = ?2", + params![target_id, source_id], + )?; + } + tx.query_row( + "SELECT COUNT(DISTINCT source_session_id) FROM mc_memory_evidence WHERE memory_id = ?1", + [target_id], + |row| row.get(0), + ) +} + /// Transaction-scoped ports used by the module facade. Every method operates on the transaction /// owned by `with_facade_command`, so the mutation and its response ledger row commit together. pub struct FacadeMutationTxn<'a> { @@ -5339,16 +5488,20 @@ impl<'a> FacadeMutationTxn<'a> { .optional() .map_err(|error| error.to_string())?; if let Some(id) = existing { - self.tx - .execute( - "UPDATE mc_memories + if input.source_session_id.is_some() { + record_memory_evidence_tx(self.tx, id, input).map_err(|error| error.to_string())?; + } else { + self.tx + .execute( + "UPDATE mc_memories SET seen_count = COALESCE(seen_count, 0) + 1, last_seen_at = ?1, updated_at = ?1 WHERE id = ?2", - params![input.now_ms, id], - ) - .map_err(|error| error.to_string())?; + params![input.now_ms, id], + ) + .map_err(|error| error.to_string())?; + } return Ok(id); } self.tx @@ -5374,7 +5527,19 @@ impl<'a> FacadeMutationTxn<'a> { ], ) .map_err(|error| error.to_string())?; - Ok(self.tx.last_insert_rowid()) + let id = self.tx.last_insert_rowid(); + record_memory_evidence_tx(self.tx, id, input).map_err(|error| error.to_string())?; + Ok(id) + } + + pub fn record_memory_evidence( + &self, + memory_id: i64, + input: InsertMemoryInput<'_>, + ) -> Result<(), String> { + record_memory_evidence_tx(self.tx, memory_id, input) + .map(|_| ()) + .map_err(|error| error.to_string()) } pub fn update_memory_content( @@ -5568,7 +5733,25 @@ impl<'a> FacadeMutationTxn<'a> { affected.push(target.clone()); affected.extend(source_rows.iter().cloned()); let merged_from = merged_from_json(&affected); - let seen_count: i64 = affected.iter().map(|memory| memory.seen_count.max(0)).sum(); + let feed_seq_before = self + .tx + .query_row( + "SELECT COALESCE(MAX(feed_seq), 0) FROM mc_changefeed", + [], + |row| row.get::<_, i64>(0), + ) + .map_err(|error| error.to_string())?; + let evidence_count = merge_memory_evidence_tx( + self.tx, + target_id, + &source_rows + .iter() + .map(|memory| memory.id) + .collect::>(), + ) + .map_err(|error| error.to_string())?; + let prior_seen_count: i64 = affected.iter().map(|memory| memory.seen_count.max(0)).sum(); + let seen_count = prior_seen_count.max(evidence_count); let retrieval_count: i64 = affected .iter() .map(|memory| memory.retrieval_count.max(0)) @@ -5641,7 +5824,12 @@ impl<'a> FacadeMutationTxn<'a> { }, ) .map_err(|error| error.to_string())?; - load_memory_full_tx(self.tx, target_id).map_err(|error| error.to_string()) + let memory = load_memory_full_tx(self.tx, target_id).map_err(|error| error.to_string())?; + if let Some(memory) = &memory { + emit_verification_memory_snapshot_tx(self.tx, memory, feed_seq_before) + .map_err(|error| error.to_string())?; + } + Ok(memory) } pub fn set_memory_verification( @@ -6461,6 +6649,7 @@ impl McStore { route_project_root: &str, ) -> Result<(), McStoreError> { self.with_note_conn_fenced(route_project_root, |tx| { + normalize_authority_route_tx(tx, context_store_uuid, project, route_project_root)?; tx.execute( "INSERT INTO mc_authority_route_bindings(route_project_root, context_store_uuid, project) VALUES (?1, ?2, ?3) @@ -11023,9 +11212,8 @@ impl McStore { } /// Insert a memory row unless an existing row already matches the project, category, - /// and normalized content hash. Duplicate hits update only bookkeeping fields such as - /// `seen_count` and timestamps, and skip the mutation log because the rendered content - /// did not change. + /// and normalized content hash. Duplicate hits count a source session once and skip the + /// mutation log because the rendered content did not change. pub fn insert_memory(&self, input: InsertMemoryInput<'_>) -> Result { if let Some(route_project_root) = input.route_project_root { self.enforce_facade_project_vocabulary( @@ -11045,14 +11233,18 @@ impl McStore { ) .optional()?; if let Some(id) = existing { - tx.execute( - "UPDATE mc_memories - SET seen_count = COALESCE(seen_count, 0) + 1, - last_seen_at = ?1, - updated_at = ?1 - WHERE id = ?2", - params![input.now_ms, id], - )?; + if input.source_session_id.is_some() { + record_memory_evidence_tx(tx, id, input)?; + } else { + tx.execute( + "UPDATE mc_memories + SET seen_count = COALESCE(seen_count, 0) + 1, + last_seen_at = ?1, + updated_at = ?1 + WHERE id = ?2", + params![input.now_ms, id], + )?; + } return Ok(id); } @@ -11077,7 +11269,9 @@ impl McStore { input.metadata_json, ], )?; - Ok(tx.last_insert_rowid()) + let id = tx.last_insert_rowid(); + record_memory_evidence_tx(tx, id, input)?; + Ok(id) })?; Ok(memory_id) } @@ -11296,7 +11490,22 @@ impl McStore { affected.push(target.clone()); affected.extend(source_rows.iter().cloned()); let merged_from = merged_from_json(&affected); - let seen_count: i64 = affected.iter().map(|memory| memory.seen_count.max(0)).sum(); + let feed_seq_before = tx.query_row( + "SELECT COALESCE(MAX(feed_seq), 0) FROM mc_changefeed", + [], + |row| row.get::<_, i64>(0), + )?; + let evidence_count = merge_memory_evidence_tx( + tx, + target_id, + &source_rows + .iter() + .map(|memory| memory.id) + .collect::>(), + )?; + let prior_seen_count: i64 = + affected.iter().map(|memory| memory.seen_count.max(0)).sum(); + let seen_count = prior_seen_count.max(evidence_count); let retrieval_count: i64 = affected .iter() .map(|memory| memory.retrieval_count.max(0)) @@ -11366,9 +11575,11 @@ impl McStore { }, )?; - Ok(MemoryMutationOutcome::Applied(Box::new( - load_memory_full_tx(tx, target_id)?, - ))) + let memory = load_memory_full_tx(tx, target_id)?; + if let Some(memory) = &memory { + emit_verification_memory_snapshot_tx(tx, memory, feed_seq_before)?; + } + Ok(MemoryMutationOutcome::Applied(Box::new(memory))) })?; match outcome { MemoryMutationOutcome::NotFound => Ok(None), @@ -11910,7 +12121,7 @@ impl McStore { other => { return Err(McStoreError::Serde(format!( "unknown historian side-channel kind {other:?}" - ))) + ))); } } Ok(()) @@ -14925,6 +15136,38 @@ impl McStore { .expect("every natural-key survivor was seeded") }) .collect::>(); + for module_row_id in module_row_ids.iter().copied().collect::>() { + tx.execute( + "DELETE FROM mc_memory_evidence WHERE memory_id = ?1", + [module_row_id], + )?; + } + for (row, module_row_id) in rows.iter().zip(&module_row_ids) { + let evidence = row + .snapshot + .get("evidence") + .cloned() + .unwrap_or_else(|| Value::Array(Vec::new())); + let evidence: Vec = serde_json::from_value(evidence) + .map_err(|error| { + rusqlite::Error::ToSqlConversionFailure(Box::new(error)) + })?; + for item in evidence { + tx.execute( + "INSERT OR IGNORE INTO mc_memory_evidence + (memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + module_row_id, + item.content_hash, + item.source_session_id, + item.source_message_id, + item.source_type, + item.observed_at, + ], + )?; + } + } drop(( memory_by_identity, memory_by_natural_key, @@ -15198,7 +15441,7 @@ impl McStore { op: "insert".to_string(), module_row_id: memory.id, content_hash: Some(memory.normalized_hash.clone()), - full_row_snapshot: memory_feed_snapshot(&memory, mapping), + full_row_snapshot: memory_feed_snapshot(conn, &memory, mapping)?, }) }) .collect::, rusqlite::Error>>()?; @@ -15410,6 +15653,7 @@ fn replace_authority_memories_tx( memory.mural_cue_rejection_count, ], )?; + replace_memory_evidence_tx(tx, existing_id, &memory.evidence)?; continue; } tx.execute( @@ -15508,6 +15752,39 @@ fn replace_authority_memories_tx( memory.mural_cue_rejection_count, ], )?; + let stored_id = tx.query_row( + "SELECT id FROM mc_memories WHERE project_path = ?1 AND category = ?2 AND normalized_hash = ?3", + params![&memory.project_path, &memory.category, &memory.normalized_hash], + |row| row.get::<_, i64>(0), + )?; + replace_memory_evidence_tx(tx, stored_id, &memory.evidence)?; + } + Ok(()) +} + +fn replace_memory_evidence_tx( + tx: &rusqlite::Transaction<'_>, + memory_id: i64, + evidence: &[ModuleMemoryEvidenceRow], +) -> rusqlite::Result<()> { + tx.execute( + "DELETE FROM mc_memory_evidence WHERE memory_id = ?1", + [memory_id], + )?; + for row in evidence { + tx.execute( + "INSERT INTO mc_memory_evidence + (memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + params![ + memory_id, + &row.content_hash, + &row.source_session_id, + row.source_message_id.as_deref(), + &row.source_type, + row.observed_at, + ], + )?; } Ok(()) } @@ -16280,8 +16557,30 @@ fn memory_mapping_feed_value( } } -fn memory_feed_snapshot(memory: &StoredMemoryFull, mapping: Value) -> Value { - serde_json::json!({ +fn memory_feed_snapshot( + conn: &rusqlite::Connection, + memory: &StoredMemoryFull, + mapping: Value, +) -> rusqlite::Result { + let evidence = { + let mut statement = conn.prepare( + "SELECT content_hash, source_session_id, source_message_id, source_type, observed_at + FROM mc_memory_evidence WHERE memory_id = ?1 + ORDER BY content_hash, source_session_id", + )?; + statement + .query_map([memory.id], |row| { + Ok(ModuleMemoryEvidenceRow { + content_hash: row.get(0)?, + source_session_id: row.get(1)?, + source_message_id: row.get(2)?, + source_type: row.get(3)?, + observed_at: row.get(4)?, + }) + })? + .collect::, _>>()? + }; + Ok(serde_json::json!({ "id": memory.id, "project_path": memory.project_path, "category": memory.category, @@ -16314,7 +16613,8 @@ fn memory_feed_snapshot(memory: &StoredMemoryFull, mapping: Value) -> Value { "mural_cue_at": memory.mural_cue_at, "mural_cue_rejection_count": memory.mural_cue_rejection_count, "mapping": mapping, - }) + "evidence": evidence, + })) } fn emit_verification_memory_snapshot_tx( @@ -16323,7 +16623,7 @@ fn emit_verification_memory_snapshot_tx( feed_seq_before: i64, ) -> rusqlite::Result<()> { let mapping = memory_mapping_feed_value(tx, memory.id)?; - let snapshot = serde_json::to_string(&memory_feed_snapshot(memory, mapping)) + let snapshot = serde_json::to_string(&memory_feed_snapshot(tx, memory, mapping)?) .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?; let enriched = tx.execute( "UPDATE mc_changefeed @@ -16665,7 +16965,7 @@ fn set_memory_mapping_tx( params![update.memory_id, project, files, now_ms], )?; let mapping = memory_mapping_feed_value(tx, update.memory_id)?; - let snapshot = serde_json::to_string(&memory_feed_snapshot(&memory, mapping)) + let snapshot = serde_json::to_string(&memory_feed_snapshot(tx, &memory, mapping)?) .map_err(|error| rusqlite::Error::ToSqlConversionFailure(Box::new(error)))?; tx.execute( "INSERT INTO mc_changefeed(domain, op, module_row_id, full_row_snapshot, content_hash) @@ -17023,9 +17323,10 @@ fn assert_memory_feed_snapshots_complete(store: &McStore) { #[cfg(test)] mod tests { - use super::*; use cortexkit_store_types::{Isolation, StorageBackend}; + use super::*; + fn descriptor(dir: &std::path::Path) -> StorageDescriptor { StorageDescriptor { module_id: "magic-context-test".to_string(), @@ -17130,6 +17431,7 @@ mod tests { category, content, source_session_id: None, + source_message_id: None, source_type: Some("tool"), importance: Some(50), expires_at: None, @@ -17199,14 +17501,18 @@ mod tests { assert!(store.delete_session("ses_delete", "/project").unwrap() >= 3); assert!(!store.has_cache_state("ses_delete").unwrap()); - assert!(store - .load_tags_for_session("ses_delete") - .unwrap() - .is_empty()); - assert!(store - .load_pending_agent_drops("ses_delete") - .unwrap() - .is_empty()); + assert!( + store + .load_tags_for_session("ses_delete") + .unwrap() + .is_empty() + ); + assert!( + store + .load_pending_agent_drops("ses_delete") + .unwrap() + .is_empty() + ); let remaining_note_types = store .inner .with_conn(|conn| { @@ -17367,19 +17673,27 @@ mod tests { drop(store); let reopened = McStore::open(&descriptor(dir.path())).unwrap(); - assert!(reopened - .knows_transform_session_root("refreshed", "/root-a") - .unwrap()); - assert!(!reopened - .knows_transform_session_root("refreshed", "/root-b") - .unwrap()); - assert!(reopened - .knows_transform_session_root("idle-live", "/root-a") - .unwrap()); + assert!( + reopened + .knows_transform_session_root("refreshed", "/root-a") + .unwrap() + ); + assert!( + !reopened + .knows_transform_session_root("refreshed", "/root-b") + .unwrap() + ); + assert!( + reopened + .knows_transform_session_root("idle-live", "/root-a") + .unwrap() + ); assert!(reopened.has_cache_state("idle-live").unwrap()); - assert!(!reopened - .knows_transform_session_root("deleted", "/root-a") - .unwrap()); + assert!( + !reopened + .knows_transform_session_root("deleted", "/root-a") + .unwrap() + ); assert!(!reopened.has_cache_state("deleted").unwrap()); } @@ -17436,12 +17750,16 @@ mod tests { }) .unwrap(); assert_eq!(stored_root, target_text); - assert!(store - .knows_transform_session_root("canonical-write", link_text) - .unwrap()); - assert!(store - .knows_transform_session_root("canonical-write", target_text) - .unwrap()); + assert!( + store + .knows_transform_session_root("canonical-write", link_text) + .unwrap() + ); + assert!( + store + .knows_transform_session_root("canonical-write", target_text) + .unwrap() + ); // Simulate a pre-migration row that retained the symlink spelling. store @@ -17464,12 +17782,16 @@ mod tests { Ok(()) }) .unwrap(); - assert!(store - .knows_transform_session_root("legacy-row", target_text) - .unwrap()); - assert!(store - .knows_transform_session_root("legacy-row", link_text) - .unwrap()); + assert!( + store + .knows_transform_session_root("legacy-row", target_text) + .unwrap() + ); + assert!( + store + .knows_transform_session_root("legacy-row", link_text) + .unwrap() + ); let missing = dir.path().join("gone"); assert_eq!(canonical_root(&missing), missing); @@ -17500,12 +17822,16 @@ mod tests { }, ) .unwrap(); - assert!(store - .knows_transform_session_root("missing-root", missing_text) - .unwrap()); - assert!(!store - .knows_transform_session_root("missing-root", "/another/gone") - .unwrap()); + assert!( + store + .knows_transform_session_root("missing-root", missing_text) + .unwrap() + ); + assert!( + !store + .knows_transform_session_root("missing-root", "/another/gone") + .unwrap() + ); } #[test] @@ -17516,7 +17842,7 @@ mod tests { let meta = ModuleMeta::default(); store.commit("ses_a", None, &core, &meta).unwrap(); // row_version now 1 - // A writer that still thinks the row is absent must conflict. + // A writer that still thinks the row is absent must conflict. let err = store.commit("ses_a", None, &core, &meta).unwrap_err(); match err { McStoreError::CasConflict { expected, found } => { @@ -18004,15 +18330,17 @@ mod tests { .unwrap(); let target_ids = vec!["a#0".to_string()]; - assert!(store - .append_pending_agent_drops_with_command( - "ses", - Some("tool-use-1"), - &target_ids, - 1, - false - ) - .is_err()); + assert!( + store + .append_pending_agent_drops_with_command( + "ses", + Some("tool-use-1"), + &target_ids, + 1, + false + ) + .is_err() + ); assert!(command_ledger_ids(&store, "ses").is_empty()); assert!(store.load_pending_agent_drops("ses").unwrap().is_empty()); @@ -18267,24 +18595,28 @@ mod tests { .unwrap(), 512 ); - assert!(store - .facade_mutation_ledger_response( - "session-facade-retention", - "ctx_memory", - "write", - "command-000", - ) - .unwrap() - .is_none()); - assert!(store - .facade_mutation_ledger_response( - "session-facade-retention", - "ctx_memory", - "write", - "command-512", - ) - .unwrap() - .is_some()); + assert!( + store + .facade_mutation_ledger_response( + "session-facade-retention", + "ctx_memory", + "write", + "command-000", + ) + .unwrap() + .is_none() + ); + assert!( + store + .facade_mutation_ledger_response( + "session-facade-retention", + "ctx_memory", + "write", + "command-512", + ) + .unwrap() + .is_some() + ); } #[test] @@ -18490,17 +18822,21 @@ mod tests { "deletion advances the generation and refreshes the cached table summary" ); - assert!(store - .append_channel1_nudge( - "ses", - "m2#0", - "\n\nhi", - 300 - ) - .unwrap()); - assert!(!store - .append_channel1_nudge("ses", "m2#0", "different", 400) - .unwrap()); + assert!( + store + .append_channel1_nudge( + "ses", + "m2#0", + "\n\nhi", + 300 + ) + .unwrap() + ); + assert!( + !store + .append_channel1_nudge("ses", "m2#0", "different", 400) + .unwrap() + ); let appends = store.load_channel1_appends("ses").unwrap(); assert_eq!(appends.len(), 1); assert_eq!( @@ -18509,23 +18845,27 @@ mod tests { ); assert!(store.append_user_hint("ses", "m1#0", "", 500).unwrap()); - assert!(!store - .append_user_hint("ses", "m1#0", "different", 600) - .unwrap()); - assert!(store - .append_user_hint( - "ses", - "m3#0", - "\n\nhit", - 700 - ) - .unwrap()); - assert_eq!( - store.load_user_hints("ses").unwrap(), - vec![ - UserHintRow { - block_id: "m1#0".to_string(), - hint_text: String::new(), + assert!( + !store + .append_user_hint("ses", "m1#0", "different", 600) + .unwrap() + ); + assert!( + store + .append_user_hint( + "ses", + "m3#0", + "\n\nhit", + 700 + ) + .unwrap() + ); + assert_eq!( + store.load_user_hints("ses").unwrap(), + vec![ + UserHintRow { + block_id: "m1#0".to_string(), + hint_text: String::new(), created_at: 500, }, UserHintRow { @@ -18683,21 +19023,27 @@ mod tests { let rejected = store .record_wrapup_command("session", "failed-command", "failed", 9, "changed", 20) .unwrap_err(); - assert!(rejected - .to_string() - .contains("nonterminal wrapup disposition")); + assert!( + rejected + .to_string() + .contains("nonterminal wrapup disposition") + ); assert_eq!( store.load_wrapup_command("session", "command").unwrap(), Some(first) ); - assert!(store - .load_wrapup_command("session", "failed-command") - .unwrap() - .is_none()); - assert!(store - .load_wrapup_command("session", "other") - .unwrap() - .is_none()); + assert!( + store + .load_wrapup_command("session", "failed-command") + .unwrap() + .is_none() + ); + assert!( + store + .load_wrapup_command("session", "other") + .unwrap() + .is_none() + ); } #[test] @@ -18769,10 +19115,12 @@ mod tests { }) .unwrap(); assert!(matches!(stale, RecordWrapupCommandOutcome::Stale { .. })); - assert!(store - .load_wrapup_command("session", "stale") - .unwrap() - .is_none()); + assert!( + store + .load_wrapup_command("session", "stale") + .unwrap() + .is_none() + ); let recorded = store .record_wrapup_command_if_current(WrapupCommandRecord { @@ -18832,9 +19180,11 @@ mod tests { assert_eq!(recorded.rounds, 3); assert_eq!(recorded.created_at, 99); assert!(recorded.summary.chars().count() <= 500); - assert!(recorded - .summary - .ends_with("; replaced failed record from 17")); + assert!( + recorded + .summary + .ends_with("; replaced failed record from 17") + ); assert_eq!( store.load_wrapup_command("session", "legacy").unwrap(), Some(recorded) @@ -19740,14 +20090,16 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let store = McStore::open(&descriptor(dir.path())).unwrap(); - assert!(store - .upsert_project_mural_artifact( - "git:project", - b"data:image/png;base64,YQ==", - "mural-a", - 100, - ) - .unwrap()); + assert!( + store + .upsert_project_mural_artifact( + "git:project", + b"data:image/png;base64,YQ==", + "mural-a", + 100, + ) + .unwrap() + ); let first = store .load_project_mural_artifact("git:project") .unwrap() @@ -19756,14 +20108,16 @@ mod tests { assert_eq!(first.content_hash, "mural-a"); assert_eq!(first.updated_at, 100); - assert!(!store - .upsert_project_mural_artifact( - "git:project", - b"data:image/png;base64,unexpected-but-same-hash", - "mural-a", - 200, - ) - .unwrap()); + assert!( + !store + .upsert_project_mural_artifact( + "git:project", + b"data:image/png;base64,unexpected-but-same-hash", + "mural-a", + 200, + ) + .unwrap() + ); let unchanged = store .load_project_mural_artifact("git:project") .unwrap() @@ -19773,14 +20127,16 @@ mod tests { "same hash must not bump artifact identity" ); - assert!(store - .upsert_project_mural_artifact( - "git:project", - b"data:image/png;base64,Yg==", - "mural-b", - 300, - ) - .unwrap()); + assert!( + store + .upsert_project_mural_artifact( + "git:project", + b"data:image/png;base64,Yg==", + "mural-b", + 300, + ) + .unwrap() + ); assert_eq!( store .load_project_mural_artifact("git:project") @@ -20173,6 +20529,369 @@ mod tests { ); } + #[test] + fn memory_evidence_is_idempotent_per_session_and_counts_independent_sessions() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let save = |session_id: &str, now_ms: i64| { + store + .insert_memory(InsertMemoryInput { + project_path: "git:project", + route_project_root: None, + category: "CONSTRAINTS", + content: "Use the shared store", + source_session_id: Some(session_id), + source_message_id: None, + source_type: Some("user"), + importance: Some(50), + expires_at: None, + metadata_json: None, + now_ms, + }) + .unwrap() + }; + + let first_id = save("session-a", 1); + assert_eq!(save("session-a", 2), first_id); + assert_eq!(save("session-b", 3), first_id); + + let (seen_count, evidence_count) = store + .inner + .with_conn(|conn| { + Ok(( + conn.query_row( + "SELECT seen_count FROM mc_memories WHERE id = ?1", + [first_id], + |row| row.get::<_, i64>(0), + )?, + conn.query_row( + "SELECT COUNT(*) FROM mc_memory_evidence WHERE memory_id = ?1", + [first_id], + |row| row.get::<_, i64>(0), + )?, + )) + }) + .unwrap(); + assert_eq!((seen_count, evidence_count), (2, 2)); + } + + #[test] + fn memory_evidence_keeps_the_content_version_observed_by_each_session() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let original = InsertMemoryInput { + project_path: "git:project", + route_project_root: None, + category: "CONSTRAINTS", + content: "Original fact", + source_session_id: Some("session-a"), + source_message_id: Some("assistant-a1"), + source_type: Some("agent"), + importance: Some(50), + expires_at: None, + metadata_json: None, + now_ms: 1, + }; + let memory_id = store.insert_memory(original).unwrap(); + store + .update_memory_content("git:project", memory_id, "Updated fact", 2) + .unwrap() + .unwrap(); + store + .insert_memory(InsertMemoryInput { + content: "Updated fact", + source_session_id: Some("session-b"), + source_message_id: Some("assistant-b1"), + now_ms: 3, + ..original + }) + .unwrap(); + + let hashes = store + .inner + .with_conn(|conn| { + let mut statement = conn.prepare( + "SELECT content_hash FROM mc_memory_evidence WHERE memory_id = ?1 ORDER BY observed_at", + )?; + statement + .query_map([memory_id], |row| row.get::<_, String>(0))? + .collect::, _>>() + }) + .unwrap(); + assert_eq!( + hashes, + vec![ + compute_normalized_memory_hash("Original fact"), + compute_normalized_memory_hash("Updated fact"), + ] + ); + let feed = store.pull_changefeed("memories", 0, 100).unwrap(); + let latest = feed + .rows + .iter() + .rev() + .find(|row| row.module_row_id == memory_id) + .unwrap(); + assert_eq!( + latest.full_row_snapshot["evidence"] + .as_array() + .unwrap() + .len(), + 2 + ); + assert_eq!( + store + .get_memory_full(memory_id) + .unwrap() + .unwrap() + .seen_count, + 2 + ); + } + + #[test] + fn memory_evidence_counts_one_session_once_across_content_versions() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let original = InsertMemoryInput { + project_path: "git:project", + route_project_root: None, + category: "CONSTRAINTS", + content: "Original fact", + source_session_id: Some("session-a"), + source_message_id: Some("assistant-a1"), + source_type: Some("agent"), + importance: Some(50), + expires_at: None, + metadata_json: None, + now_ms: 1, + }; + let memory_id = store.insert_memory(original).unwrap(); + store + .update_memory_content("git:project", memory_id, "Updated fact", 2) + .unwrap() + .unwrap(); + + store + .insert_memory(InsertMemoryInput { + content: "Updated fact", + source_message_id: Some("assistant-a2"), + now_ms: 3, + ..original + }) + .unwrap(); + + assert_eq!( + store + .get_memory_full(memory_id) + .unwrap() + .unwrap() + .seen_count, + 1 + ); + } + + #[test] + fn authority_state_sync_replaces_the_complete_memory_evidence_set() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let row = ModuleMemoryRow { + id: 7, + project_path: "git:project".to_string(), + category: "CONSTRAINTS".to_string(), + content: "Synced fact".to_string(), + normalized_hash: "synced-hash".to_string(), + status: "active".to_string(), + verification_status: "unverified".to_string(), + evidence: vec![ModuleMemoryEvidenceRow { + content_hash: "synced-hash".to_string(), + source_session_id: "session-a".to_string(), + source_message_id: Some("assistant-a1".to_string()), + source_type: "agent".to_string(), + observed_at: 11, + }], + ..Default::default() + }; + + store + .inner + .with_conn_fenced(|tx| replace_authority_memories_tx(tx, "/repo", &[row])) + .unwrap(); + + let evidence = store + .inner + .with_conn(|conn| { + conn.query_row( + "SELECT content_hash, source_session_id, source_message_id, source_type FROM mc_memory_evidence", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, String>(3)?, + )) + }, + ) + }) + .unwrap(); + assert_eq!( + evidence, + ( + "synced-hash".to_string(), + "session-a".to_string(), + Some("assistant-a1".to_string()), + "agent".to_string(), + ) + ); + } + + #[test] + fn memory_merge_unions_episode_evidence_onto_the_canonical_row() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let insert = |content: &str, session_id: &str, now_ms: i64| { + store + .insert_memory(InsertMemoryInput { + project_path: "git:project", + route_project_root: None, + category: "CONSTRAINTS", + content, + source_session_id: Some(session_id), + source_message_id: None, + source_type: Some("user"), + importance: Some(50), + expires_at: None, + metadata_json: None, + now_ms, + }) + .unwrap() + }; + let canonical_id = insert("First phrasing", "session-a", 1); + let source_id = insert("Independent phrasing", "session-b", 2); + + store + .merge_memories("git:project", canonical_id, &[source_id], "Merged fact", 3) + .unwrap() + .unwrap(); + + let evidence = store + .inner + .with_conn(|conn| { + let mut statement = conn.prepare( + "SELECT source_session_id, content_hash FROM mc_memory_evidence WHERE memory_id = ?1 ORDER BY source_session_id", + )?; + let sessions = statement + .query_map([canonical_id], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>()?; + Ok(sessions) + }) + .unwrap(); + assert_eq!( + evidence, + vec![ + ( + "session-a".to_string(), + compute_normalized_memory_hash("First phrasing") + ), + ( + "session-b".to_string(), + compute_normalized_memory_hash("Independent phrasing") + ), + ] + ); + } + + #[test] + fn memory_merge_preserves_a_legacy_seen_count_with_sparse_evidence() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let insert = |content: &str, session_id: &str, now_ms: i64| { + store + .insert_memory(InsertMemoryInput { + project_path: "git:project", + route_project_root: None, + category: "CONSTRAINTS", + content, + source_session_id: Some(session_id), + source_message_id: None, + source_type: Some("user"), + importance: Some(50), + expires_at: None, + metadata_json: None, + now_ms, + }) + .unwrap() + }; + let canonical_id = insert("First phrasing", "session-a", 1); + let source_id = insert("Independent phrasing", "session-b", 2); + store + .inner + .with_conn(|conn| { + conn.execute( + "UPDATE mc_memories SET seen_count = 10 WHERE id = ?1", + [canonical_id], + )?; + Ok(()) + }) + .unwrap(); + + let merged = store + .merge_memories("git:project", canonical_id, &[source_id], "Merged fact", 3) + .unwrap() + .unwrap(); + + assert_eq!(merged.seen_count, 10); + } + + #[test] + fn memory_merge_preserves_the_prior_aggregate_seen_count() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let insert = |content: &str, session_id: &str, now_ms: i64| { + store + .insert_memory(InsertMemoryInput { + project_path: "git:project", + route_project_root: None, + category: "CONSTRAINTS", + content, + source_session_id: Some(session_id), + source_message_id: None, + source_type: Some("agent"), + importance: Some(50), + expires_at: None, + metadata_json: None, + now_ms, + }) + .unwrap() + }; + let canonical_id = insert("First phrasing", "session-a", 1); + let source_id = insert("Independent phrasing", "session-b", 2); + store + .inner + .with_conn(|conn| { + conn.execute( + "UPDATE mc_memories SET seen_count = 3 WHERE id = ?1", + [canonical_id], + )?; + conn.execute( + "UPDATE mc_memories SET seen_count = 4 WHERE id = ?1", + [source_id], + )?; + Ok(()) + }) + .unwrap(); + + let merged = store + .merge_memories("git:project", canonical_id, &[source_id], "Merged fact", 3) + .unwrap() + .unwrap(); + + assert_eq!(merged.seen_count, 7); + } + #[test] fn double_open_same_path_is_rejected_by_lease() { let dir = tempfile::tempdir().unwrap(); @@ -20814,10 +21533,10 @@ mod tests { // memory 10: two updates → latest-wins (single terminal correction). log_mutation(&store, proj, "update", 10, "v1"); // id 1 log_mutation(&store, proj, "update", 10, "v2"); // id 2 (newer wins) - // memory 20: an archive then a later update → terminal (archive) outranks update. + // memory 20: an archive then a later update → terminal (archive) outranks update. log_mutation(&store, proj, "archive", 20, ""); // id 3 terminal log_mutation(&store, proj, "update", 20, "resurrect?"); // id 4 must NOT win - // memory 30: in the log but NOT in the rendered manifest → excluded. + // memory 30: in the log but NOT in the rendered manifest → excluded. log_mutation(&store, proj, "update", 30, "off-m0"); let rendered = [10i64, 20]; @@ -20851,10 +21570,12 @@ mod tests { assert_eq!(after[0].target_memory_id, 20); // empty manifest → no corrections (nothing in m0 to correct). - assert!(store - .memory_mutations_for_render(&projects, 0, &[]) - .unwrap() - .is_empty()); + assert!( + store + .memory_mutations_for_render(&projects, 0, &[]) + .unwrap() + .is_empty() + ); } #[test] @@ -20929,17 +21650,19 @@ mod tests { .unwrap(); let after = store.workspace_fingerprint(own, 0).unwrap(); assert_eq!(before, after); - assert!(store - .inner - .with_conn(|conn| conn - .query_row( - "SELECT epoch FROM mc_memory_visibility_epoch WHERE project_path = ?1", - params![foreign], - |row| row.get::<_, i64>(0), - ) - .optional() - .map(|value| value.is_none())) - .unwrap()); + assert!( + store + .inner + .with_conn(|conn| conn + .query_row( + "SELECT epoch FROM mc_memory_visibility_epoch WHERE project_path = ?1", + params![foreign], + |row| row.get::<_, i64>(0), + ) + .optional() + .map(|value| value.is_none())) + .unwrap() + ); } #[test] @@ -21287,11 +22010,13 @@ mod tests { .unwrap() .unwrap(); assert_eq!(archived.status, "archived"); - assert!(archived - .metadata_json - .as_deref() - .unwrap_or("") - .contains("archive_reason")); + assert!( + archived + .metadata_json + .as_deref() + .unwrap_or("") + .contains("archive_reason") + ); let mutations = store .memory_mutations_for_render(&[project.to_string()], before, &[id]) .unwrap(); @@ -21490,10 +22215,12 @@ mod tests { ); // a project in NO workspace → None (single-project fast path) - assert!(store - .resolve_workspace_membership("git:loner") - .unwrap() - .is_none()); + assert!( + store + .resolve_workspace_membership("git:loner") + .unwrap() + .is_none() + ); } #[test] @@ -22016,10 +22743,12 @@ mod tests { } let pending = store.historian_side_channel_status("ses").unwrap(); assert_eq!(pending.pending_count, 1); - assert!(pending - .last_failure - .as_deref() - .is_some_and(|error| error.contains(failed_kind))); + assert!( + pending + .last_failure + .as_deref() + .is_some_and(|error| error.contains(failed_kind)) + ); let retry = store .drain_historian_side_channels("ses", i64::MAX, 32) @@ -22163,10 +22892,12 @@ mod tests { }) .unwrap_err(); assert!(matches!(err, HistorianPublishError::CasConflict { .. })); - assert!(store - .load_chunk_transcripts_for_range("ses", 10, 21) - .unwrap() - .is_empty()); + assert!( + store + .load_chunk_transcripts_for_range("ses", 10, 21) + .unwrap() + .is_empty() + ); assert!(store.load_compartment_events("ses").unwrap().is_empty()); assert_eq!( store @@ -22210,10 +22941,12 @@ mod tests { raw_chunk_messages: None, }) .unwrap(); - assert!(store - .load_chunk_transcripts_for_range("ses", 10, 21) - .unwrap() - .is_empty()); + assert!( + store + .load_chunk_transcripts_for_range("ses", 10, 21) + .unwrap() + .is_empty() + ); } #[test] @@ -22468,10 +23201,12 @@ mod tests { assert_eq!(dismissal_feed.rows.len(), 1); assert_eq!(dismissal_feed.rows[0].module_row_id, first.id); assert_eq!(store.read_notes("git:proj", "ses", 25, 0).unwrap().len(), 1); - assert!(store - .search_notes_like("git:other", "ses", "pagination") - .unwrap() - .is_empty()); + assert!( + store + .search_notes_like("git:other", "ses", "pagination") + .unwrap() + .is_empty() + ); } #[test] @@ -22506,10 +23241,12 @@ mod tests { NoteCasOutcome::Applied(note) => note, other => panic!("unexpected cold-start update outcome: {other:?}"), }; - assert!(store - .read_project_notes("git:other", None, &["pending"], 25, 0) - .unwrap() - .is_empty()); + assert!( + store + .read_project_notes("git:other", None, &["pending"], 25, 0) + .unwrap() + .is_empty() + ); assert!(matches!( store.update_note_cas( "git:other", @@ -22547,10 +23284,12 @@ mod tests { .claim_note_delivery("git:proj", "serve-session", "pass-1", "pass-1", 40) .unwrap(); assert_eq!(first.len(), 1); - assert!(!store - .claim_note_delivery("git:proj", "serve-session", "pass-2", "pass-2", 50) - .unwrap() - .is_empty()); + assert!( + !store + .claim_note_delivery("git:proj", "serve-session", "pass-2", "pass-2", 50) + .unwrap() + .is_empty() + ); assert_eq!( store .ack_note_delivery("git:proj", "serve-session", "pass-2", 60) @@ -22559,10 +23298,12 @@ mod tests { ); // A newer acknowledged delivery closes the older lost attempt, so the // surfaced note cannot be delivered forever. - assert!(store - .claim_note_delivery("git:proj", "serve-session", "pass-3", "pass-3", 70) - .unwrap() - .is_empty()); + assert!( + store + .claim_note_delivery("git:proj", "serve-session", "pass-3", "pass-3", 70) + .unwrap() + .is_empty() + ); let surfaced = store .read_project_notes("git:proj", None, &["surfaced"], 25, 0) @@ -22581,10 +23322,12 @@ mod tests { ) .unwrap(); assert!(matches!(dismissed, NoteCasOutcome::Applied(note) if note.status == "dismissed")); - assert!(store - .claim_note_delivery("git:proj", "serve-session", "pass-4", "pass-4", 90) - .unwrap() - .is_empty()); + assert!( + store + .claim_note_delivery("git:proj", "serve-session", "pass-4", "pass-4", 90) + .unwrap() + .is_empty() + ); } #[test] @@ -22786,14 +23529,18 @@ mod tests { assert_eq!(loaded.meta.historian.state, HistorianPhase::Publishing); assert_eq!(loaded.meta.publication_floor_ordinal, None); assert!(store.load_compartments("ses").unwrap().is_empty()); - assert!(store - .load_chunk_transcripts_for_range("ses", 10, 21) - .unwrap() - .is_empty()); - assert!(store - .load_active_memories("git:proj", i64::MAX) - .unwrap() - .is_empty()); + assert!( + store + .load_chunk_transcripts_for_range("ses", 10, 21) + .unwrap() + .is_empty() + ); + assert!( + store + .load_active_memories("git:proj", i64::MAX) + .unwrap() + .is_empty() + ); } #[test] @@ -22916,11 +23663,13 @@ mod tests { .unwrap(); assert_eq!(outcome.revert_epoch, 1); assert_eq!(outcome.row_version, rv + 1); - assert!(outcome - .last_recut - .as_deref() - .unwrap() - .contains("dropped seq 2..3")); + assert!( + outcome + .last_recut + .as_deref() + .unwrap() + .contains("dropped seq 2..3") + ); let loaded = store.load("ses").unwrap(); assert_eq!(loaded.meta.revert_epoch, 1); assert_eq!(loaded.meta.last_recut, outcome.last_recut); @@ -23003,9 +23752,10 @@ mod tests { #[cfg(test)] mod shadow_tests { - use super::*; use cortexkit_store_types::{Isolation, StorageBackend}; + use super::*; + fn store(dir: &std::path::Path) -> McStore { McStore::open(&StorageDescriptor { module_id: "magic-context-test".to_string(), @@ -23054,15 +23804,16 @@ mod shadow_tests { let store = store(dir.path()); let route_project_root = "/worktrees/repo"; let identity = "git:identity"; - let insert = |project_path: &str, content: &str, now_ms| { + let insert = |project_path: &str, content: &str, session_id: &str, now_ms| { store .insert_memory(InsertMemoryInput { project_path, route_project_root: None, category: "CONSTRAINTS", content, - source_session_id: None, - source_type: Some("agent"), + source_session_id: Some(session_id), + source_message_id: None, + source_type: Some("user"), importance: Some(50), expires_at: None, metadata_json: None, @@ -23070,9 +23821,20 @@ mod shadow_tests { }) .unwrap() }; - let canonical = insert(identity, "same fact", 1); - let duplicate = insert(route_project_root, "same fact", 2); - let singleton = insert(route_project_root, "path-only fact", 3); + let canonical = insert(identity, "same fact", "session-a", 1); + let duplicate = insert(route_project_root, "same fact", "session-b", 2); + let singleton = insert(route_project_root, "path-only fact", "session-c", 3); + let evidence_before = store + .inner + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM mc_memory_evidence WHERE memory_id IN (?1, ?2)", + params![canonical, duplicate], + |row| row.get::<_, i64>(0), + ) + }) + .unwrap(); + assert_eq!(evidence_before, 2); store .inner .with_conn_fenced(|tx| { @@ -23098,6 +23860,17 @@ mod shadow_tests { .project_path, identity ); + let canonical_evidence = store + .inner + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM mc_memory_evidence WHERE memory_id = ?1", + [canonical], + |row| row.get::<_, i64>(0), + ) + }) + .unwrap(); + assert_eq!(canonical_evidence, 2); assert_eq!( store .get_memory_full(canonical) @@ -23106,10 +23879,12 @@ mod shadow_tests { .project_path, identity ); - assert!(store - .load_active_memories(route_project_root, 10) - .unwrap() - .is_empty()); + assert!( + store + .load_active_memories(route_project_root, 10) + .unwrap() + .is_empty() + ); let feed = store.pull_changefeed("memories", 0, 100).unwrap(); assert!( feed.rows @@ -23188,12 +23963,14 @@ mod shadow_tests { store.get_memory_full(1).unwrap().unwrap().project_path, "git:identity" ); - assert!(store - .pull_changefeed("memories", 0, 100) - .unwrap() - .rows - .iter() - .any(|row| row.module_row_id == 2 && row.op == "tombstone")); + assert!( + store + .pull_changefeed("memories", 0, 100) + .unwrap() + .rows + .iter() + .any(|row| row.module_row_id == 2 && row.op == "tombstone") + ); } #[test] @@ -23345,10 +24122,11 @@ mod shadow_tests { identity ); let feed = store.pull_changefeed("memories", 0, 100).unwrap(); - assert!(feed - .rows - .iter() - .any(|row| row.module_row_id == 2 && row.op == "tombstone")); + assert!( + feed.rows + .iter() + .any(|row| row.module_row_id == 2 && row.op == "tombstone") + ); } fn apply_state_sync_sections( @@ -23789,14 +24567,16 @@ mod shadow_tests { }) .unwrap(); assert_eq!(row, ("updated fact".to_string(), 50, None)); - assert!(store - .pull_changefeed("memories", 0, 100) - .unwrap() - .rows - .iter() - .any(|row| { - row.op == "update" && row.full_row_snapshot["classified_at"].is_null() - })); + assert!( + store + .pull_changefeed("memories", 0, 100) + .unwrap() + .rows + .iter() + .any(|row| { + row.op == "update" && row.full_row_snapshot["classified_at"].is_null() + }) + ); } } @@ -23828,6 +24608,7 @@ mod shadow_tests { category: "CONSTRAINTS", content: "must not split", source_session_id: None, + source_message_id: None, source_type: Some("agent"), importance: Some(50), expires_at: None, @@ -23851,6 +24632,7 @@ mod shadow_tests { category: "CONSTRAINTS", content: "first", source_session_id: None, + source_message_id: None, source_type: Some("tool"), importance: Some(50), expires_at: None, @@ -23917,6 +24699,7 @@ mod shadow_tests { category: "CONSTRAINTS", content: "classified fact", source_session_id: None, + source_message_id: None, source_type: Some("dreamer"), importance: Some(50), expires_at: None, @@ -24702,6 +25485,7 @@ mod shadow_tests { category: "CONSTRAINTS", content: "rejected", source_session_id: None, + source_message_id: None, source_type: Some("tool"), importance: Some(50), expires_at: None, @@ -24709,10 +25493,12 @@ mod shadow_tests { now_ms: 3, }) }); - assert!(rejected - .unwrap_err() - .to_string() - .contains("authority_draining")); + assert!( + rejected + .unwrap_err() + .to_string() + .contains("authority_draining") + ); assert_eq!( store .pull_changefeed("memories", 0, 100) @@ -24732,6 +25518,7 @@ mod shadow_tests { category: "CONSTRAINTS", content: "late", source_session_id: None, + source_message_id: None, source_type: Some("tool"), importance: Some(50), expires_at: None, @@ -24803,9 +25590,11 @@ mod shadow_tests { store .authority_begin_drain("store-uuid", "project", "memories", "first", 200, 100) .unwrap(); - assert!(store - .authority_begin_drain("store-uuid", "project", "memories", "second", 250, 150) - .is_err()); + assert!( + store + .authority_begin_drain("store-uuid", "project", "memories", "second", 250, 150) + .is_err() + ); let resumed = store .authority_begin_drain("store-uuid", "project", "memories", "second", 400, 201) .unwrap(); @@ -24839,31 +25628,35 @@ mod shadow_tests { .unwrap(); let live_token = second.coordinator_token.clone().expect("second token"); assert_ne!(stale_token, live_token); - assert!(store - .authority_drain_step( - "store-uuid", - "project", - "memories", - second.generation, - "seed", - Some(0), - &stale_token, - 150, - ) - .is_err()); - assert!(store - .authority_finish_drain( - "store-uuid", - "project", - "memories", - second.generation, - "hash", - "hash", - true, - &stale_token, - 150, - ) - .is_err()); + assert!( + store + .authority_drain_step( + "store-uuid", + "project", + "memories", + second.generation, + "seed", + Some(0), + &stale_token, + 150, + ) + .is_err() + ); + assert!( + store + .authority_finish_drain( + "store-uuid", + "project", + "memories", + second.generation, + "hash", + "hash", + true, + &stale_token, + 150, + ) + .is_err() + ); store .authority_drain_step( "store-uuid", @@ -25400,12 +26193,14 @@ mod shadow_tests { ) .unwrap(); assert_eq!(mapping.accepted, vec![2]); - assert!(store - .pull_changefeed("memories", 0, 100) - .unwrap() - .rows - .iter() - .any(|row| row.full_row_snapshot.get("mapping").is_some())); + assert!( + store + .pull_changefeed("memories", 0, 100) + .unwrap() + .rows + .iter() + .any(|row| row.full_row_snapshot.get("mapping").is_some()) + ); assert_memory_feed_snapshots_complete(&store); // The live-snapshot arm feeds the mirror resnapshot healer, so its rows must @@ -25422,10 +26217,11 @@ mod shadow_tests { ); } } - assert!(live - .rows - .iter() - .any(|row| row.full_row_snapshot["source_type"].as_str().is_some())); + assert!( + live.rows + .iter() + .any(|row| row.full_row_snapshot["source_type"].as_str().is_some()) + ); } #[test] @@ -25609,9 +26405,10 @@ mod shadow_tests { #[cfg(test)] mod lineage_descent_tests { - use super::*; use cortexkit_store_types::{Isolation, StorageBackend}; + use super::*; + fn store(dir: &std::path::Path) -> McStore { McStore::open(&StorageDescriptor { module_id: "magic-context-lineage-test".to_string(), @@ -25947,11 +26744,13 @@ mod lineage_descent_tests { }) .unwrap(); assert_eq!(c.source_key.as_deref(), Some("B")); - assert!(store - .load_compartments("C") - .unwrap() - .iter() - .any(|row| row.end_message_id == "b-work#0")); + assert!( + store + .load_compartments("C") + .unwrap() + .iter() + .any(|row| row.end_message_id == "b-work#0") + ); let mut unmarked_meta = ModuleMeta { initialized: true, @@ -25995,11 +26794,13 @@ mod lineage_descent_tests { }) .unwrap(); assert_eq!(e.source_key.as_deref(), Some("A")); - assert!(!store - .load_compartments("E") - .unwrap() - .iter() - .any(|row| row.end_message_id == "aborted-own-row#0")); + assert!( + !store + .load_compartments("E") + .unwrap() + .iter() + .any(|row| row.end_message_id == "aborted-own-row#0") + ); } #[test] @@ -26264,9 +27065,11 @@ mod lineage_descent_tests { now_ms: 1, }) .unwrap_err(); - assert!(error - .to_string() - .contains("lineage descent validation failed")); + assert!( + error + .to_string() + .contains("lineage descent validation failed") + ); assert!(store.load("B").unwrap().row_version.is_none()); assert!(store.load_compartments("B").unwrap().is_empty()); let prior_after = store.load("A").unwrap(); diff --git a/packages/pi-plugin/src/tools/ctx-memory.test.ts b/packages/pi-plugin/src/tools/ctx-memory.test.ts index 493062f27..cbedd1f79 100644 --- a/packages/pi-plugin/src/tools/ctx-memory.test.ts +++ b/packages/pi-plugin/src/tools/ctx-memory.test.ts @@ -1112,3 +1112,89 @@ describe("createCtxMemoryTool", () => { }); }); }); + +describe("Pi ctx_memory provenance", () => { + it("preserves content-bound evidence when memories merge", async () => { + const db = createTestDb(); + try { + const tool = createCtxMemoryTool({ + db, + resolveProjectIdentity: () => "git:project", + }); + const ctx = { + cwd: "/repo", + sessionManager: { getSessionId: () => "pi-session" }, + } as never; + await tool.execute( + "call-1", + { action: "write", category: "CONSTRAINTS", content: "First phrasing" }, + new AbortController().signal, + () => undefined, + ctx, + ); + await tool.execute( + "call-2", + { + action: "write", + category: "CONSTRAINTS", + content: "Second phrasing", + }, + new AbortController().signal, + () => undefined, + ctx, + ); + const ids = db + .prepare( + "SELECT id FROM memories ORDER BY id", + ) + .all() + .map((row) => row.id); + + await tool.execute( + "call-3", + { + action: "merge", + ids, + category: "CONSTRAINTS", + content: "Canonical phrasing", + }, + new AbortController().signal, + () => undefined, + ctx, + ); + + const canonical = db + .prepare( + "SELECT id FROM memories WHERE content = 'Canonical phrasing'", + ) + .get(); + expect(canonical).toBeDefined(); + expect( + db + .prepare( + `SELECT content_hash, source_message_id, source_type + FROM memory_evidence WHERE memory_id = ? ORDER BY observed_at`, + ) + .all(canonical?.id), + ).toEqual([ + { + content_hash: expect.any(String), + source_message_id: null, + source_type: "agent", + }, + { + content_hash: expect.any(String), + source_message_id: null, + source_type: "agent", + }, + { + content_hash: expect.any(String), + source_message_id: null, + source_type: "agent", + }, + ]); + } finally { + closeQuietly(db); + } + }); +}); diff --git a/packages/pi-plugin/src/tools/ctx-memory.ts b/packages/pi-plugin/src/tools/ctx-memory.ts index 85f415cc6..3b1051159 100644 --- a/packages/pi-plugin/src/tools/ctx-memory.ts +++ b/packages/pi-plugin/src/tools/ctx-memory.ts @@ -42,13 +42,14 @@ import { hasMemoryClassifiedAtColumn, hasMemoryShareableColumn, insertMemory, + insertMemoryIdempotent, type Memory, type MemoryCategory, mergeMemoryStats, + recordMemoryEvidence, saveEmbedding, supersededMemory, updateMemoryContent, - updateMemorySeenCount, V2_MEMORY_CATEGORIES, } from "@magic-context/core/features/magic-context/memory"; import { @@ -471,26 +472,19 @@ export function createCtxMemoryTool( return err("Error: 'category' is required when action is 'write'."); } - const existing = getMemoryByHash( - deps.db, - projectIdentity, - rawCategory, - computeNormalizedHash(content), - ); - if (existing) { - updateMemorySeenCount(deps.db, existing.id); - return ok( - `Memory already exists [ID: ${existing.id}] in ${rawCategory} (seen count incremented).`, - ); - } - - const memory = insertMemory(deps.db, { + const insertResult = insertMemoryIdempotent(deps.db, { projectPath: projectIdentity, category: rawCategory, content, sourceSessionId: sessionId, sourceType: dreamerAllowed ? "dreamer" : "agent", }); + if (!insertResult.inserted) { + return ok( + `Memory already exists [ID: ${insertResult.memory.id}] in ${rawCategory}.`, + ); + } + const memory = insertResult.memory; queueEmbedding({ deps, projectIdentity, memoryId: memory.id, content }); // Do NOT invalidate the m[0]/m[1] cache here. An additive write is a @@ -597,6 +591,13 @@ export function createCtxMemoryTool( category: memory.category, newContent: content, }); + recordMemoryEvidence(deps.db, memory.id, { + projectPath: targetIdentity, + category: memory.category, + content, + sourceSessionId: sessionId, + sourceType: dreamerAllowed ? "dreamer" : "agent", + }); }); queueEmbedding({ deps, @@ -784,6 +785,13 @@ export function createCtxMemoryTool( mergedFrom, mergedStatus, ); + recordMemoryEvidence(deps.db, canonicalMemory.id, { + projectPath: projectIdentity, + category, + content, + sourceSessionId: sessionId, + sourceType: dreamerAllowed ? "dreamer" : "agent", + }); for (const memory of sourceMemories) { if (memory.id === canonicalMemory.id) { diff --git a/packages/plugin/docs/MEMORY-DESIGN.md b/packages/plugin/docs/MEMORY-DESIGN.md index 4548ca1f5..90992938d 100644 --- a/packages/plugin/docs/MEMORY-DESIGN.md +++ b/packages/plugin/docs/MEMORY-DESIGN.md @@ -166,6 +166,15 @@ Agent-initiated writes and deletes: the `ctx_memory` tool allows explicit write/ Two counters track different signals: - **`seen_count`**: incremented when historian re-extracts the same fact in a later session. Indicates the fact is being repeatedly discovered, suggesting durability. + +Episode provenance lives in `memory_evidence`, keyed by `(memory_id, content_hash, +source_session_id)`. `content_hash` is the memory's normalized content hash at the +time of observation, so edits and semantic merges retain the evidence for each +wording instead of rebinding old sessions to new text. `source_session_id` is the +host session/conversation key in every runtime. `source_message_id` means a +host-native owner message only: OpenCode supplies the assistant message that owns a +tool call; Pi and the Rust facade leave it NULL because they expose no trustworthy +message link. Primary tool writes are therefore `agent`, never inferred as `user`. - **`retrieval_count`**: incremented only when the agent actively searches for and retrieves this memory via `ctx_memory(action="search", ...)`. Indicates the fact is actively useful. Permanence promotion keys off `retrieval_count >= 3`, not `seen_count`. A fact that gets re-extracted 10 times but never retrieved may just be noise. A fact retrieved 3 times is proven useful. diff --git a/packages/plugin/src/features/magic-context/context-authority.test.ts b/packages/plugin/src/features/magic-context/context-authority.test.ts index 8489a3ad7..cf8f27a6c 100644 --- a/packages/plugin/src/features/magic-context/context-authority.test.ts +++ b/packages/plugin/src/features/magic-context/context-authority.test.ts @@ -78,6 +78,56 @@ function protocol(seedCalls: { bytes: number[] }): AuthorityModuleClient { } describe("memory authority protocol", () => { + test("mirror sync installs the module's content-bound evidence set", () => { + const database = db(); + applyMirrorPage({ + db: database, + page: { + domain: "memories", + cursor: 0, + next_cursor: 1, + has_more: false, + rows: [ + { + feed_seq: 1, + domain: "memories", + op: "insert", + module_row_id: 9, + full_row_snapshot: { + project_path: "/repo", + category: "CONSTRAINTS", + content: "Mirrored fact", + normalized_hash: "fact-hash", + evidence: [ + { + content_hash: "fact-hash", + source_session_id: "session-a", + source_message_id: "assistant-a1", + source_type: "agent", + observed_at: 11, + }, + ], + }, + content_hash: "fact-hash", + }, + ], + }, + }); + + expect( + database + .prepare( + "SELECT content_hash, source_session_id, source_message_id, source_type FROM memory_evidence", + ) + .get(), + ).toEqual({ + content_hash: "fact-hash", + source_session_id: "session-a", + source_message_id: "assistant-a1", + source_type: "agent", + }); + }); + test("historical sparse note feed rows preserve rich local columns", () => { const database = db(); const localStoreUuid = ensureContextStoreUuid(database); diff --git a/packages/plugin/src/features/magic-context/context-authority.ts b/packages/plugin/src/features/magic-context/context-authority.ts index 72c83db29..145a93590 100644 --- a/packages/plugin/src/features/magic-context/context-authority.ts +++ b/packages/plugin/src/features/magic-context/context-authority.ts @@ -1064,6 +1064,8 @@ interface MirrorPageStatements { upsertPendingReference: Statement; deleteMemoryVerifications: Statement; insertMemoryVerification: Statement; + deleteMemoryEvidence: Statement; + insertMemoryEvidence: Statement; noteById: Statement; noteIdByStoreId: Statement; insertNote: Statement; @@ -1184,6 +1186,12 @@ function prepareMirrorPageStatements(db: Database): MirrorPageStatements { insertMemoryVerification: db.prepare( "INSERT INTO memory_verifications(memory_id, file_path, verified_at, mapped_at) VALUES (?, ?, ?, ?)", ), + deleteMemoryEvidence: db.prepare("DELETE FROM memory_evidence WHERE memory_id = ?"), + insertMemoryEvidence: db.prepare( + `INSERT INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) VALUES (?, ?, ?, ?, ?, ?)`, + ), noteById: db.prepare("SELECT * FROM notes WHERE id = ?"), noteIdByStoreId: db.prepare( "SELECT id FROM notes WHERE id = ? AND type = 'smart' AND project_path = ?", @@ -1648,6 +1656,33 @@ function applyMemoryRow(db: Database, feed: ChangefeedRow, statements: MirrorPag } } } + if (has("evidence")) { + if (!Array.isArray(row.evidence)) { + throw new Error("memory feed evidence must be an array"); + } + statements.deleteMemoryEvidence.run(contextId); + for (const item of row.evidence) { + if (!item || typeof item !== "object" || Array.isArray(item)) { + throw new Error("memory feed evidence row must be an object"); + } + const evidence = item as Record; + const contentHash = rowString(evidence, "content_hash"); + const sourceSessionId = rowString(evidence, "source_session_id"); + const sourceType = rowString(evidence, "source_type"); + const observedAt = rowNumber(evidence, "observed_at", -1); + if (!contentHash || !sourceSessionId || !sourceType || observedAt < 0) { + throw new Error("memory feed evidence row is incomplete"); + } + statements.insertMemoryEvidence.run( + contextId, + contentHash, + sourceSessionId, + rowNullableString(evidence, "source_message_id"), + sourceType, + observedAt, + ); + } + } } function repairNullClobberedMemoryRows(statements: MirrorPageStatements): void { diff --git a/packages/plugin/src/features/magic-context/dreamer/retrospective-learnings.ts b/packages/plugin/src/features/magic-context/dreamer/retrospective-learnings.ts index da28bea54..6391ffdcf 100644 --- a/packages/plugin/src/features/magic-context/dreamer/retrospective-learnings.ts +++ b/packages/plugin/src/features/magic-context/dreamer/retrospective-learnings.ts @@ -1,6 +1,5 @@ import type { Database } from "../../../shared/sqlite"; -import { computeNormalizedHash } from "../memory/normalize-hash"; -import { getMemoryByHash, insertMemory } from "../memory/storage-memory"; +import { insertMemoryIdempotent } from "../memory/storage-memory"; import type { MemoryCategory } from "../memory/types"; import { insertUserMemoryCandidates } from "../user-memory/storage-user-memory"; @@ -173,16 +172,7 @@ export function applyRetrospectiveLearnings(args: { if (learning.route === "memory") { if (!learning.category) continue; - // Skip an already-stored identical memory rather than throwing on the - // UNIQUE(project_path, category, normalized_hash) constraint. - const existing = getMemoryByHash( - args.db, - args.projectIdentity, - learning.category, - computeNormalizedHash(learning.content), - ); - if (existing) continue; - insertMemory(args.db, { + const inserted = insertMemoryIdempotent(args.db, { projectPath: args.projectIdentity, category: learning.category, content: learning.content, @@ -190,7 +180,7 @@ export function applyRetrospectiveLearnings(args: { sourceType: "dreamer", metadataJson: JSON.stringify({ source: "retrospective" }), }); - result.memoryWritten += 1; + if (inserted.inserted) result.memoryWritten += 1; continue; } diff --git a/packages/plugin/src/features/magic-context/memory/promotion.ts b/packages/plugin/src/features/magic-context/memory/promotion.ts index 0977bf9d9..b8d183354 100644 --- a/packages/plugin/src/features/magic-context/memory/promotion.ts +++ b/packages/plugin/src/features/magic-context/memory/promotion.ts @@ -2,13 +2,7 @@ import { sessionLog } from "../../../shared/logger"; import type { Database } from "../../../shared/sqlite"; import { CATEGORY_DEFAULT_TTL, PROMOTABLE_CATEGORIES } from "./constants"; import { embedTextForProject } from "./embedding"; -import { computeNormalizedHash } from "./normalize-hash"; -import { - getMemoryByHash, - getMemoryById, - insertMemory, - updateMemorySeenCount, -} from "./storage-memory"; +import { getMemoryById, insertMemoryIdempotent } from "./storage-memory"; import { saveEmbeddingIfHashMatches } from "./storage-memory-embeddings"; import type { MemoryCategory, MemoryInput } from "./types"; @@ -59,14 +53,6 @@ export function promoteSessionFactsDurable( continue; } - const normalizedHash = computeNormalizedHash(fact.content); - const existingMemory = getMemoryByHash(db, projectPath, fact.category, normalizedHash); - - if (existingMemory) { - updateMemorySeenCount(db, existingMemory.id); - continue; - } - const memoryInput: MemoryInput = { projectPath, category: fact.category, @@ -76,8 +62,10 @@ export function promoteSessionFactsDurable( expiresAt: resolveExpiresAt(fact.category), }; - const memory = insertMemory(db, memoryInput); - refs.push({ memoryId: memory.id, content: memory.content }); + const result = insertMemoryIdempotent(db, memoryInput); + if (result.inserted) { + refs.push({ memoryId: result.memory.id, content: result.memory.content }); + } } return refs; diff --git a/packages/plugin/src/features/magic-context/memory/relocate-memory.ts b/packages/plugin/src/features/magic-context/memory/relocate-memory.ts index 46243936c..9dbf7556a 100644 --- a/packages/plugin/src/features/magic-context/memory/relocate-memory.ts +++ b/packages/plugin/src/features/magic-context/memory/relocate-memory.ts @@ -77,12 +77,30 @@ export function rekeyMemoryRowWithCollisionMerge( | undefined; if (collision && collision.id !== rowId) { - const mergedSeen = Math.max(collision.seen_count ?? 1, row.seen_count ?? 1); - if (mergedSeen !== (collision.seen_count ?? 1)) { - db.prepare("UPDATE memories SET seen_count = ? WHERE id = ?").run( - mergedSeen, - collision.id, - ); + const hasEvidence = db + .prepare( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_evidence'", + ) + .get(); + if (hasEvidence) { + db.prepare( + `INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence WHERE memory_id = ?`, + ).run(collision.id, rowId); + db.prepare( + `UPDATE memories SET seen_count = MAX( + COALESCE(seen_count, 1), + ?, + (SELECT COUNT(DISTINCT source_session_id) FROM memory_evidence WHERE memory_id = ?) + ) WHERE id = ?`, + ).run(row.seen_count ?? 1, collision.id, collision.id); + } else { + db.prepare( + "UPDATE memories SET seen_count = MAX(COALESCE(seen_count, 1), ?) WHERE id = ?", + ).run(row.seen_count ?? 1, collision.id); } // Preserve an embedding on the surviving target BEFORE the source row's // embedding FK-cascades away on DELETE (memory_embeddings.memory_id @@ -182,9 +200,50 @@ export function copyMemoriesToProject( `INSERT OR IGNORE INTO memory_embeddings (memory_id, embedding, model_id) SELECT ?, embedding, model_id FROM memory_embeddings WHERE memory_id = ?`, ); + const hasEvidence = db + .prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_evidence'") + .get(); + const copyEvidenceStmt = hasEvidence + ? db.prepare( + `INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence WHERE memory_id = ?`, + ) + : null; let relocated = 0; let skipped = 0; for (const id of ids) { + const source = db + .prepare("SELECT category, normalized_hash, seen_count FROM memories WHERE id = ?") + .get(id) as + | { category?: string; normalized_hash?: string; seen_count?: number } + | undefined; + const collision = + source?.category && source.normalized_hash + ? (db + .prepare( + "SELECT id FROM memories WHERE project_path = ? AND category = ? AND normalized_hash = ? LIMIT 1", + ) + .get(toIdentity, source.category, source.normalized_hash) as + | { id?: number } + | undefined) + : undefined; + if (collision?.id !== undefined) { + copyEvidenceStmt?.run(collision.id, id); + if (copyEvidenceStmt) { + db.prepare( + `UPDATE memories SET seen_count = MAX( + COALESCE(seen_count, 1), + ?, + (SELECT COUNT(DISTINCT source_session_id) FROM memory_evidence WHERE memory_id = ?) + ) WHERE id = ?`, + ).run(source?.seen_count ?? 1, collision.id, collision.id); + } + skipped += 1; + continue; + } const result = insertStmt.run(toIdentity, id) as { changes?: number; lastInsertRowid?: number | bigint; @@ -192,6 +251,7 @@ export function copyMemoriesToProject( if ((result.changes ?? 0) > 0) { relocated += 1; copyEmbeddingStmt.run(Number(result.lastInsertRowid), id); + copyEvidenceStmt?.run(Number(result.lastInsertRowid), id); } else { skipped += 1; } diff --git a/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts b/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts new file mode 100644 index 000000000..81fa0d944 --- /dev/null +++ b/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts @@ -0,0 +1,240 @@ +/// + +import { afterEach, describe, expect, it } from "bun:test"; +import { Database } from "../../../shared/sqlite"; +import { closeQuietly } from "../../../shared/sqlite-helpers"; +import { runMigrations } from "../migrations"; +import { initializeDatabase } from "../storage-db"; +import { computeNormalizedHash } from "./normalize-hash"; +import { copyMemoriesToProject, rekeyMemoryRowWithCollisionMerge } from "./relocate-memory"; +import { + archiveMemory, + deleteMemory, + getMemoryById, + insertMemoryIdempotent, + mergeMemoryStats, + updateMemoryContent, +} from "./storage-memory"; + +type EvidenceRow = { + memory_id: number; + content_hash: string; + source_session_id: string; + source_message_id: string | null; + source_type: string; +}; + +let db: Database; + +function makeDatabase(): Database { + const database = new Database(":memory:"); + initializeDatabase(database); + runMigrations(database); + return database; +} + +function evidence(memoryId: number): EvidenceRow[] { + return db + .prepare( + `SELECT memory_id, content_hash, source_session_id, source_message_id, source_type + FROM memory_evidence + WHERE memory_id = ? + ORDER BY source_session_id`, + ) + .all(memoryId) as EvidenceRow[]; +} + +function save(content: string, sessionId: string, messageId: string) { + return insertMemoryIdempotent(db, { + projectPath: "git:project", + category: "CONSTRAINTS", + content, + sourceSessionId: sessionId, + sourceMessageId: messageId, + sourceType: "user", + }).memory; +} + +afterEach(() => { + if (db) closeQuietly(db); +}); + +describe("memory evidence lifecycle", () => { + it("counts one exact observation per session episode", () => { + db = makeDatabase(); + + const first = save("Use the shared store", "session-a", "user-a1"); + save("Use the shared store", "session-a", "user-a2"); + const corroborated = save("Use the shared store", "session-b", "user-b1"); + + expect(corroborated.id).toBe(first.id); + expect(corroborated.seenCount).toBe(2); + expect(evidence(first.id)).toEqual([ + { + memory_id: first.id, + content_hash: first.normalizedHash, + source_session_id: "session-a", + source_message_id: "user-a1", + source_type: "user", + }, + { + memory_id: first.id, + content_hash: first.normalizedHash, + source_session_id: "session-b", + source_message_id: "user-b1", + source_type: "user", + }, + ]); + }); + + it("preserves evidence through update archive and delete lifecycle", () => { + db = makeDatabase(); + const memory = save("Original fact", "session-a", "user-a1"); + + updateMemoryContent(db, memory.id, "Updated fact", computeNormalizedHash("Updated fact")); + save("Updated fact", "session-b", "assistant-b1"); + archiveMemory(db, memory.id); + + expect(evidence(memory.id).map((row) => row.content_hash)).toEqual([ + computeNormalizedHash("Original fact"), + computeNormalizedHash("Updated fact"), + ]); + deleteMemory(db, memory.id); + expect(evidence(memory.id)).toEqual([]); + }); + + it("counts one session once across content versions", () => { + db = makeDatabase(); + const memory = save("Original fact", "session-a", "user-a1"); + + updateMemoryContent(db, memory.id, "Updated fact", computeNormalizedHash("Updated fact")); + const updated = save("Updated fact", "session-a", "user-a2"); + + expect(evidence(memory.id)).toHaveLength(2); + expect(updated.seenCount).toBe(1); + }); + + it("unions source evidence onto the canonical memory during merge", () => { + db = makeDatabase(); + const canonical = save("First phrasing", "session-a", "user-a1"); + const source = save("Independent phrasing", "session-b", "user-b1"); + + mergeMemoryStats( + db, + canonical.id, + canonical.seenCount + source.seenCount, + 0, + JSON.stringify([canonical.id, source.id]), + "active", + ); + + expect(evidence(canonical.id).map((row) => row.source_session_id)).toEqual([ + "session-a", + "session-b", + ]); + expect(getMemoryById(db, canonical.id)?.seenCount).toBe(2); + }); + + it("preserves a legacy seen count during a sparsely evidenced merge", () => { + db = makeDatabase(); + const canonical = save("First phrasing", "session-a", "user-a1"); + const source = save("Independent phrasing", "session-b", "user-b1"); + db.prepare("UPDATE memories SET seen_count = 10 WHERE id = ?").run(canonical.id); + + mergeMemoryStats( + db, + canonical.id, + canonical.seenCount + source.seenCount, + 0, + JSON.stringify([canonical.id, source.id]), + "active", + ); + + expect(getMemoryById(db, canonical.id)?.seenCount).toBe(10); + }); + + it("preserves evidence when identity relocation merges an exact collision", () => { + db = makeDatabase(); + const target = save("Same fact", "session-a", "user-a1"); + const source = insertMemoryIdempotent(db, { + projectPath: "git:old-project", + category: "CONSTRAINTS", + content: "Same fact", + sourceSessionId: "session-b", + sourceMessageId: "user-b1", + sourceType: "user", + }).memory; + + db.transaction(() => { + rekeyMemoryRowWithCollisionMerge(db, source.id, "git:old-project", "git:project"); + })(); + + expect(evidence(target.id).map((row) => row.source_session_id)).toEqual([ + "session-a", + "session-b", + ]); + expect(getMemoryById(db, source.id)).toBeNull(); + }); + + it("reconciles relocation collisions against unioned distinct-session evidence", () => { + db = makeDatabase(); + const target = save("Same fact", "session-a", "user-a1"); + const source = insertMemoryIdempotent(db, { + projectPath: "git:old-project", + category: "CONSTRAINTS", + content: "Same fact", + sourceSessionId: "session-b", + sourceMessageId: "user-b1", + sourceType: "user", + }).memory; + db.prepare( + `INSERT INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) VALUES (?, ?, 'session-c', 'user-c1', 'user', ?)`, + ).run(source.id, source.normalizedHash, Date.now()); + db.prepare("UPDATE memories SET seen_count = 2 WHERE id = ?").run(source.id); + + db.transaction(() => { + rekeyMemoryRowWithCollisionMerge(db, source.id, "git:old-project", "git:project"); + })(); + + expect(evidence(target.id)).toHaveLength(3); + expect(getMemoryById(db, target.id)?.seenCount).toBe(3); + }); + + it("rolls back a new memory when evidence insertion fails", () => { + db = makeDatabase(); + db.exec(` + CREATE TRIGGER fail_memory_evidence BEFORE INSERT ON memory_evidence + BEGIN + SELECT RAISE(ABORT, 'injected evidence failure'); + END; + `); + + expect(() => save("Atomic fact", "session-a", "assistant-a1")).toThrow( + "injected evidence failure", + ); + expect(db.prepare("SELECT COUNT(*) AS count FROM memories").get()).toEqual({ count: 0 }); + }); + + it("unions evidence when a copy collides with an existing target", () => { + db = makeDatabase(); + const target = save("Same fact", "session-a", "assistant-a1"); + const source = insertMemoryIdempotent(db, { + projectPath: "git:source", + category: "CONSTRAINTS", + content: "Same fact", + sourceSessionId: "session-b", + sourceMessageId: "assistant-b1", + sourceType: "agent", + }).memory; + + db.transaction(() => copyMemoriesToProject(db, [source.id], "git:project"))(); + + expect(evidence(target.id).map((row) => row.source_session_id)).toEqual([ + "session-a", + "session-b", + ]); + expect(getMemoryById(db, target.id)?.seenCount).toBe(2); + }); +}); diff --git a/packages/plugin/src/features/magic-context/memory/storage-memory.ts b/packages/plugin/src/features/magic-context/memory/storage-memory.ts index 2728b9d74..95d0a245e 100644 --- a/packages/plugin/src/features/magic-context/memory/storage-memory.ts +++ b/packages/plugin/src/features/magic-context/memory/storage-memory.ts @@ -108,6 +108,7 @@ const memoryImportanceColumnCache = new WeakMap(); const memoryScopeColumnCache = new WeakMap(); const memoryShareableColumnCache = new WeakMap(); const memoryClassifiedAtColumnCache = new WeakMap(); +const memoryEvidenceTableCache = new WeakMap(); export interface MemoryCountsByStatus { total: number; @@ -237,6 +238,81 @@ function isUniqueConstraintError(error: unknown): boolean { ); } +function hasMemoryEvidenceTable(db: Database): boolean { + const cached = memoryEvidenceTableCache.get(db); + if (cached !== undefined) return cached; + const present = Boolean( + db + .prepare( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'memory_evidence'", + ) + .get(), + ); + memoryEvidenceTableCache.set(db, present); + return present; +} + +export interface MemoryEvidence { + readonly content_hash: string; + readonly source_session_id: string; + readonly source_message_id: string | null; + readonly source_type: MemorySourceType; + readonly observed_at: number; +} + +export function getMemoryEvidence(db: Database, memoryId: number): MemoryEvidence[] { + if (!hasMemoryEvidenceTable(db)) return []; + return db + .prepare( + `SELECT content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence WHERE memory_id = ? + ORDER BY content_hash, source_session_id`, + ) + .all(memoryId) as MemoryEvidence[]; +} + +function recordMemoryEvidenceInCurrentTransaction( + db: Database, + memoryId: number, + input: MemoryInput, +): void { + if (!input.sourceSessionId || !hasMemoryEvidenceTable(db)) return; + const memory = db.prepare("SELECT normalized_hash FROM memories WHERE id = ?").get(memoryId) as + | { normalized_hash?: string } + | undefined; + if (!memory?.normalized_hash) return; + const now = Date.now(); + const result = db + .prepare( + `INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) VALUES (?, ?, ?, ?, ?, ?)`, + ) + .run( + memoryId, + memory.normalized_hash, + input.sourceSessionId, + input.sourceMessageId ?? null, + input.sourceType ?? "historian", + now, + ) as { changes?: number }; + if ((result.changes ?? 0) === 0) return; + db.prepare( + `UPDATE memories + SET seen_count = MAX( + COALESCE(seen_count, 1), + (SELECT COUNT(DISTINCT source_session_id) FROM memory_evidence WHERE memory_id = ?) + ), + last_seen_at = ?, + updated_at = ? + WHERE id = ?`, + ).run(memoryId, now, now, memoryId); +} + +export function recordMemoryEvidence(db: Database, memoryId: number, input: MemoryInput): void { + db.transaction(() => recordMemoryEvidenceInCurrentTransaction(db, memoryId, input))(); +} + function isNullableString(value: unknown): value is string | null { return value === null || typeof value === "string"; } @@ -480,7 +556,7 @@ function getMergeMemoryStatsStatement(db: Database): PreparedStatement { let stmt = mergeMemoryStatsStatements.get(db); if (!stmt) { stmt = db.prepare( - "UPDATE memories SET seen_count = ?, retrieval_count = ?, merged_from = ?, status = ?, updated_at = ? WHERE id = ?", + "UPDATE memories SET seen_count = MAX(COALESCE(seen_count, 1), ?, ?), retrieval_count = ?, merged_from = ?, status = ?, updated_at = ? WHERE id = ?", ); mergeMemoryStatsStatements.set(db, stmt); } @@ -611,19 +687,23 @@ function assertTsMemoryIdWriteAllowed(db: Database, id: number): Memory | null { } export function insertMemory(db: Database, input: MemoryInput): Memory { - assertTsMemoryWriteAllowed(db, input.projectPath); - const now = Date.now(); - const normalizedHash = computeNormalizedHash(input.content); - const insertValues = buildInsertMemoryValues( - input, - normalizedHash, - now, - hasMemoryImportanceColumn(db), - ); - const result = getInsertMemoryStatement(db).run(...insertValues); - - const insertedResult = result as { lastInsertRowid?: number | bigint }; - const inserted = loadInsertedMemory(db, insertedResult.lastInsertRowid); + const inserted = db.transaction(() => { + assertTsMemoryWriteAllowed(db, input.projectPath); + const now = Date.now(); + const normalizedHash = computeNormalizedHash(input.content); + const insertValues = buildInsertMemoryValues( + input, + normalizedHash, + now, + hasMemoryImportanceColumn(db), + ); + const result = getInsertMemoryStatement(db).run(...insertValues) as { + lastInsertRowid?: number | bigint; + }; + const memory = loadInsertedMemory(db, result.lastInsertRowid); + recordMemoryEvidence(db, memory.id, input); + return getMemoryById(db, memory.id) ?? memory; + })(); invalidateProject(input.projectPath); return inserted; @@ -636,22 +716,34 @@ export function insertMemory(db: Database, input: MemoryInput): Memory { * surfacing a transient write failure. */ export function insertMemoryIdempotent(db: Database, input: MemoryInput): InsertMemoryResult { + const existing = getMemoryByHash( + db, + input.projectPath, + input.category, + computeNormalizedHash(input.content), + ); + if (existing) { + if (input.sourceSessionId && hasMemoryEvidenceTable(db)) { + recordMemoryEvidence(db, existing.id, input); + } else { + updateMemorySeenCount(db, existing.id); + } + return { memory: getMemoryById(db, existing.id) ?? existing, inserted: false }; + } try { - return { memory: insertMemory(db, input), inserted: true }; + const memory = insertMemory(db, input); + return { memory: getMemoryById(db, memory.id) ?? memory, inserted: true }; } catch (error) { - if (!isUniqueConstraintError(error)) { - throw error; - } + if (!isUniqueConstraintError(error)) throw error; const normalizedHash = computeNormalizedHash(input.content); - const existing = getMemoryByHash(db, input.projectPath, input.category, normalizedHash); - if (!existing) { - throw error; + const raced = getMemoryByHash(db, input.projectPath, input.category, normalizedHash); + if (!raced) throw error; + if (input.sourceSessionId && hasMemoryEvidenceTable(db)) { + recordMemoryEvidence(db, raced.id, input); + } else { + updateMemorySeenCount(db, raced.id); } - updateMemorySeenCount(db, existing.id); - return { - memory: getMemoryById(db, existing.id) ?? existing, - inserted: false, - }; + return { memory: getMemoryById(db, raced.id) ?? raced, inserted: false }; } } @@ -1142,8 +1234,26 @@ export function mergeMemoryStats( status: MemoryStatus, ): void { assertTsMemoryIdWriteAllowed(db, id); + let evidenceCount: number | null = null; + if (hasMemoryEvidenceTable(db)) { + db.prepare( + `INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence + WHERE memory_id IN (SELECT value FROM json_each(?))`, + ).run(id, mergedFrom); + const row = db + .prepare( + "SELECT COUNT(DISTINCT source_session_id) AS count FROM memory_evidence WHERE memory_id = ?", + ) + .get(id) as { count?: number } | undefined; + if (typeof row?.count === "number" && row.count > 0) evidenceCount = row.count; + } getMergeMemoryStatsStatement(db).run( seenCount, + evidenceCount ?? 0, retrievalCount, mergedFrom, status, diff --git a/packages/plugin/src/features/magic-context/memory/types.ts b/packages/plugin/src/features/magic-context/memory/types.ts index c68570b30..f4db95c66 100644 --- a/packages/plugin/src/features/magic-context/memory/types.ts +++ b/packages/plugin/src/features/magic-context/memory/types.ts @@ -22,6 +22,7 @@ export type MemoryCategory = export type MemoryStatus = "active" | "permanent" | "archived"; export type MemoryScope = "project" | "ecosystem" | "universe"; export type VerificationStatus = "unverified" | "verified" | "stale" | "flagged"; +/** `user` requires a host-verified user-message link. Tool calls default to `agent`. */ export type MemorySourceType = "historian" | "agent" | "dreamer" | "user"; export interface Memory { @@ -58,6 +59,9 @@ export interface MemoryInput { content: string; importance?: number | null; sourceSessionId?: string; + /** Host-native message that owns the observation. OpenCode tool calls expose + * the assistant tool-owner message; runtimes without that link leave this absent. */ + sourceMessageId?: string; sourceType?: MemorySourceType; expiresAt?: number | null; metadataJson?: string | null; diff --git a/packages/plugin/src/features/magic-context/migrations-v74.test.ts b/packages/plugin/src/features/magic-context/migrations-v74.test.ts index 7f4d6cfed..ed2007200 100644 --- a/packages/plugin/src/features/magic-context/migrations-v74.test.ts +++ b/packages/plugin/src/features/magic-context/migrations-v74.test.ts @@ -36,7 +36,7 @@ describe("migration v74: detected context-limit provenance", () => { runMigrations(db); expect(columnNames(db, "session_meta")).toContain("detected_context_limit_provenance"); - expect(LATEST_SUPPORTED_VERSION).toBe(78); + expect(LATEST_SUPPORTED_VERSION).toBe(79); expect(LATEST_SUPPORTED_VERSION).toBe(LATEST_MIGRATION_VERSION); } finally { closeQuietly(db); diff --git a/packages/plugin/src/features/magic-context/migrations-v76.test.ts b/packages/plugin/src/features/magic-context/migrations-v76.test.ts index 11ddb4723..51fa7c0a0 100644 --- a/packages/plugin/src/features/magic-context/migrations-v76.test.ts +++ b/packages/plugin/src/features/magic-context/migrations-v76.test.ts @@ -43,7 +43,7 @@ describe("migration v76: retina condition compilation", () => { "compile_status", ]), ); - expect(LATEST_SUPPORTED_VERSION).toBe(78); + expect(LATEST_SUPPORTED_VERSION).toBe(79); expect(LATEST_SUPPORTED_VERSION).toBe(LATEST_MIGRATION_VERSION); expect(() => db diff --git a/packages/plugin/src/features/magic-context/migrations-v77.test.ts b/packages/plugin/src/features/magic-context/migrations-v77.test.ts index 5a89dc0da..a612fb6e0 100644 --- a/packages/plugin/src/features/magic-context/migrations-v77.test.ts +++ b/packages/plugin/src/features/magic-context/migrations-v77.test.ts @@ -37,7 +37,7 @@ describe("migration v77: durable candidate provenance", () => { expect(columnNames(db, "user_memories")).toContain("source_candidate_provenance"); expect(columnNames(db, "primers")).toContain("source_candidate_provenance"); - expect(LATEST_SUPPORTED_VERSION).toBe(78); + expect(LATEST_SUPPORTED_VERSION).toBe(79); expect(LATEST_SUPPORTED_VERSION).toBe(LATEST_MIGRATION_VERSION); } finally { closeQuietly(db); diff --git a/packages/plugin/src/features/magic-context/migrations-v78.test.ts b/packages/plugin/src/features/magic-context/migrations-v78.test.ts index 849d973de..9b73d0404 100644 --- a/packages/plugin/src/features/magic-context/migrations-v78.test.ts +++ b/packages/plugin/src/features/magic-context/migrations-v78.test.ts @@ -46,7 +46,7 @@ describe("migration v78: migration_pending journal", () => { "phase", "created_at", ]); - expect(LATEST_SUPPORTED_VERSION).toBe(78); + expect(LATEST_SUPPORTED_VERSION).toBe(79); expect(LATEST_SUPPORTED_VERSION).toBe(LATEST_MIGRATION_VERSION); } finally { closeQuietly(db); diff --git a/packages/plugin/src/features/magic-context/migrations-v79.test.ts b/packages/plugin/src/features/magic-context/migrations-v79.test.ts new file mode 100644 index 000000000..1c263d9e4 --- /dev/null +++ b/packages/plugin/src/features/magic-context/migrations-v79.test.ts @@ -0,0 +1,49 @@ +/// + +import { describe, expect, it } from "bun:test"; +import { Database } from "../../shared/sqlite"; +import { closeQuietly } from "../../shared/sqlite-helpers"; +import { runMigrations } from "./migrations"; +import { initializeDatabase, LATEST_SUPPORTED_VERSION } from "./storage-db"; + +describe("migration v79: memory evidence", () => { + it("creates the evidence identity and backfills known source sessions", () => { + const db = new Database(":memory:"); + try { + initializeDatabase(db); + db.exec(` + INSERT INTO memories ( + project_path, category, content, normalized_hash, + source_session_id, source_type, first_seen_at, + created_at, updated_at, last_seen_at + ) VALUES ( + 'git:project', 'CONSTRAINTS', 'Fact', 'hash', + 'session-a', 'user', 11, 11, 11, 11 + ); + `); + + runMigrations(db); + + expect( + db + .prepare( + `SELECT memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence`, + ) + .all(), + ).toEqual([ + { + memory_id: 1, + content_hash: "hash", + source_session_id: "session-a", + source_message_id: null, + source_type: "user", + observed_at: 11, + }, + ]); + expect(LATEST_SUPPORTED_VERSION).toBe(79); + } finally { + closeQuietly(db); + } + }); +}); diff --git a/packages/plugin/src/features/magic-context/migrations.ts b/packages/plugin/src/features/magic-context/migrations.ts index a222fc06c..cceca7da2 100644 --- a/packages/plugin/src/features/magic-context/migrations.ts +++ b/packages/plugin/src/features/magic-context/migrations.ts @@ -2818,6 +2818,32 @@ export const MIGRATIONS: Migration[] = [ `); }, }, + { + version: 79, + description: "preserve content-bound per-session memory evidence provenance", + up(db: Database): void { + if (!tableExists(db, "memories")) return; + db.exec(` + CREATE TABLE IF NOT EXISTS memory_evidence ( + memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + content_hash TEXT NOT NULL, + source_session_id TEXT NOT NULL, + source_message_id TEXT, + source_type TEXT NOT NULL, + observed_at INTEGER NOT NULL, + PRIMARY KEY(memory_id, content_hash, source_session_id) + ); + CREATE INDEX IF NOT EXISTS idx_memory_evidence_session + ON memory_evidence(source_session_id, memory_id); + INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT id, normalized_hash, source_session_id, NULL, COALESCE(source_type, 'historian'), first_seen_at + FROM memories + WHERE source_session_id IS NOT NULL; + `); + }, + }, ]; /** diff --git a/packages/plugin/src/features/magic-context/storage-db.ts b/packages/plugin/src/features/magic-context/storage-db.ts index c0e99fc9d..c653c0c77 100644 --- a/packages/plugin/src/features/magic-context/storage-db.ts +++ b/packages/plugin/src/features/magic-context/storage-db.ts @@ -93,7 +93,7 @@ export function __resetSchemaFenceStateForTests(): void { lastMigrationOnOpenRefusal = null; } -export const LATEST_SUPPORTED_VERSION = 78; +export const LATEST_SUPPORTED_VERSION = 79; // chmod is meaningless on Windows (POSIX modes are not honored), so all // permission tightening is skipped there. mkdir's `mode` is likewise ignored. @@ -1067,6 +1067,18 @@ export function initializeDatabase(db: Database): void { UNIQUE(project_path, category, normalized_hash) ); + CREATE TABLE IF NOT EXISTS memory_evidence ( + memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + content_hash TEXT NOT NULL, + source_session_id TEXT NOT NULL, + source_message_id TEXT, + source_type TEXT NOT NULL, + observed_at INTEGER NOT NULL, + PRIMARY KEY(memory_id, content_hash, source_session_id) + ); + CREATE INDEX IF NOT EXISTS idx_memory_evidence_session + ON memory_evidence(source_session_id, memory_id); + CREATE TABLE IF NOT EXISTS memory_embeddings ( -- FK-cascade audit (v12): memory_embeddings.memory_id -> memories.id -- uses ON DELETE CASCADE, so SQLite PRAGMA foreign_keys must be ON on diff --git a/packages/plugin/src/features/magic-context/storage-identity-merge.test.ts b/packages/plugin/src/features/magic-context/storage-identity-merge.test.ts index 75126949a..6f528aa08 100644 --- a/packages/plugin/src/features/magic-context/storage-identity-merge.test.ts +++ b/packages/plugin/src/features/magic-context/storage-identity-merge.test.ts @@ -60,6 +60,14 @@ describe("project identity merge", () => { const database = makeDb(); const sourceId = insertMemory(database, "dir:old", "legacy", "same-hash"); const targetId = insertMemory(database, "git:new", "canonical", "same-hash"); + database + .prepare( + `INSERT INTO memory_evidence + (memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at) + VALUES (?, 'same-hash', 'session-source', 'assistant-source', 'agent', 1), + (?, 'same-hash', 'session-target', 'assistant-target', 'agent', 2)`, + ) + .run(sourceId, targetId); database .prepare("INSERT INTO project_state(project_path, project_memory_epoch) VALUES (?, 4)") .run("git:new"); @@ -119,6 +127,16 @@ describe("project identity merge", () => { expect(database.prepare("SELECT COUNT(*) AS count FROM identity_merge_log").get()).toEqual({ count: report.changedRows, }); + expect( + database + .prepare( + "SELECT source_session_id FROM memory_evidence WHERE memory_id = ? ORDER BY source_session_id", + ) + .all(targetId), + ).toEqual([ + { source_session_id: "session-source" }, + { source_session_id: "session-target" }, + ]); }); test("preserves the oldest open broad cycle when task schedule rows collide", async () => { diff --git a/packages/plugin/src/features/magic-context/storage-identity-merge.ts b/packages/plugin/src/features/magic-context/storage-identity-merge.ts index 415227ad3..7c0ee64d5 100644 --- a/packages/plugin/src/features/magic-context/storage-identity-merge.ts +++ b/packages/plugin/src/features/magic-context/storage-identity-merge.ts @@ -201,7 +201,23 @@ function mergeMemoryRow( .get(toIdentity, row.category, row.normalized_hash, sourceId) as SqliteRow | undefined; if (collision && typeof collision.id === "number") { const targetId = collision.id; - const mergedSeen = Math.max(Number(collision.seen_count ?? 1), Number(row.seen_count ?? 1)); + db.prepare( + `INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence WHERE memory_id = ?`, + ).run(targetId, sourceId); + const evidenceRow = db + .prepare( + "SELECT COUNT(DISTINCT source_session_id) AS count FROM memory_evidence WHERE memory_id = ?", + ) + .get(targetId) as { count?: number } | undefined; + const mergedSeen = Math.max( + Number(collision.seen_count ?? 1), + Number(row.seen_count ?? 1), + evidenceRow?.count ?? 0, + ); const sourceClassifiedAt = Number(row.classified_at ?? 0); const targetClassifiedAt = Number(collision.classified_at ?? 0); if (sourceClassifiedAt > targetClassifiedAt) { diff --git a/packages/plugin/src/features/magic-context/v22-deferred-backfill.test.ts b/packages/plugin/src/features/magic-context/v22-deferred-backfill.test.ts index 4c7a44f66..60026a70a 100644 --- a/packages/plugin/src/features/magic-context/v22-deferred-backfill.test.ts +++ b/packages/plugin/src/features/magic-context/v22-deferred-backfill.test.ts @@ -150,6 +150,14 @@ describe("runDeferredV22Backfill", () => { const database = makeDb(); const firstId = insertMemory(database, "/proj/canonical", "dup-hash"); const secondId = insertMemory(database, "/proj/symlinked", "dup-hash"); + database + .prepare( + `INSERT INTO memory_evidence + (memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at) + VALUES (?, 'dup-hash', 'session-a', NULL, 'historian', 1), + (?, 'dup-hash', 'session-b', NULL, 'historian', 2)`, + ) + .run(firstId, secondId); // Give the second (later) row a higher seen_count to verify merge keeps max. database.prepare("UPDATE memories SET seen_count = 9 WHERE id = ?").run(secondId); @@ -170,6 +178,13 @@ describe("runDeferredV22Backfill", () => { // The earlier row survives (UPDATE'd first); seen_count merged to the max (9). expect(survivors[0].id).toBe(firstId); expect(survivors[0].seen_count).toBe(9); + expect( + database + .prepare( + "SELECT source_session_id FROM memory_evidence WHERE memory_id = ? ORDER BY source_session_id", + ) + .all(firstId), + ).toEqual([{ source_session_id: "session-a" }, { source_session_id: "session-b" }]); // No legacy rows remain. const unresolved = database .prepare( diff --git a/packages/plugin/src/features/magic-context/v22-deferred-backfill.ts b/packages/plugin/src/features/magic-context/v22-deferred-backfill.ts index 43068eba0..36e3ac32b 100644 --- a/packages/plugin/src/features/magic-context/v22-deferred-backfill.ts +++ b/packages/plugin/src/features/magic-context/v22-deferred-backfill.ts @@ -308,6 +308,16 @@ export async function runDeferredV22Backfill( `INSERT OR IGNORE INTO memory_embeddings (memory_id, embedding, model_id) SELECT ?, embedding, model_id FROM memory_embeddings WHERE memory_id = ?`, ); + const preserveEvidence = db.prepare( + `INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence WHERE memory_id = ?`, + ); + const evidenceCount = db.prepare( + "SELECT COUNT(DISTINCT source_session_id) AS count FROM memory_evidence WHERE memory_id = ?", + ); const deleteMemoryRow = db.prepare("DELETE FROM memories WHERE id = ?"); for (const row of resolvedRows) { @@ -321,7 +331,13 @@ export async function runDeferredV22Backfill( // delete the source legacy row. The embedding row FK-cascades on // delete. The mutation log is unaffected (no render-visible // change — both rows held identical content). - const mergedSeen = Math.max(collision.seen_count ?? 1, row.seen_count ?? 1); + preserveEvidence.run(collision.id, row.id); + const count = evidenceCount.get(collision.id) as { count?: number } | undefined; + const mergedSeen = Math.max( + collision.seen_count ?? 1, + row.seen_count ?? 1, + count?.count ?? 0, + ); if (mergedSeen !== (collision.seen_count ?? 1)) { bumpSeenCount.run(mergedSeen, collision.id); } diff --git a/packages/plugin/src/hooks/magic-context/module-state-sync.ts b/packages/plugin/src/hooks/magic-context/module-state-sync.ts index ad02dc8e4..0400ce999 100644 --- a/packages/plugin/src/hooks/magic-context/module-state-sync.ts +++ b/packages/plugin/src/hooks/magic-context/module-state-sync.ts @@ -6,6 +6,7 @@ import { getMaxMemoryIdForProjects, getMemoriesByProject, getMemoriesByProjects, + getMemoryEvidence, readNewMemoriesForM1Union, } from "../../features/magic-context/memory/storage-memory"; import type { ContextDatabase } from "../../features/magic-context/storage"; @@ -1485,6 +1486,7 @@ export async function buildModuleStateSyncPayload(args: { superseded_by_memory_id: memory.supersededByMemoryId, merged_from: memory.mergedFrom, metadata_json: memory.metadataJson, + evidence: getMemoryEvidence(args.pass.db, memory.id), })); const renderedMemoryIds = memoryMutationsChanged ? args.force diff --git a/packages/plugin/src/hooks/magic-context/rust-mode-transform.ts b/packages/plugin/src/hooks/magic-context/rust-mode-transform.ts index c955234bd..2838de55f 100644 --- a/packages/plugin/src/hooks/magic-context/rust-mode-transform.ts +++ b/packages/plugin/src/hooks/magic-context/rust-mode-transform.ts @@ -902,12 +902,28 @@ function authoritySeedRows( return memoryRows.map((snapshot) => { const id = Number(snapshot.id); const mapping = mappings.get(id); + const evidence = + domain === "memories" + ? db + .prepare( + `SELECT content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence WHERE memory_id = ? + ORDER BY content_hash, source_session_id`, + ) + .all(id) + : undefined; const seededSnapshot = domain === "memories" && mapping - ? { ...snapshot, mapping: mapping.hasSentinel ? null : mapping.files } - : domain === "notes" && snapshot.project_path == null - ? { ...snapshot, project_path: projectPath } - : snapshot; + ? { + ...snapshot, + evidence, + mapping: mapping.hasSentinel ? null : mapping.files, + } + : domain === "memories" + ? { ...snapshot, evidence } + : domain === "notes" && snapshot.project_path == null + ? { ...snapshot, project_path: projectPath } + : snapshot; return { source_row_id: snapshot.id, snapshot: seededSnapshot }; }); } diff --git a/packages/plugin/src/tools/ctx-memory/tools.test.ts b/packages/plugin/src/tools/ctx-memory/tools.test.ts index db461aaee..68c03cf18 100644 --- a/packages/plugin/src/tools/ctx-memory/tools.test.ts +++ b/packages/plugin/src/tools/ctx-memory/tools.test.ts @@ -79,6 +79,17 @@ function createTestDb(dbPath = ":memory:"): Database { PRIMARY KEY (memory_id, model_id) ); + CREATE TABLE IF NOT EXISTS memory_evidence + ( + memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE, + content_hash TEXT NOT NULL, + source_session_id TEXT NOT NULL, + source_message_id TEXT, + source_type TEXT NOT NULL, + observed_at INTEGER NOT NULL, + PRIMARY KEY (memory_id, content_hash, source_session_id) + ); + CREATE TABLE IF NOT EXISTS embedding_identity_active ( project_path TEXT NOT NULL, scope TEXT NOT NULL, @@ -214,7 +225,12 @@ function createTestDb(dbPath = ":memory:"): Database { } const toolContext = (sessionID = "ses-memory", agent = "general") => - ({ sessionID, agent, directory: "/repo/project" }) as never; + ({ + sessionID, + messageID: "msg-assistant-tool-owner", + agent, + directory: "/repo/project", + }) as never; const dreamerToolContext = (directory: string) => ({ sessionID: "ses-dream", agent: DREAMER_AGENT, directory }) as never; @@ -489,7 +505,7 @@ describe("createCtxMemoryTools", () => { expect(getMemoriesByProject(db, "/repo/project")).toHaveLength(0); }); - it("creates a new memory with agent source type", async () => { + it("attributes primary writes to the agent-owned tool-call message", async () => { const result = await tools.ctx_memory.execute( { action: "write", @@ -506,6 +522,13 @@ describe("createCtxMemoryTools", () => { expect(memories[0]?.sourceType).toBe("agent"); expect(memories[0]?.sourceSessionId).toBe("ses-memory"); expect(memories[0]?.category).toBe("USER_DIRECTIVES"); + expect( + db + .prepare( + "SELECT source_message_id, source_type FROM memory_evidence WHERE memory_id = ?", + ) + .get(memories[0]?.id), + ).toEqual({ source_message_id: "msg-assistant-tool-owner", source_type: "agent" }); }); it("does not bump project memory epoch for additive writes", async () => { diff --git a/packages/plugin/src/tools/ctx-memory/tools.ts b/packages/plugin/src/tools/ctx-memory/tools.ts index d1b571f7e..480826a8d 100644 --- a/packages/plugin/src/tools/ctx-memory/tools.ts +++ b/packages/plugin/src/tools/ctx-memory/tools.ts @@ -15,7 +15,6 @@ import { mergeMemoryStats, saveEmbeddingIfHashMatches, supersededMemory, - updateMemorySeenCount, V2_MEMORY_CATEGORIES, } from "../../features/magic-context/memory"; import { @@ -28,6 +27,7 @@ import { computeNormalizedHash } from "../../features/magic-context/memory/norma import { hasMemoryClassifiedAtColumn, hasMemoryShareableColumn, + recordMemoryEvidence, } from "../../features/magic-context/memory/storage-memory"; import { normalizeStoredProjectPath, @@ -554,28 +554,17 @@ function createCtxMemoryTool(deps: CtxMemoryToolDeps): ToolDefinition { return `Error: Unknown memory category '${rawCategory}'.`; } - const existingMemory = getMemoryByHash( - deps.db, - projectPath, - category, - computeNormalizedHash(content), - ); - if (existingMemory) { - updateMemorySeenCount(deps.db, existingMemory.id); - requestRustMemorySync(deps, toolContext.sessionID); - return `Memory already exists [ID: ${existingMemory.id}] in ${category} (seen count incremented).`; - } - const insertResult = insertMemoryIdempotent(deps.db, { projectPath: projectPath, category, content, sourceSessionId: toolContext.sessionID, + sourceMessageId: toolContext.messageID, sourceType: toolContext.agent === DREAMER_AGENT ? "dreamer" : getSourceType(deps), }); if (!insertResult.inserted) { - return `Memory already exists [ID: ${insertResult.memory.id}] in ${category} (seen count incremented).`; + return `Memory already exists [ID: ${insertResult.memory.id}] in ${category}.`; } queueMemoryEmbedding({ @@ -676,6 +665,15 @@ function createCtxMemoryTool(deps: CtxMemoryToolDeps): ToolDefinition { category: memory.category, newContent: content, }); + recordMemoryEvidence(deps.db, memory.id, { + projectPath, + category: memory.category, + content, + sourceSessionId: toolContext.sessionID, + sourceMessageId: toolContext.messageID, + sourceType: + toolContext.agent === DREAMER_AGENT ? "dreamer" : getSourceType(deps), + }); }); queueMemoryEmbedding({ deps, @@ -821,6 +819,7 @@ function createCtxMemoryTool(deps: CtxMemoryToolDeps): ToolDefinition { category, content, sourceSessionId: toolContext.sessionID, + sourceMessageId: toolContext.messageID, sourceType: toolContext.agent === DREAMER_AGENT ? "dreamer" @@ -847,6 +846,15 @@ function createCtxMemoryTool(deps: CtxMemoryToolDeps): ToolDefinition { mergedFrom, mergedStatus, ); + recordMemoryEvidence(deps.db, nextCanonical.id, { + projectPath, + category, + content, + sourceSessionId: toolContext.sessionID, + sourceMessageId: toolContext.messageID, + sourceType: + toolContext.agent === DREAMER_AGENT ? "dreamer" : getSourceType(deps), + }); for (const memory of sourceMemories) { if (memory.id === nextCanonical.id) { From 381623a6df1981b294563c482d6da45821ff8186 Mon Sep 17 00:00:00 2001 From: Cole Leavitt Date: Wed, 19 Aug 2026 04:35:59 -0400 Subject: [PATCH 2/7] fix(memory): harden evidence migration replay --- .../magic-context/migrations-armed-replay.test.ts | 1 + .../src/features/magic-context/migrations.ts | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/packages/plugin/src/features/magic-context/migrations-armed-replay.test.ts b/packages/plugin/src/features/magic-context/migrations-armed-replay.test.ts index a3f18d80b..3d17ee72d 100644 --- a/packages/plugin/src/features/magic-context/migrations-armed-replay.test.ts +++ b/packages/plugin/src/features/magic-context/migrations-armed-replay.test.ts @@ -364,6 +364,7 @@ function populateForVersion(db: DatabaseType, version: number, state: ReplayStat case 76: case 77: case 78: + case 79: if (!state.armed) throw new Error(`migration v${version} reached an unarmed store`); populateModuleOwnedRows(db, version, state); return; diff --git a/packages/plugin/src/features/magic-context/migrations.ts b/packages/plugin/src/features/magic-context/migrations.ts index cceca7da2..b96299e62 100644 --- a/packages/plugin/src/features/magic-context/migrations.ts +++ b/packages/plugin/src/features/magic-context/migrations.ts @@ -2823,6 +2823,11 @@ export const MIGRATIONS: Migration[] = [ description: "preserve content-bound per-session memory evidence provenance", up(db: Database): void { if (!tableExists(db, "memories")) return; + const memoryColumns = new Set( + (db.prepare("PRAGMA table_info(memories)").all() as Array<{ name: string }>).map( + (row) => row.name, + ), + ); db.exec(` CREATE TABLE IF NOT EXISTS memory_evidence ( memory_id INTEGER NOT NULL REFERENCES memories(id) ON DELETE CASCADE, @@ -2835,6 +2840,15 @@ export const MIGRATIONS: Migration[] = [ ); CREATE INDEX IF NOT EXISTS idx_memory_evidence_session ON memory_evidence(source_session_id, memory_id); + `); + if ( + ["id", "normalized_hash", "source_session_id", "source_type", "first_seen_at"].some( + (column) => !memoryColumns.has(column), + ) + ) { + return; + } + db.exec(` INSERT OR IGNORE INTO memory_evidence ( memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at ) From a881706dc8e0e890fbea0963fdc0bff610c45a99 Mon Sep 17 00:00:00 2001 From: Cole Leavitt Date: Wed, 19 Aug 2026 04:41:34 -0400 Subject: [PATCH 3/7] fix(memory): sync evidence-only updates --- packages/plugin/src/tools/ctx-memory/tools.test.ts | 13 ++++++++++++- packages/plugin/src/tools/ctx-memory/tools.ts | 1 + 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/packages/plugin/src/tools/ctx-memory/tools.test.ts b/packages/plugin/src/tools/ctx-memory/tools.test.ts index 68c03cf18..755a079c9 100644 --- a/packages/plugin/src/tools/ctx-memory/tools.test.ts +++ b/packages/plugin/src/tools/ctx-memory/tools.test.ts @@ -384,6 +384,17 @@ describe("createCtxMemoryTools", () => { expect(getMemoriesByProject(db, "/repo/project")).toHaveLength(1); expect(syncSessions).toEqual(["ses-memory"]); + const duplicate = await rustTools.ctx_memory.execute( + { + action: "write", + category: "USER_DIRECTIVES", + content: "Keep the context database authoritative.", + }, + toolContext("ses-second"), + ); + expect(duplicate).toContain("Memory already exists"); + expect(syncSessions).toEqual(["ses-memory", "ses-second"]); + const tsTools = createCtxMemoryTools({ db, resolveProjectPath: () => "/repo/project", @@ -398,7 +409,7 @@ describe("createCtxMemoryTools", () => { }, toolContext(), ); - expect(syncSessions).toEqual(["ses-memory"]); + expect(syncSessions).toEqual(["ses-memory", "ses-second"]); }); it("routes all module-owned memory actions without writing the TS table", async () => { diff --git a/packages/plugin/src/tools/ctx-memory/tools.ts b/packages/plugin/src/tools/ctx-memory/tools.ts index 480826a8d..7883f6321 100644 --- a/packages/plugin/src/tools/ctx-memory/tools.ts +++ b/packages/plugin/src/tools/ctx-memory/tools.ts @@ -564,6 +564,7 @@ function createCtxMemoryTool(deps: CtxMemoryToolDeps): ToolDefinition { toolContext.agent === DREAMER_AGENT ? "dreamer" : getSourceType(deps), }); if (!insertResult.inserted) { + requestRustMemorySync(deps, toolContext.sessionID); return `Memory already exists [ID: ${insertResult.memory.id}] in ${category}.`; } From 9c6b18acff401eab410b021a2d9892aa49fed0a4 Mon Sep 17 00:00:00 2001 From: Cole Leavitt Date: Wed, 19 Aug 2026 06:51:28 -0400 Subject: [PATCH 4/7] fix(memory): preserve episode evidence invariants --- crates/mc-module/src/lib.rs | 25 +- crates/mc-store/src/lib.rs | 300 +++++++++++++++--- .../pi-plugin/src/tools/ctx-memory.test.ts | 38 +++ .../magic-context/context-authority.test.ts | 48 +++ .../magic-context/memory/relocate-memory.ts | 17 +- .../memory/storage-memory-evidence.test.ts | 29 +- .../magic-context/memory/storage-memory.ts | 100 ++++-- .../magic-context/storage-identity-merge.ts | 21 +- 8 files changed, 469 insertions(+), 109 deletions(-) diff --git a/crates/mc-module/src/lib.rs b/crates/mc-module/src/lib.rs index ba85575bb..b51343d3b 100644 --- a/crates/mc-module/src/lib.rs +++ b/crates/mc-module/src/lib.rs @@ -1699,7 +1699,7 @@ struct ModuleMemoryWire { #[serde(default)] mural_cue_rejection_count: i64, #[serde(default)] - evidence: Vec, + evidence: Option>, } #[derive(Debug, Clone, Deserialize)] @@ -1817,17 +1817,18 @@ impl ModuleMemoryWire { mural_cue_hash: self.mural_cue_hash, mural_cue_at: self.mural_cue_at, mural_cue_rejection_count: self.mural_cue_rejection_count, - evidence: self - .evidence - .into_iter() - .map(|row| ModuleMemoryEvidenceRow { - content_hash: row.content_hash, - source_session_id: row.source_session_id, - source_message_id: row.source_message_id, - source_type: row.source_type, - observed_at: row.observed_at, - }) - .collect(), + evidence: self.evidence.map(|evidence| { + evidence + .into_iter() + .map(|row| ModuleMemoryEvidenceRow { + content_hash: row.content_hash, + source_session_id: row.source_session_id, + source_message_id: row.source_message_id, + source_type: row.source_type, + observed_at: row.observed_at, + }) + .collect() + }), } } } diff --git a/crates/mc-store/src/lib.rs b/crates/mc-store/src/lib.rs index 5b9a5a51a..0da2b875e 100644 --- a/crates/mc-store/src/lib.rs +++ b/crates/mc-store/src/lib.rs @@ -2483,7 +2483,12 @@ fn normalize_authority_route_tx( rows }; for (source_id, canonical_id) in collisions { - merge_memory_evidence_tx(tx, canonical_id, &[source_id])?; + let legacy_seen_baseline = legacy_seen_baseline_tx(tx, &[canonical_id, source_id])?; + let evidence_count = merge_memory_evidence_tx(tx, canonical_id, &[source_id])?; + tx.execute( + "UPDATE mc_memories SET seen_count = ?1 WHERE id = ?2", + params![legacy_seen_baseline + evidence_count, canonical_id], + )?; } tx.execute( "DELETE FROM mc_memories @@ -2503,15 +2508,6 @@ fn normalize_authority_route_tx( )", params![context_store_uuid, project, route_project_root], )?; - tx.execute( - "UPDATE mc_memories - SET seen_count = MAX( - seen_count, - (SELECT COUNT(DISTINCT source_session_id) FROM mc_memory_evidence WHERE memory_id = mc_memories.id) - ) - WHERE project_path = ?1", - [project], - )?; tx.execute( "UPDATE mc_memories SET project_path = ?2 @@ -4569,7 +4565,9 @@ pub struct ModuleMemoryRow { pub mural_cue_hash: Option, pub mural_cue_at: Option, pub mural_cue_rejection_count: i64, - pub evidence: Vec, + /// `None` means an older sparse snapshot omitted evidence; `Some([])` is an + /// authoritative clear. + pub evidence: Option>, } #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -5405,10 +5403,17 @@ fn record_memory_evidence_tx( let Some(source_session_id) = input.source_session_id else { return Ok(false); }; - let content_hash: String = tx.query_row( - "SELECT normalized_hash FROM mc_memories WHERE id = ?1", + let (content_hash, original_source_session_id, prior_evidence_count): ( + String, + Option, + i64, + ) = tx.query_row( + "SELECT normalized_hash, source_session_id, + (SELECT COUNT(DISTINCT source_session_id) + FROM mc_memory_evidence WHERE memory_id = ?1) + FROM mc_memories WHERE id = ?1", [memory_id], - |row| row.get(0), + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), )?; let feed_seq_before = tx.query_row( "SELECT COALESCE(MAX(feed_seq), 0) FROM mc_changefeed", @@ -5429,16 +5434,21 @@ fn record_memory_evidence_tx( ], )? > 0; if inserted { + let evidenced_seen_count = if prior_evidence_count > 0 { + prior_evidence_count + } else if original_source_session_id.as_deref() == Some(source_session_id) { + 1 + } else { + 0 + }; tx.execute( "UPDATE mc_memories - SET seen_count = MAX( - COALESCE(seen_count, 1), - (SELECT COUNT(DISTINCT source_session_id) FROM mc_memory_evidence WHERE memory_id = ?1) - ), - last_seen_at = ?2, - updated_at = ?2 + SET seen_count = MAX(COALESCE(seen_count, 1) - ?2, 0) + + (SELECT COUNT(DISTINCT source_session_id) FROM mc_memory_evidence WHERE memory_id = ?1), + last_seen_at = ?3, + updated_at = ?3 WHERE id = ?1", - params![memory_id, input.now_ms], + params![memory_id, evidenced_seen_count, input.now_ms], )?; if let Some(memory) = load_memory_full_tx(tx, memory_id)? { emit_verification_memory_snapshot_tx(tx, &memory, feed_seq_before)?; @@ -5468,6 +5478,28 @@ fn merge_memory_evidence_tx( ) } +fn legacy_seen_baseline_tx( + tx: &rusqlite::Transaction<'_>, + memory_ids: &[i64], +) -> rusqlite::Result { + let mut baseline = 0; + for memory_id in memory_ids { + baseline += tx.query_row( + "SELECT MAX( + COALESCE(seen_count, 1) - ( + SELECT COUNT(DISTINCT source_session_id) + FROM mc_memory_evidence WHERE memory_id = ?1 + ), + 0 + ) + FROM mc_memories WHERE id = ?1", + [memory_id], + |row| row.get::<_, i64>(0), + )?; + } + Ok(baseline) +} + /// Transaction-scoped ports used by the module facade. Every method operates on the transaction /// owned by `with_facade_command`, so the mutation and its response ledger row commit together. pub struct FacadeMutationTxn<'a> { @@ -5741,6 +5773,9 @@ impl<'a> FacadeMutationTxn<'a> { |row| row.get::<_, i64>(0), ) .map_err(|error| error.to_string())?; + let affected_ids = affected.iter().map(|memory| memory.id).collect::>(); + let legacy_seen_baseline = + legacy_seen_baseline_tx(self.tx, &affected_ids).map_err(|error| error.to_string())?; let evidence_count = merge_memory_evidence_tx( self.tx, target_id, @@ -5750,8 +5785,7 @@ impl<'a> FacadeMutationTxn<'a> { .collect::>(), ) .map_err(|error| error.to_string())?; - let prior_seen_count: i64 = affected.iter().map(|memory| memory.seen_count.max(0)).sum(); - let seen_count = prior_seen_count.max(evidence_count); + let seen_count = legacy_seen_baseline + evidence_count; let retrieval_count: i64 = affected .iter() .map(|memory| memory.retrieval_count.max(0)) @@ -11495,6 +11529,8 @@ impl McStore { [], |row| row.get::<_, i64>(0), )?; + let affected_ids = affected.iter().map(|memory| memory.id).collect::>(); + let legacy_seen_baseline = legacy_seen_baseline_tx(tx, &affected_ids)?; let evidence_count = merge_memory_evidence_tx( tx, target_id, @@ -11503,9 +11539,7 @@ impl McStore { .map(|memory| memory.id) .collect::>(), )?; - let prior_seen_count: i64 = - affected.iter().map(|memory| memory.seen_count.max(0)).sum(); - let seen_count = prior_seen_count.max(evidence_count); + let seen_count = legacy_seen_baseline + evidence_count; let retrieval_count: i64 = affected .iter() .map(|memory| memory.retrieval_count.max(0)) @@ -15136,18 +15170,23 @@ impl McStore { .expect("every natural-key survivor was seeded") }) .collect::>(); - for module_row_id in module_row_ids.iter().copied().collect::>() { + let evidence_authoritative_ids = rows + .iter() + .zip(&module_row_ids) + .filter_map(|(row, module_row_id)| { + row.snapshot.get("evidence").map(|_| *module_row_id) + }) + .collect::>(); + for module_row_id in evidence_authoritative_ids { tx.execute( "DELETE FROM mc_memory_evidence WHERE memory_id = ?1", [module_row_id], )?; } for (row, module_row_id) in rows.iter().zip(&module_row_ids) { - let evidence = row - .snapshot - .get("evidence") - .cloned() - .unwrap_or_else(|| Value::Array(Vec::new())); + let Some(evidence) = row.snapshot.get("evidence").cloned() else { + continue; + }; let evidence: Vec = serde_json::from_value(evidence) .map_err(|error| { rusqlite::Error::ToSqlConversionFailure(Box::new(error)) @@ -15653,7 +15692,9 @@ fn replace_authority_memories_tx( memory.mural_cue_rejection_count, ], )?; - replace_memory_evidence_tx(tx, existing_id, &memory.evidence)?; + if let Some(evidence) = &memory.evidence { + replace_memory_evidence_tx(tx, existing_id, evidence)?; + } continue; } tx.execute( @@ -15757,7 +15798,9 @@ fn replace_authority_memories_tx( params![&memory.project_path, &memory.category, &memory.normalized_hash], |row| row.get::<_, i64>(0), )?; - replace_memory_evidence_tx(tx, stored_id, &memory.evidence)?; + if let Some(evidence) = &memory.evidence { + replace_memory_evidence_tx(tx, stored_id, evidence)?; + } } Ok(()) } @@ -20085,6 +20128,72 @@ mod tests { assert_eq!(count, 1); } + #[test] + fn authority_seed_distinguishes_absent_evidence_from_an_explicit_clear() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let snapshot = serde_json::json!({ + "id": 100, + "project_path": "git:project", + "category": "CONSTRAINTS", + "content": "seeded fact", + "normalized_hash": "seeded-hash", + "updated_at": 100, + "status": "active", + "evidence": [{ + "content_hash": "seeded-hash", + "source_session_id": "session-a", + "source_message_id": "assistant-a1", + "source_type": "agent", + "observed_at": 11 + }] + }); + let seed = |snapshot: Value| { + store + .seed_authority_rows( + "current-store", + "git:project", + "memories", + &[AuthoritySeedRow { + source_row_id: 100, + snapshot, + }], + ) + .unwrap() + }; + + let ids = seed(snapshot.clone()); + let mut sparse = snapshot.clone(); + sparse.as_object_mut().unwrap().remove("evidence"); + seed(sparse); + let count = store + .inner + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM mc_memory_evidence WHERE memory_id = ?1", + [ids[0]], + |row| row.get::<_, i64>(0), + ) + }) + .unwrap(); + assert_eq!(count, 1); + + let mut clear = snapshot; + clear["evidence"] = Value::Array(Vec::new()); + seed(clear); + let count = store + .inner + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM mc_memory_evidence WHERE memory_id = ?1", + [ids[0]], + |row| row.get::<_, i64>(0), + ) + }) + .unwrap(); + assert_eq!(count, 0); + } + #[test] fn project_mural_artifact_upsert_is_hash_gated() { let dir = tempfile::tempdir().unwrap(); @@ -20575,6 +20684,59 @@ mod tests { assert_eq!((seen_count, evidence_count), (2, 2)); } + #[test] + fn memory_evidence_advances_a_migrated_legacy_baseline_for_a_new_session() { + let dir = tempfile::tempdir().unwrap(); + let store = McStore::open(&descriptor(dir.path())).unwrap(); + let save = |session_id: &str, now_ms: i64| { + store + .insert_memory(InsertMemoryInput { + project_path: "git:project", + route_project_root: None, + category: "CONSTRAINTS", + content: "Migrated fact", + source_session_id: Some(session_id), + source_message_id: None, + source_type: Some("user"), + importance: Some(50), + expires_at: None, + metadata_json: None, + now_ms, + }) + .unwrap() + }; + let memory_id = save("session-a", 1); + store + .inner + .with_conn(|conn| { + conn.execute( + "UPDATE mc_memories SET seen_count = 10 WHERE id = ?1", + [memory_id], + )?; + Ok(()) + }) + .unwrap(); + + save("session-a", 2); + assert_eq!( + store + .get_memory_full(memory_id) + .unwrap() + .unwrap() + .seen_count, + 10 + ); + save("session-b", 3); + assert_eq!( + store + .get_memory_full(memory_id) + .unwrap() + .unwrap() + .seen_count, + 11 + ); + } + #[test] fn memory_evidence_keeps_the_content_version_observed_by_each_session() { let dir = tempfile::tempdir().unwrap(); @@ -20692,7 +20854,7 @@ mod tests { } #[test] - fn authority_state_sync_replaces_the_complete_memory_evidence_set() { + fn authority_state_sync_distinguishes_absent_evidence_from_an_explicit_clear() { let dir = tempfile::tempdir().unwrap(); let store = McStore::open(&descriptor(dir.path())).unwrap(); let row = ModuleMemoryRow { @@ -20703,19 +20865,21 @@ mod tests { normalized_hash: "synced-hash".to_string(), status: "active".to_string(), verification_status: "unverified".to_string(), - evidence: vec![ModuleMemoryEvidenceRow { + evidence: Some(vec![ModuleMemoryEvidenceRow { content_hash: "synced-hash".to_string(), source_session_id: "session-a".to_string(), source_message_id: Some("assistant-a1".to_string()), source_type: "agent".to_string(), observed_at: 11, - }], + }]), ..Default::default() }; store .inner - .with_conn_fenced(|tx| replace_authority_memories_tx(tx, "/repo", &[row])) + .with_conn_fenced(|tx| { + replace_authority_memories_tx(tx, "/repo", std::slice::from_ref(&row)) + }) .unwrap(); let evidence = store @@ -20744,6 +20908,42 @@ mod tests { "agent".to_string(), ) ); + + let sparse = ModuleMemoryRow { + evidence: None, + ..row.clone() + }; + store + .inner + .with_conn_fenced(|tx| replace_authority_memories_tx(tx, "/repo", &[sparse])) + .unwrap(); + let evidence_count = store + .inner + .with_conn(|conn| { + conn.query_row("SELECT COUNT(*) FROM mc_memory_evidence", [], |row| { + row.get(0) + }) + }) + .unwrap(); + assert_eq!(evidence_count, 1); + + let clear = ModuleMemoryRow { + evidence: Some(Vec::new()), + ..row + }; + store + .inner + .with_conn_fenced(|tx| replace_authority_memories_tx(tx, "/repo", &[clear])) + .unwrap(); + let evidence_count = store + .inner + .with_conn(|conn| { + conn.query_row("SELECT COUNT(*) FROM mc_memory_evidence", [], |row| { + row.get(0) + }) + }) + .unwrap(); + assert_eq!(evidence_count, 0); } #[test] @@ -20843,7 +21043,7 @@ mod tests { .unwrap() .unwrap(); - assert_eq!(merged.seen_count, 10); + assert_eq!(merged.seen_count, 11); } #[test] @@ -23824,6 +24024,20 @@ mod shadow_tests { let canonical = insert(identity, "same fact", "session-a", 1); let duplicate = insert(route_project_root, "same fact", "session-b", 2); let singleton = insert(route_project_root, "path-only fact", "session-c", 3); + store + .inner + .with_conn(|conn| { + conn.execute( + "UPDATE mc_memories SET seen_count = 10 WHERE id = ?1", + [canonical], + )?; + conn.execute( + "UPDATE mc_memories SET seen_count = 4 WHERE id = ?1", + [duplicate], + )?; + Ok(()) + }) + .unwrap(); let evidence_before = store .inner .with_conn(|conn| { @@ -23871,6 +24085,14 @@ mod shadow_tests { }) .unwrap(); assert_eq!(canonical_evidence, 2); + assert_eq!( + store + .get_memory_full(canonical) + .unwrap() + .unwrap() + .seen_count, + 14 + ); assert_eq!( store .get_memory_full(canonical) diff --git a/packages/pi-plugin/src/tools/ctx-memory.test.ts b/packages/pi-plugin/src/tools/ctx-memory.test.ts index cbedd1f79..dd4f8e387 100644 --- a/packages/pi-plugin/src/tools/ctx-memory.test.ts +++ b/packages/pi-plugin/src/tools/ctx-memory.test.ts @@ -1114,6 +1114,44 @@ describe("createCtxMemoryTool", () => { }); describe("Pi ctx_memory provenance", () => { + it("advances a migrated legacy baseline only for a distinct session", async () => { + const db = createTestDb(); + try { + let sessionId = "pi-session-a"; + const tool = createCtxMemoryTool({ + db, + resolveProjectIdentity: () => "git:project", + }); + const ctx = { + cwd: "/repo", + sessionManager: { getSessionId: () => sessionId }, + } as never; + const write = () => + tool.execute( + "call-write", + { action: "write", category: "CONSTRAINTS", content: "Migrated Pi fact" }, + new AbortController().signal, + () => undefined, + ctx, + ); + + await write(); + const memory = db + .prepare("SELECT id FROM memories") + .get(); + expect(memory).toBeDefined(); + db.prepare("UPDATE memories SET seen_count = 10 WHERE id = ?").run(memory?.id); + + await write(); + expect(getMemoryById(db, memory?.id ?? -1)?.seenCount).toBe(10); + sessionId = "pi-session-b"; + await write(); + expect(getMemoryById(db, memory?.id ?? -1)?.seenCount).toBe(11); + } finally { + closeQuietly(db); + } + }); + it("preserves content-bound evidence when memories merge", async () => { const db = createTestDb(); try { diff --git a/packages/plugin/src/features/magic-context/context-authority.test.ts b/packages/plugin/src/features/magic-context/context-authority.test.ts index cf8f27a6c..ac61a62a2 100644 --- a/packages/plugin/src/features/magic-context/context-authority.test.ts +++ b/packages/plugin/src/features/magic-context/context-authority.test.ts @@ -128,6 +128,54 @@ describe("memory authority protocol", () => { }); }); + test("sparse evidence snapshots preserve evidence while explicit empty snapshots clear it", () => { + const database = db(); + const snapshot = (feedSeq: number, evidence?: readonly Record[]) => ({ + feed_seq: feedSeq, + domain: "memories" as const, + op: "update" as const, + module_row_id: 9, + full_row_snapshot: { + project_path: "/repo", + category: "CONSTRAINTS", + content: "Mirrored fact", + normalized_hash: "fact-hash", + ...(evidence === undefined ? {} : { evidence }), + }, + content_hash: "fact-hash", + }); + const apply = (feedSeq: number, evidence?: readonly Record[]) => + applyMirrorPage({ + db: database, + page: { + domain: "memories", + cursor: feedSeq - 1, + next_cursor: feedSeq, + has_more: false, + rows: [snapshot(feedSeq, evidence)], + }, + }); + + apply(1, [ + { + content_hash: "fact-hash", + source_session_id: "session-a", + source_message_id: "assistant-a1", + source_type: "agent", + observed_at: 11, + }, + ]); + apply(2); + expect(database.prepare("SELECT COUNT(*) AS count FROM memory_evidence").get()).toEqual({ + count: 1, + }); + + apply(3, []); + expect(database.prepare("SELECT COUNT(*) AS count FROM memory_evidence").get()).toEqual({ + count: 0, + }); + }); + test("historical sparse note feed rows preserve rich local columns", () => { const database = db(); const localStoreUuid = ensureContextStoreUuid(database); diff --git a/packages/plugin/src/features/magic-context/memory/relocate-memory.ts b/packages/plugin/src/features/magic-context/memory/relocate-memory.ts index 9dbf7556a..34d8e2e81 100644 --- a/packages/plugin/src/features/magic-context/memory/relocate-memory.ts +++ b/packages/plugin/src/features/magic-context/memory/relocate-memory.ts @@ -1,4 +1,5 @@ import type { Database } from "../../../shared/sqlite"; +import { mergeMemoryEpisodeEvidence } from "./storage-memory"; import type { MemoryStatus } from "./types"; /** @@ -83,20 +84,10 @@ export function rekeyMemoryRowWithCollisionMerge( ) .get(); if (hasEvidence) { + const mergedSeenCount = mergeMemoryEpisodeEvidence(db, collision.id, [rowId]); db.prepare( - `INSERT OR IGNORE INTO memory_evidence ( - memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at - ) - SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at - FROM memory_evidence WHERE memory_id = ?`, - ).run(collision.id, rowId); - db.prepare( - `UPDATE memories SET seen_count = MAX( - COALESCE(seen_count, 1), - ?, - (SELECT COUNT(DISTINCT source_session_id) FROM memory_evidence WHERE memory_id = ?) - ) WHERE id = ?`, - ).run(row.seen_count ?? 1, collision.id, collision.id); + "UPDATE memories SET seen_count = MAX(COALESCE(seen_count, 1), ?) WHERE id = ?", + ).run(mergedSeenCount ?? row.seen_count ?? 1, collision.id); } else { db.prepare( "UPDATE memories SET seen_count = MAX(COALESCE(seen_count, 1), ?) WHERE id = ?", diff --git a/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts b/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts index 81fa0d944..b5fec681b 100644 --- a/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts +++ b/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts @@ -13,6 +13,7 @@ import { getMemoryById, insertMemoryIdempotent, mergeMemoryStats, + ModuleMemoryAuthorityError, updateMemoryContent, } from "./storage-memory"; @@ -87,6 +88,32 @@ describe("memory evidence lifecycle", () => { ]); }); + it("rejects MODULE-managed exact duplicates before recording evidence", () => { + db = makeDatabase(); + const memory = save("Authority-owned fact", "session-a", "user-a1"); + db.prepare( + "INSERT INTO authority_managed(project_path, context_store_uuid, marked_at) VALUES (?, ?, ?)", + ).run("git:project", "store-uuid", Date.now()); + + expect(() => save("Authority-owned fact", "session-b", "user-b1")).toThrow( + ModuleMemoryAuthorityError, + ); + expect(evidence(memory.id)).toHaveLength(1); + expect(getMemoryById(db, memory.id)?.seenCount).toBe(1); + }); + + it("advances a migrated legacy baseline for a new session without double counting", () => { + db = makeDatabase(); + const memory = save("Migrated fact", "session-a", "user-a1"); + db.prepare("UPDATE memories SET seen_count = 10 WHERE id = ?").run(memory.id); + + save("Migrated fact", "session-a", "user-a2"); + expect(getMemoryById(db, memory.id)?.seenCount).toBe(10); + + save("Migrated fact", "session-b", "user-b1"); + expect(getMemoryById(db, memory.id)?.seenCount).toBe(11); + }); + it("preserves evidence through update archive and delete lifecycle", () => { db = makeDatabase(); const memory = save("Original fact", "session-a", "user-a1"); @@ -150,7 +177,7 @@ describe("memory evidence lifecycle", () => { "active", ); - expect(getMemoryById(db, canonical.id)?.seenCount).toBe(10); + expect(getMemoryById(db, canonical.id)?.seenCount).toBe(11); }); it("preserves evidence when identity relocation merges an exact collision", () => { diff --git a/packages/plugin/src/features/magic-context/memory/storage-memory.ts b/packages/plugin/src/features/magic-context/memory/storage-memory.ts index 95d0a245e..fa887dba0 100644 --- a/packages/plugin/src/features/magic-context/memory/storage-memory.ts +++ b/packages/plugin/src/features/magic-context/memory/storage-memory.ts @@ -277,8 +277,15 @@ function recordMemoryEvidenceInCurrentTransaction( input: MemoryInput, ): void { if (!input.sourceSessionId || !hasMemoryEvidenceTable(db)) return; - const memory = db.prepare("SELECT normalized_hash FROM memories WHERE id = ?").get(memoryId) as - | { normalized_hash?: string } + const memory = db + .prepare( + `SELECT normalized_hash, source_session_id, + (SELECT COUNT(DISTINCT source_session_id) + FROM memory_evidence WHERE memory_id = ?) AS evidence_count + FROM memories WHERE id = ?`, + ) + .get(memoryId, memoryId) as + | { normalized_hash?: string; source_session_id?: string | null; evidence_count?: number } | undefined; if (!memory?.normalized_hash) return; const now = Date.now(); @@ -299,14 +306,22 @@ function recordMemoryEvidenceInCurrentTransaction( if ((result.changes ?? 0) === 0) return; db.prepare( `UPDATE memories - SET seen_count = MAX( - COALESCE(seen_count, 1), - (SELECT COUNT(DISTINCT source_session_id) FROM memory_evidence WHERE memory_id = ?) - ), + SET seen_count = MAX(COALESCE(seen_count, 1) - ?, 0) + + (SELECT COUNT(DISTINCT source_session_id) FROM memory_evidence WHERE memory_id = ?), last_seen_at = ?, updated_at = ? WHERE id = ?`, - ).run(memoryId, now, now, memoryId); + ).run( + (memory.evidence_count ?? 0) > 0 + ? memory.evidence_count + : memory.source_session_id === input.sourceSessionId + ? 1 + : 0, + memoryId, + now, + now, + memoryId, + ); } export function recordMemoryEvidence(db: Database, memoryId: number, input: MemoryInput): void { @@ -556,7 +571,7 @@ function getMergeMemoryStatsStatement(db: Database): PreparedStatement { let stmt = mergeMemoryStatsStatements.get(db); if (!stmt) { stmt = db.prepare( - "UPDATE memories SET seen_count = MAX(COALESCE(seen_count, 1), ?, ?), retrieval_count = ?, merged_from = ?, status = ?, updated_at = ? WHERE id = ?", + "UPDATE memories SET seen_count = MAX(COALESCE(seen_count, 1), ?), retrieval_count = ?, merged_from = ?, status = ?, updated_at = ? WHERE id = ?", ); mergeMemoryStatsStatements.set(db, stmt); } @@ -716,6 +731,7 @@ export function insertMemory(db: Database, input: MemoryInput): Memory { * surfacing a transient write failure. */ export function insertMemoryIdempotent(db: Database, input: MemoryInput): InsertMemoryResult { + assertTsMemoryWriteAllowed(db, input.projectPath); const existing = getMemoryByHash( db, input.projectPath, @@ -747,6 +763,46 @@ export function insertMemoryIdempotent(db: Database, input: MemoryInput): Insert } } +export function mergeMemoryEpisodeEvidence( + db: Database, + targetId: number, + sourceIds: readonly number[], +): number | null { + if (!hasMemoryEvidenceTable(db)) return null; + const ids = [targetId, ...sourceIds]; + const placeholders = ids.map(() => "?").join(", "); + const rows = db + .prepare( + `SELECT memories.id, memories.seen_count, + COUNT(DISTINCT memory_evidence.source_session_id) AS evidence_count + FROM memories + LEFT JOIN memory_evidence ON memory_evidence.memory_id = memories.id + WHERE memories.id IN (${placeholders}) + GROUP BY memories.id, memories.seen_count`, + ) + .all(...ids) as Array<{ seen_count?: number; evidence_count?: number }>; + const legacyBaseline = rows.reduce( + (sum, row) => + sum + Math.max((row.seen_count ?? 1) - (row.evidence_count ?? 0), 0), + 0, + ); + for (const sourceId of sourceIds) { + db.prepare( + `INSERT OR IGNORE INTO memory_evidence ( + memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at + ) + SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at + FROM memory_evidence WHERE memory_id = ?`, + ).run(targetId, sourceId); + } + const union = db + .prepare( + "SELECT COUNT(DISTINCT source_session_id) AS count FROM memory_evidence WHERE memory_id = ?", + ) + .get(targetId) as { count?: number } | undefined; + return legacyBaseline + (union?.count ?? 0); +} + export function getMemoryByHash( db: Database, projectPath: string, @@ -1234,26 +1290,16 @@ export function mergeMemoryStats( status: MemoryStatus, ): void { assertTsMemoryIdWriteAllowed(db, id); - let evidenceCount: number | null = null; - if (hasMemoryEvidenceTable(db)) { - db.prepare( - `INSERT OR IGNORE INTO memory_evidence ( - memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at - ) - SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at - FROM memory_evidence - WHERE memory_id IN (SELECT value FROM json_each(?))`, - ).run(id, mergedFrom); - const row = db - .prepare( - "SELECT COUNT(DISTINCT source_session_id) AS count FROM memory_evidence WHERE memory_id = ?", - ) - .get(id) as { count?: number } | undefined; - if (typeof row?.count === "number" && row.count > 0) evidenceCount = row.count; - } + const mergedIds = db + .prepare("SELECT value AS id FROM json_each(?) WHERE type = 'integer' AND value <> ?") + .all(mergedFrom, id) as Array<{ id: number }>; + const mergedEpisodeCount = mergeMemoryEpisodeEvidence( + db, + id, + mergedIds.map((row) => row.id), + ); getMergeMemoryStatsStatement(db).run( - seenCount, - evidenceCount ?? 0, + mergedEpisodeCount ?? seenCount, retrievalCount, mergedFrom, status, diff --git a/packages/plugin/src/features/magic-context/storage-identity-merge.ts b/packages/plugin/src/features/magic-context/storage-identity-merge.ts index 7c0ee64d5..442ac7969 100644 --- a/packages/plugin/src/features/magic-context/storage-identity-merge.ts +++ b/packages/plugin/src/features/magic-context/storage-identity-merge.ts @@ -1,4 +1,5 @@ import type { Database } from "../../shared/sqlite"; +import { mergeMemoryEpisodeEvidence } from "./memory/storage-memory"; const IDENTITY_COLUMNS = new Set(["project_path", "project_identity"]); const DERIVED_TABLE_SUFFIXES = [ @@ -201,23 +202,9 @@ function mergeMemoryRow( .get(toIdentity, row.category, row.normalized_hash, sourceId) as SqliteRow | undefined; if (collision && typeof collision.id === "number") { const targetId = collision.id; - db.prepare( - `INSERT OR IGNORE INTO memory_evidence ( - memory_id, content_hash, source_session_id, source_message_id, source_type, observed_at - ) - SELECT ?, content_hash, source_session_id, source_message_id, source_type, observed_at - FROM memory_evidence WHERE memory_id = ?`, - ).run(targetId, sourceId); - const evidenceRow = db - .prepare( - "SELECT COUNT(DISTINCT source_session_id) AS count FROM memory_evidence WHERE memory_id = ?", - ) - .get(targetId) as { count?: number } | undefined; - const mergedSeen = Math.max( - Number(collision.seen_count ?? 1), - Number(row.seen_count ?? 1), - evidenceRow?.count ?? 0, - ); + const mergedSeen = + mergeMemoryEpisodeEvidence(db, targetId, [sourceId]) ?? + Math.max(Number(collision.seen_count ?? 1), Number(row.seen_count ?? 1)); const sourceClassifiedAt = Number(row.classified_at ?? 0); const targetClassifiedAt = Number(collision.classified_at ?? 0); if (sourceClassifiedAt > targetClassifiedAt) { From ebcc0ff0006758e966112bc32b48c09f17c0711e Mon Sep 17 00:00:00 2001 From: Cole Leavitt Date: Wed, 19 Aug 2026 06:55:03 -0400 Subject: [PATCH 5/7] fix(rust): satisfy memory evidence lifetimes --- crates/mc-store/src/lib.rs | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/crates/mc-store/src/lib.rs b/crates/mc-store/src/lib.rs index 0da2b875e..aa0478ba6 100644 --- a/crates/mc-store/src/lib.rs +++ b/crates/mc-store/src/lib.rs @@ -16611,17 +16611,16 @@ fn memory_feed_snapshot( FROM mc_memory_evidence WHERE memory_id = ?1 ORDER BY content_hash, source_session_id", )?; - statement - .query_map([memory.id], |row| { - Ok(ModuleMemoryEvidenceRow { - content_hash: row.get(0)?, - source_session_id: row.get(1)?, - source_message_id: row.get(2)?, - source_type: row.get(3)?, - observed_at: row.get(4)?, - }) - })? - .collect::, _>>()? + let rows = statement.query_map([memory.id], |row| { + Ok(ModuleMemoryEvidenceRow { + content_hash: row.get(0)?, + source_session_id: row.get(1)?, + source_message_id: row.get(2)?, + source_type: row.get(3)?, + observed_at: row.get(4)?, + }) + })?; + rows.collect::, _>>()? }; Ok(serde_json::json!({ "id": memory.id, @@ -20775,9 +20774,9 @@ mod tests { let mut statement = conn.prepare( "SELECT content_hash FROM mc_memory_evidence WHERE memory_id = ?1 ORDER BY observed_at", )?; - statement - .query_map([memory_id], |row| row.get::<_, String>(0))? - .collect::, _>>() + let rows = + statement.query_map([memory_id], |row| row.get::<_, String>(0))?; + rows.collect::, _>>() }) .unwrap(); assert_eq!( @@ -20917,7 +20916,7 @@ mod tests { .inner .with_conn_fenced(|tx| replace_authority_memories_tx(tx, "/repo", &[sparse])) .unwrap(); - let evidence_count = store + let evidence_count: i64 = store .inner .with_conn(|conn| { conn.query_row("SELECT COUNT(*) FROM mc_memory_evidence", [], |row| { @@ -20935,7 +20934,7 @@ mod tests { .inner .with_conn_fenced(|tx| replace_authority_memories_tx(tx, "/repo", &[clear])) .unwrap(); - let evidence_count = store + let evidence_count: i64 = store .inner .with_conn(|conn| { conn.query_row("SELECT COUNT(*) FROM mc_memory_evidence", [], |row| { From 0fc893296a7dfaaa5365edd25c602e675e8d9fd7 Mon Sep 17 00:00:00 2001 From: Cole Leavitt Date: Wed, 19 Aug 2026 07:20:15 -0400 Subject: [PATCH 6/7] style(memory): format evidence invariants --- .../magic-context/memory/storage-memory-evidence.test.ts | 2 +- .../plugin/src/features/magic-context/memory/storage-memory.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts b/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts index b5fec681b..7687eb327 100644 --- a/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts +++ b/packages/plugin/src/features/magic-context/memory/storage-memory-evidence.test.ts @@ -12,8 +12,8 @@ import { deleteMemory, getMemoryById, insertMemoryIdempotent, - mergeMemoryStats, ModuleMemoryAuthorityError, + mergeMemoryStats, updateMemoryContent, } from "./storage-memory"; diff --git a/packages/plugin/src/features/magic-context/memory/storage-memory.ts b/packages/plugin/src/features/magic-context/memory/storage-memory.ts index fa887dba0..4176ed7a1 100644 --- a/packages/plugin/src/features/magic-context/memory/storage-memory.ts +++ b/packages/plugin/src/features/magic-context/memory/storage-memory.ts @@ -782,8 +782,7 @@ export function mergeMemoryEpisodeEvidence( ) .all(...ids) as Array<{ seen_count?: number; evidence_count?: number }>; const legacyBaseline = rows.reduce( - (sum, row) => - sum + Math.max((row.seen_count ?? 1) - (row.evidence_count ?? 0), 0), + (sum, row) => sum + Math.max((row.seen_count ?? 1) - (row.evidence_count ?? 0), 0), 0, ); for (const sourceId of sourceIds) { From 1975b8a81574d0881b88d3a0291016e3a7911510 Mon Sep 17 00:00:00 2001 From: Cole Leavitt Date: Wed, 19 Aug 2026 07:38:40 -0400 Subject: [PATCH 7/7] style(pi): format memory evidence regression --- packages/pi-plugin/src/tools/ctx-memory.test.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/packages/pi-plugin/src/tools/ctx-memory.test.ts b/packages/pi-plugin/src/tools/ctx-memory.test.ts index dd4f8e387..e9281472e 100644 --- a/packages/pi-plugin/src/tools/ctx-memory.test.ts +++ b/packages/pi-plugin/src/tools/ctx-memory.test.ts @@ -1129,7 +1129,11 @@ describe("Pi ctx_memory provenance", () => { const write = () => tool.execute( "call-write", - { action: "write", category: "CONSTRAINTS", content: "Migrated Pi fact" }, + { + action: "write", + category: "CONSTRAINTS", + content: "Migrated Pi fact", + }, new AbortController().signal, () => undefined, ctx, @@ -1140,7 +1144,9 @@ describe("Pi ctx_memory provenance", () => { .prepare("SELECT id FROM memories") .get(); expect(memory).toBeDefined(); - db.prepare("UPDATE memories SET seen_count = 10 WHERE id = ?").run(memory?.id); + db.prepare("UPDATE memories SET seen_count = 10 WHERE id = ?").run( + memory?.id, + ); await write(); expect(getMemoryById(db, memory?.id ?? -1)?.seenCount).toBe(10);