From 7b2ef74964eb014e498db6f76783e6ce755a028e Mon Sep 17 00:00:00 2001 From: cdeust Date: Fri, 3 Jul 2026 18:25:29 +0200 Subject: [PATCH 1/2] fix(infrastructure): make first-install embedding model download reachable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SentenceTransformer loading set HF_HUB_OFFLINE=1 before import, then unset the env var and re-imported on cache miss — but huggingface_hub freezes that env var into module constants at first import, so the retry still ran offline and fresh installs failed with a misleading "couldn't connect to huggingface.co" error. Load with an explicit local_files_only=True first (hermetic when cached, same privacy posture), fall back to a plain online load on cache miss. Reproduced and verified in a clean-env harness run on 2026-07-03. Requires sentence-transformers >= 3.0.0 (local_files_only kwarg absent in v2.3.0, present in v3.0.0 — verified against tagged sources); floors raised accordingly. Also add psycopg-pool to the benchmarks extra: pg_store.py imports ConnectionPool at module level, so the extra could not run the benchmarks it exists for. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KVjk4LSGVEjuzCgLMBzvQ2 --- mcp_server/infrastructure/embedding_engine.py | 44 ++++++++----------- pyproject.toml | 7 +-- uv.lock | 10 +++-- 3 files changed, 29 insertions(+), 32 deletions(-) diff --git a/mcp_server/infrastructure/embedding_engine.py b/mcp_server/infrastructure/embedding_engine.py index 02e5d48f..78c4147c 100644 --- a/mcp_server/infrastructure/embedding_engine.py +++ b/mcp_server/infrastructure/embedding_engine.py @@ -176,11 +176,9 @@ def _ensure_model(self) -> None: if self._model is not None or self._unavailable: return try: - import os - - # Collect cache-miss exception types: OSError covers old HF versions; - # LocalEntryNotFoundError covers newer huggingface_hub (>=0.22) where - # HF_HUB_OFFLINE=1 raises a non-OSError on cache miss. + # Collect cache-miss exception types: OSError covers transformers' + # config loader; LocalEntryNotFoundError covers huggingface_hub + # (>=0.22) which raises a non-OSError on local-only cache miss. _cache_miss: tuple[type[Exception], ...] = (OSError,) try: from huggingface_hub.errors import LocalEntryNotFoundError @@ -189,31 +187,27 @@ def _ensure_model(self) -> None: except ImportError: pass - # Set offline mode BEFORE importing sentence_transformers to prevent - # unauthenticated HF Hub requests. The import itself initializes - # huggingface_hub which checks this env var at module load time. - had_offline = os.environ.get("HF_HUB_OFFLINE") - os.environ["HF_HUB_OFFLINE"] = "1" device = self._resolve_device() + from sentence_transformers import SentenceTransformer + + # local_files_only=True keeps the load hermetic (no HF Hub + # requests) whenever the model is already cached. It must be + # an explicit argument, not the HF_HUB_OFFLINE env var: hub + # freezes that env var into module constants at first import, + # so the previous unset-env-and-retry fallback still ran + # offline, leaving the download path unreachable on a fresh + # install (reproduced 2026-07-03, clean-env harness run). + # source: kwarg added in sentence-transformers v3.0.0 + # (absent in v2.3.0 SentenceTransformer.py; pyproject floor + # raised to match). try: - from sentence_transformers import SentenceTransformer - - self._model = SentenceTransformer(self._model_name, device=device) + self._model = SentenceTransformer( + self._model_name, device=device, local_files_only=True + ) except _cache_miss: - # Model not in local cache — need to download it once - if had_offline is None: - del os.environ["HF_HUB_OFFLINE"] - else: - os.environ["HF_HUB_OFFLINE"] = had_offline + # Model not in local cache — download it once. logger.info("Downloading embedding model: %s", self._model_name) - from sentence_transformers import SentenceTransformer - self._model = SentenceTransformer(self._model_name, device=device) - finally: - if had_offline is None: - os.environ.pop("HF_HUB_OFFLINE", None) - else: - os.environ["HF_HUB_OFFLINE"] = had_offline # sentence-transformers 5.x renamed get_sentence_embedding_dimension # → get_embedding_dimension. Prefer the new name; fall back for <5. diff --git a/pyproject.toml b/pyproject.toml index c1f63b3c..aa497f6e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ # distribution ships full-quality recall by default. The plugin already # installs them (scripts/setup.sh, scripts/launcher.py); this closes the # gap for the MCPB build (`uv run`). uv.lock pins ST 5.4.1 / flashrank 0.2.10. - "sentence-transformers>=2.2.0", + "sentence-transformers>=3.0.0", "flashrank>=0.2.0", ] @@ -93,9 +93,10 @@ viz-tile = [ ] benchmarks = [ "datasets>=2.14.0", # HuggingFace dataset loader (BEAM, LoCoMo, LongMemEval) - "sentence-transformers>=2.2.0", # all-MiniLM-L6-v2 embeddings + "sentence-transformers>=3.0.0", # all-MiniLM-L6-v2 embeddings "flashrank>=0.2.0", # cross-encoder reranking (ms-marco-MiniLM-L-12-v2) "psycopg[binary]>=3.1", # PostgreSQL driver + "psycopg-pool>=3.2", # pg_store.py imports ConnectionPool at module level "pgvector>=0.3", # vector similarity in PostgreSQL "pymupdf>=1.24.0", # PDF parsing for some benchmark sources ] @@ -109,7 +110,7 @@ dev = [ # timeout meant no way to identify the offender. 300s is generous # vs current measurements; @pytest.mark.timeout(N) opts out per test. "pytest-timeout>=2.3.0", - "sentence-transformers>=2.2.0", + "sentence-transformers>=3.0.0", "networkx>=3.0", ] diff --git a/uv.lock b/uv.lock index 84a5a7b0..61b4cebb 100644 --- a/uv.lock +++ b/uv.lock @@ -1234,7 +1234,7 @@ wheels = [ [[package]] name = "hypermnesia-mcp" -version = "3.24.1" +version = "4.0.0" source = { editable = "." } dependencies = [ { name = "fastmcp" }, @@ -1252,6 +1252,7 @@ benchmarks = [ { name = "flashrank" }, { name = "pgvector" }, { name = "psycopg", extra = ["binary"] }, + { name = "psycopg-pool" }, { name = "pymupdf" }, { name = "sentence-transformers" }, ] @@ -1315,6 +1316,7 @@ requires-dist = [ { name = "pillow", marker = "extra == 'viz-tile'", specifier = ">=10.0" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'benchmarks'", specifier = ">=3.1" }, { name = "psycopg", extras = ["binary"], marker = "extra == 'postgresql'", specifier = ">=3.1" }, + { name = "psycopg-pool", marker = "extra == 'benchmarks'", specifier = ">=3.2" }, { name = "psycopg-pool", marker = "extra == 'postgresql'", specifier = ">=3.2" }, { name = "pyarrow", marker = "extra == 'viz-tile'", specifier = ">=15.0" }, { name = "pydantic", specifier = ">=2.0.0" }, @@ -1324,9 +1326,9 @@ requires-dist = [ { name = "pytest-asyncio", marker = "extra == 'dev'", specifier = ">=0.23.0" }, { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, { name = "pytest-timeout", marker = "extra == 'dev'", specifier = ">=2.3.0" }, - { name = "sentence-transformers", specifier = ">=2.2.0" }, - { name = "sentence-transformers", marker = "extra == 'benchmarks'", specifier = ">=2.2.0" }, - { name = "sentence-transformers", marker = "extra == 'dev'", specifier = ">=2.2.0" }, + { name = "sentence-transformers", specifier = ">=3.0.0" }, + { name = "sentence-transformers", marker = "extra == 'benchmarks'", specifier = ">=3.0.0" }, + { name = "sentence-transformers", marker = "extra == 'dev'", specifier = ">=3.0.0" }, { name = "sqlite-vec", marker = "extra == 'sqlite'", specifier = ">=0.1.1" }, { name = "tree-sitter", marker = "extra == 'codebase'", specifier = ">=0.24.0,<0.26" }, { name = "tree-sitter-language-pack", marker = "extra == 'codebase'", specifier = ">=0.24.0,<1.7" }, From 8d8646ad2752e00df2419dbdf5b15da7c25e5a2f Mon Sep 17 00:00:00 2001 From: cdeust Date: Fri, 3 Jul 2026 18:30:18 +0200 Subject: [PATCH 2/2] feat(benchmarks): one-command LongMemEval reproduction harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit make longmemeval runs the full official 500-question benchmark from a clean machine: downloads the dataset from the authors' HF repository (sha256-pinned, byte-identical to the file behind every published result), provisions an ephemeral PostgreSQL + pgvector container isolated from any existing install, runs the untouched production runner through the production PL/pgSQL recall path, prints the metric table, and tears down. make longmemeval-smoke gives a 10-question harness check. benchmarks/README.md documents requirements, measured wall-clock (39.6 min full run, a3_longmemeval_post_refactor.md), and the metric scope: session-level retrieval Recall@10/MRR versus the LongMemEval paper's best retrieval configuration (78.4% R@10) — explicitly not comparable to end-to-end QA-accuracy leaderboards. Verified three ways on 2026-07-03: warm run, clean-environment run (fresh uv cache, fresh HF cache, no dataset, no image), and a full 500-question run reproducing the published R@10 98.4% exactly (MRR 0.915; record in benchmarks/results/harness_repro/). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01KVjk4LSGVEjuzCgLMBzvQ2 --- Makefile | 14 ++ benchmarks/README.md | 63 ++++++++ benchmarks/repro_longmemeval.sh | 135 ++++++++++++++++++ .../longmemeval_full_20260703.json | 50 +++++++ 4 files changed, 262 insertions(+) create mode 100644 Makefile create mode 100644 benchmarks/README.md create mode 100755 benchmarks/repro_longmemeval.sh create mode 100644 benchmarks/results/harness_repro/longmemeval_full_20260703.json diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..838f2b76 --- /dev/null +++ b/Makefile @@ -0,0 +1,14 @@ +# Benchmark reproduction targets. See benchmarks/README.md for what each +# measures, requirements, and honest wall-clock numbers. + +.PHONY: longmemeval longmemeval-smoke + +# Full LongMemEval-S run: 500 questions through the production recall +# path. ~40 min on a laptop (CPU embeddings), fully local, no API keys. +longmemeval: + bash benchmarks/repro_longmemeval.sh + +# 10-question sanity run — verifies the whole harness end to end +# (dataset, ephemeral DB, embeddings, recall) in a few minutes. +longmemeval-smoke: + bash benchmarks/repro_longmemeval.sh --limit 10 diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..ec974189 --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,63 @@ +# Reproducing the Cortex benchmarks + +Every published Cortex retrieval number comes from the scripts in this +directory, run against the **production code path**: data is ingested +through `mcp_server.core.memory_ingest`, retrieval goes through the same +PL/pgSQL `recall_memories()` + FlashRank reranking that serves live MCP +calls. There is no benchmark-only retriever. + +## One-command reproduction (LongMemEval) + +Requirements: Docker, [uv](https://docs.astral.sh/uv/), ~1.5 GB free disk +(dataset + embedding models), no API keys. + +```bash +make longmemeval-smoke # 10 questions, a few minutes — verifies the harness +make longmemeval # full 500 questions, ~40 min on a laptop +``` + +The harness downloads the official LongMemEval-S dataset from the +authors' Hugging Face repository (sha256-pinned), provisions an +ephemeral PostgreSQL + pgvector container on port 55432 (it never +touches an existing Cortex install), runs the benchmark, prints the +Recall@K / MRR table, and removes the container. `KEEP_DB=1 make +longmemeval` keeps the database for inspection. Every run also emits a +reproducibility manifest (commit, config, dataset hash) via +`benchmarks/_repro.py`. + +Measured wall-clock for the full run: **39.6 min** on Apple Silicon with +CPU embeddings (`benchmarks/results/a3_longmemeval_post_refactor.md`). + +## What the numbers mean (metric scope) + +Cortex reports **session-level retrieval Recall@10 and MRR**: for each +of the 500 questions, all haystack sessions are loaded into the store, +production recall runs, and the run scores whether the answer-bearing +session(s) appear in the top 10. + +- The comparable published baseline is the best retrieval configuration + in the LongMemEval paper itself (Wu et al., ICLR 2025): **Recall@10 + 78.4%**. Cortex: **98.4%** (n=500). +- This is **not** the end-to-end QA accuracy that LLM-answering + leaderboards report (an LLM answers from the retrieved context and a + judge scores the answer). Retrieval recall and QA accuracy are + different measurements; do not compare one to the other. +- The same scoping discipline applies to BEAM: see the BEAM note in + `CLAUDE.md` — the retrieval-proxy MRR there is used only for + within-system comparisons, never as a head-to-head claim. + +If your numbers differ from the published ones, open an issue with the +printed reproducibility manifest and we will publish the discrepancy. + +## Other benchmarks + +| Benchmark | Runner | Dataset | +|---|---|---| +| LongMemEval (ICLR 2025), 500 Q | `longmemeval/run_benchmark.py` | auto-downloaded by the harness | +| LoCoMo (ACL 2024), 1,986 Q | `locomo/run_benchmark.py` | see runner's download hint | +| BEAM (ICLR 2026) | `beam/run_benchmark.py --split 100K` | see runner's download hint | +| MemoryAgentBench, EverMemBench, Episodic | respective `run_benchmark.py` | see runner's download hint | + +Ablation studies (per-mechanism lesion runs) live in +`benchmarks/lib/ablation_runner` and their results under +`benchmarks/results/ablation/`. diff --git a/benchmarks/repro_longmemeval.sh b/benchmarks/repro_longmemeval.sh new file mode 100755 index 00000000..78d6cd08 --- /dev/null +++ b/benchmarks/repro_longmemeval.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +# One-command reproduction of the LongMemEval retrieval benchmark. +# +# make longmemeval # full 500-question run +# make longmemeval-smoke # 10-question sanity run +# +# What it does: +# 1. Downloads the official LongMemEval-S dataset (sha256-pinned). +# 2. Starts an ephemeral PostgreSQL + pgvector container, isolated +# from any existing Cortex install (separate port and database). +# 3. Runs benchmarks/longmemeval/run_benchmark.py through the SAME +# PL/pgSQL recall path production uses (no benchmark-only retriever). +# 4. Prints the Recall@K / MRR table and tears the container down. +# +# Measured wall-clock for the full 500 questions: 39.6 min on Apple +# Silicon with CPU embeddings. +# source: benchmarks/results/a3_longmemeval_post_refactor.md (2026-04-17) +# +# Metric scope: session-level retrieval Recall@10 / MRR. This is NOT the +# end-to-end QA accuracy that LLM-answering leaderboards report. The +# comparable published baseline is the best retrieval configuration in +# the LongMemEval paper (Wu et al., ICLR 2025): Recall@10 78.4%. +# +# Environment overrides: +# CORTEX_BENCH_PORT host port for the ephemeral PG (default 55432) +# KEEP_DB=1 keep the container running after the run +# +# Extra arguments are passed through to run_benchmark.py +# (e.g. --limit 10, --n-runs 3, --results-out path.json). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +DATASET_PATH="$REPO_ROOT/benchmarks/longmemeval/longmemeval_s.json" + +# Official dataset location, per the LongMemEval authors' HF repository. +# source: https://huggingface.co/datasets/xiaowu0162/LongMemEval +DATASET_URL="https://huggingface.co/datasets/xiaowu0162/LongMemEval/resolve/main/longmemeval_s" +# source: measured 2026-07-03 against the HF copy (278,025,796 bytes), +# byte-identical to the file behind every published Cortex result. +DATASET_SHA256="08d8dad4be43ee2049a22ff5674eb86725d0ce5ff434cde2627e5e8e7e117894" + +# pgvector's official image; any PostgreSQL >= 15 with the vector +# extension available works (mcp_server/infrastructure/pg_schema.py +# creates the extensions itself on first connect). +PG_IMAGE="pgvector/pgvector:pg16" +PG_PORT="${CORTEX_BENCH_PORT:-55432}" +CONTAINER="cortex-bench-pg" +BENCH_DB_URL="postgresql://postgres:cortex_bench@localhost:${PG_PORT}/cortex_bench" + +started_container=0 + +need_cmd() { + command -v "$1" >/dev/null 2>&1 || { + echo "error: '$1' is required but not installed." >&2 + exit 1 + } +} + +sha256_of() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +fetch_dataset() { + if [ -f "$DATASET_PATH" ]; then + echo "==> Dataset present: $DATASET_PATH" + else + echo "==> Downloading LongMemEval-S (~265 MB) from the official HF repo..." + curl -L --fail --progress-bar -o "$DATASET_PATH" "$DATASET_URL" + fi + echo "==> Verifying dataset checksum..." + local actual + actual="$(sha256_of "$DATASET_PATH")" + if [ "$actual" != "$DATASET_SHA256" ]; then + echo "error: dataset checksum mismatch." >&2 + echo " expected: $DATASET_SHA256" >&2 + echo " actual: $actual" >&2 + echo "The upstream file changed or the download is corrupt." >&2 + echo "Delete $DATASET_PATH and retry; open an issue if it persists." >&2 + exit 1 + fi + echo "==> Checksum OK." +} + +start_db() { + if docker ps --format '{{.Names}}' | grep -q "^${CONTAINER}$"; then + echo "==> Reusing running container ${CONTAINER}." + return + fi + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + echo "==> Starting ephemeral PostgreSQL (${PG_IMAGE}) on port ${PG_PORT}..." + docker run -d --name "$CONTAINER" \ + -e POSTGRES_PASSWORD=cortex_bench \ + -e POSTGRES_DB=cortex_bench \ + -p "${PG_PORT}:5432" \ + "$PG_IMAGE" >/dev/null + started_container=1 + echo "==> Waiting for PostgreSQL to accept connections..." + until docker exec "$CONTAINER" pg_isready -U postgres -d cortex_bench \ + >/dev/null 2>&1; do + sleep 1 + done +} + +teardown() { + if [ "$started_container" = "1" ] && [ "${KEEP_DB:-0}" != "1" ]; then + echo "==> Removing ephemeral container ${CONTAINER} (KEEP_DB=1 to keep)." + docker rm -f "$CONTAINER" >/dev/null 2>&1 || true + fi +} + +main() { + need_cmd docker + need_cmd curl + need_cmd uv + if ! docker info >/dev/null 2>&1; then + echo "error: the Docker daemon is not running." >&2 + echo "Start it (Docker Desktop, 'colima start', ...) and retry." >&2 + exit 1 + fi + fetch_dataset + start_db + trap teardown EXIT + echo "==> Running the benchmark through the production recall path..." + echo " (full 500-question run takes ~40 min; pass --limit N for less)" + cd "$REPO_ROOT" + DATABASE_URL="$BENCH_DB_URL" uv run --extra benchmarks python \ + benchmarks/longmemeval/run_benchmark.py "$@" +} + +main "$@" diff --git a/benchmarks/results/harness_repro/longmemeval_full_20260703.json b/benchmarks/results/harness_repro/longmemeval_full_20260703.json new file mode 100644 index 00000000..10f7ac00 --- /dev/null +++ b/benchmarks/results/harness_repro/longmemeval_full_20260703.json @@ -0,0 +1,50 @@ +{ + "overall_mrr": 0.9152214285714285, + "overall_recall10": 0.984, + "category_mrr": { + "Single-session (user)": 0.8319841269841269, + "Multi-session reasoning": 0.9638680033416877, + "Single-session (preference)": 0.6941666666666666, + "Temporal reasoning": 0.9184389545291801, + "Knowledge updates": 0.9256410256410257, + "Single-session (assistant)": 1.0 + }, + "category_recall10": { + "Single-session (user)": 0.9714285714285714, + "Multi-session reasoning": 1.0, + "Single-session (preference)": 0.9, + "Temporal reasoning": 0.9774436090225563, + "Knowledge updates": 1.0, + "Single-session (assistant)": 1.0 + }, + "elapsed_s": 2264.9849442080013, + "consolidation_total_wall_s": 0.0, + "consolidation_call_count": 0, + "manifest": { + "with_consolidation": false, + "with_consolidation_note": "Scores collected with consolidation=False do NOT reflect production behaviour. Consolidation-only mechanisms (CASCADE, INTERFERENCE, HOMEOSTATIC_PLASTICITY, SYNAPTIC_PLASTICITY, MICROGLIAL_PRUNING, TWO_STAGE_MODEL, EMOTIONAL_DECAY, TRIPARTITE_SYNAPSE, SCHEMA_ENGINE) are exercised only when with_consolidation=True. Delta between the two conditions is unmeasured in this run.", + "ablate_mechanism": null, + "ablate_env_var": null, + "n_questions": 500, + "n_runs": 1, + "consolidation_call_count": 0, + "consolidation_total_wall_s": 0.0, + "repro": { + "git_commit": "1501428524f04408053ce6b54a5941df1e4b807b", + "git_dirty": true, + "python_version": "3.13.12 (main, Feb 3 2026, 17:53:27) [Clang 17.0.0 (clang-1700.6.3.2)]", + "platform_system": "Darwin", + "platform_machine": "arm64", + "platform_node": "PTECH-MBP-cdeust", + "timestamp_utc": "2026-07-03T15:30:35.074124+00:00", + "lib_versions": { + "sentence-transformers": "5.4.1", + "torch": "2.11.0", + "numpy": "2.4.4", + "psycopg": "3.3.3", + "pgvector": "0.4.2", + "flashrank": "0.2.10" + } + } + } +} \ No newline at end of file