From 2e5dd7b8085d76d96d55c80a297d08ef46034fa9 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 12 Sep 2026 08:16:00 -0700 Subject: [PATCH 1/3] feat(graph): improve community detection quality --- CHANGELOG.md | 10 +- COMPATIBILITY.md | 32 + Cargo.lock | 1 + MIGRATION.md | 27 + PERFORMANCE.md | 25 + crates/compass-cli/src/history_build.rs | 107 +- crates/compass-cli/src/label_commands.rs | 9 +- crates/compass-cli/src/lib.rs | 14 +- crates/compass-cli/tests/history_cli.rs | 85 +- crates/compass-core/src/build_state.rs | 64 + crates/compass-core/src/cluster_existing.rs | 197 ++- crates/compass-core/src/pipeline.rs | 132 +- .../community_quality_qualification.rs | 671 ++++++++++ crates/compass-graph/src/analyze.rs | 36 +- crates/compass-graph/src/cluster.rs | 106 +- .../compass-graph/src/community/artifact.rs | 351 +++++ crates/compass-graph/src/community/build.rs | 1174 +++++++++++++++++ .../compass-graph/src/community/identity.rs | 22 + .../src/community/incremental.rs | 322 +++++ crates/compass-graph/src/community/leiden.rs | 470 +++++++ crates/compass-graph/src/community/mod.rs | 31 + crates/compass-graph/src/community/quality.rs | 806 +++++++++++ .../compass-graph/src/community/topology.rs | 573 ++++++++ crates/compass-graph/src/lib.rs | 14 + crates/compass-history/Cargo.toml | 1 + crates/compass-history/src/artifacts.rs | 172 ++- crates/compass-history/tests/roundtrip.rs | 62 + crates/compass-mcp/tests/code_query_tools.rs | 1 + crates/compass-output/src/lib.rs | 4 + crates/compass-output/src/viewer_model.rs | 17 +- docs/README.md | 3 + docs/concepts/community-detection.md | 93 ++ ...unity-detection-quality-qualification.json | 704 ++++++++++ ...mmunity-detection-quality-qualification.md | 89 ++ ...nity-detection-quality-technical-design.md | 1108 ++++++++++++++++ docs/reference/commands.md | 16 + docs/reference/outputs.md | 23 + scripts/qualify_code_graph_v1.sh | 40 + 38 files changed, 7434 insertions(+), 178 deletions(-) create mode 100644 crates/compass-graph/examples/community_quality_qualification.rs create mode 100644 crates/compass-graph/src/community/artifact.rs create mode 100644 crates/compass-graph/src/community/build.rs create mode 100644 crates/compass-graph/src/community/identity.rs create mode 100644 crates/compass-graph/src/community/incremental.rs create mode 100644 crates/compass-graph/src/community/leiden.rs create mode 100644 crates/compass-graph/src/community/mod.rs create mode 100644 crates/compass-graph/src/community/quality.rs create mode 100644 crates/compass-graph/src/community/topology.rs create mode 100644 docs/concepts/community-detection.md create mode 100644 docs/implementation/community-detection-quality-qualification.json create mode 100644 docs/implementation/community-detection-quality-qualification.md create mode 100644 docs/implementation/community-detection-quality-technical-design.md diff --git a/CHANGELOG.md b/CHANGELOG.md index f8a56d02a..0ce9e0bf5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,15 @@ witness relationships without printing opaque graph-node IDs. Canonical JSON and SARIF retain every exact identity and remain unchanged. +- Replace production community detection for typed graphs with deterministic + native Leiden over a versioned typed-evidence topology. Publish strict, + digest-bound `compass.community-quality/1` evidence, preserve frozen + influence and full-quality fallback during incremental updates, and retain + fixed resolution as the default because the bounded three-candidate selector + did not meet its clustering-time gate. Community membership and graph-local + IDs may change; Base Graph nodes, relationships, direction, multiplicity, + provenance, and `compass.graph/1` remain unchanged. + ## 0.3.24 - 2026-09-11 - Add `compass ensure` as an idempotent agent-session and linked-worktree @@ -28,7 +37,6 @@ edge-ordered adjacency capability; older sidecars remain valid recovery inputs but directional store queries fail with an explicit rebuild instruction instead of returning a backend-dependent truncated subset. - - Refactor universal language metadata around `UniversalEvidenceProducer` and `UniversalEvidencePipeline`. `UniversalCandidate`/`UniversalComplete` are now the clearer lifecycle states `Qualifying`/`Qualified`; the serialized diff --git a/COMPATIBILITY.md b/COMPATIBILITY.md index fbe4a2e46..27807d608 100644 --- a/COMPATIBILITY.md +++ b/COMPATIBILITY.md @@ -614,6 +614,38 @@ The `extract --code-only` profile excludes document extractors from structural node and edge publication while retaining the scanned file inventory and its status records. +## Community detection profile cutover + +Typed clustered graphs use the complete profile +`seeded-leiden-modularity/v1` + `typed-evidence-undirected/v1` + +`community-quality/v1` + `fixed-resolution/v1`, seed `42`, and +`community-limits/v1`. The default resolution is fixed at `1`; an explicit +`--resolution N` remains a single fixed positive finite resolution. The +bounded three-candidate selector has identity `bounded-multiresolution/v1` but +remains qualification-only because it exceeded the clustering-time acceptance +gate. + +This is a compatibility-sensitive membership cutover without a +`compass.graph/1` schema change. Community numeric IDs, membership, labels, +reports, and architecture groupings may change. Base Graph node and edge +identity, direction, multiplicity, anchors, provenance, and canonical encoding +do not change as a consequence of clustering. The complete profile enters the +configuration digest and current/history build profiles, so old output is +rebuilt coherently rather than partially reused. + +Clustered typed builds add strict `compass.community-quality/1` at +`community-quality.json`. Readers must validate its self-digest, graph +generation, exact canonical graph digest, and profile identity and reject +unknown majors or fields. Missing evidence on an older, schema-less legacy, or +unclustered graph means unavailable. Direct reclustering of a schema-less +legacy graph retains `seeded-louvain/v1` compatibility and publishes no quality +sidecar. + +Historical realizations and their sidecars are immutable. Compass never +substitutes Louvain results under a Leiden profile or interprets one profile's +member IDs as another profile's result. Existing `cohesion` remains the public +density projection, now calculated by the shared quality evaluator. + ## Compass Store release contract The first supported local store line is `0.3.x`. Its logical machine formats diff --git a/Cargo.lock b/Cargo.lock index 13ec9b5be..c414bd5e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1358,6 +1358,7 @@ dependencies = [ "atomicwrites", "compass-analysis", "compass-files", + "compass-graph", "compass-ir", "compass-model", "prolly-map", diff --git a/MIGRATION.md b/MIGRATION.md index 7e285587a..f73ea261f 100644 --- a/MIGRATION.md +++ b/MIGRATION.md @@ -25,6 +25,33 @@ Use `--engine json` until the rebuild completes. Do not edit or copy SQLite tables to add the capability marker; the directional index key order must be rebuilt from the validated graph. +## Rebuild communities for the Leiden profile + +Typed clustered graphs now use `seeded-leiden-modularity/v1` over +`typed-evidence-undirected/v1` with the fixed-resolution selector. Run a normal +forced build after upgrading: + +```bash +compass update --force +``` + +Community membership, numeric IDs, labels, architecture groupings, and derived +reports may change. Do not copy old `community` attributes or label signatures +into the new graph. Base Graph node and relationship identity, direction, +multiplicity, anchors, and provenance are unchanged by clustering. + +Omitting `--resolution` uses fixed resolution `1`; an explicit +`--resolution N` uses exactly one positive finite value. Automatic +multi-resolution selection is not a production default. Clustered typed builds +add `community-quality.json`, bound to the exact `graph.json`. Upgrade strict +artifact readers to accept `compass.community-quality/1` and reject unknown +majors, unknown fields, digest mismatch, or profile mismatch. Missing evidence +on an older or legacy graph means unavailable, not zero quality. + +Published historical realizations remain immutable. New materializations use +the complete Leiden profile fingerprint; Compass does not rewrite or silently +reinterpret older memberships. + ## Frontend graph vocabulary Recent pre-release builds can add React-oriented `renders` edges and UI/server diff --git a/PERFORMANCE.md b/PERFORMANCE.md index 184d43e19..e3176f8a7 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -1242,6 +1242,31 @@ three-sample median of about 2,353 MiB. These native-volume measurements avoid the multi-second publication variance observed on the mounted workspace, but remain runner-specific rather than a cross-platform guarantee. +## Community detection qualification + +The version-1 community-quality runner evaluates 15 deterministic fixture +families, including input permutations, planted partitions, articulation, hub, +direction, confidence, containment, isolate, and resolution-limit cases: + +```bash +./scripts/qualify_code_graph_v1.sh --community-quality +``` + +On 2026-09-12, an aarch64 macOS debug build ran the complete compact fixture +set 50 times per process. Seven-process medians were 1.25 seconds for +compatibility Louvain, 1.82 seconds for fixed-resolution typed Leiden, and +3.01 seconds for three-candidate typed Leiden. Compact setup-heavy debug +timings are diagnostic, not the pinned real-repository release oracle, and no +RSS or cold-build claim is derived from them. + +The fixture quality gates pass, including deterministic equality, connected +communities, exact required recovery, and improved ring-of-cliques and +articulation recovery. Production nevertheless uses fixed resolution; the +automatic selector remains qualification-only until pinned-corpus clustering, +cold-build, RSS, and incremental gates pass. See the +[qualification report](docs/implementation/community-detection-quality-qualification.md) +for exact fixture results and omissions. + ## Versioned history qualification Build a release binary, then measure a clean real repository: diff --git a/crates/compass-cli/src/history_build.rs b/crates/compass-cli/src/history_build.rs index edb0130ea..eef81f929 100644 --- a/crates/compass-cli/src/history_build.rs +++ b/crates/compass-cli/src/history_build.rs @@ -10,6 +10,10 @@ use compass_core::{ build_graph_with_layers_retained, }; use compass_files::{DetectOptions, IgnorePolicy, Manifest, ManifestKind, ProjectConfig, detect}; +use compass_graph::{ + COMPATIBILITY_CLUSTER_SEED_TEXT, COMPATIBILITY_CLUSTER_SELECTOR, QUALITY_CLUSTER_ALGORITHM, + QUALITY_CLUSTER_LIMITS, QUALITY_CLUSTER_QUALITY, QUALITY_CLUSTER_TOPOLOGY, +}; use compass_history::{ BuildProfile, CompletedGraphArtifacts, CompletionEvidence, GraphArtifacts, HISTORY_GRAPH_SCHEMA, HistoryError, MAX_DIAGNOSTIC_BYTES, @@ -91,13 +95,15 @@ impl HistoryBuildOptions { "--token-budget", "default", ); - push_profile_option( - &profile, - &mut forwarded, - "resolution", - "--resolution", - "none", - ); + if profile.value("cluster_resolution_policy") == Some("fixed/v1") { + push_profile_option( + &profile, + &mut forwarded, + "resolution", + "--resolution", + "none", + ); + } push_profile_option( &profile, &mut forwarded, @@ -139,7 +145,10 @@ impl HistoryBuildOptions { )); } }; - insert_current_engine_profile(&mut profile, deep)?; + let excludes_hubs = profile + .value("exclude_hubs") + .is_some_and(|value| value != "none"); + insert_current_engine_profile(&mut profile, deep, excludes_hubs)?; if profile.value("ocr_mode").is_none() { profile.insert("ocr_mode", "off")?; } @@ -199,7 +208,7 @@ impl HistoryBuildOptions { resolve_provider(&mut values)?; } let mut profile = BuildProfile::default(); - insert_current_engine_profile(&mut profile, values.deep)?; + insert_current_engine_profile(&mut profile, values.deep, values.exclude_hubs.is_some())?; for (key, value) in [ ("gitignore", values.gitignore.to_string()), ("code_only", values.code_only.to_string()), @@ -307,6 +316,9 @@ impl HistoryBuildOptions { for exclude in &values.excludes { forwarded.extend(["--exclude".to_owned(), exclude.clone()]); } + // The qualified production profile is fixed-resolution Leiden even + // when the user omitted the flag. Forward the resolved value so a + // historical subprocess materializes the exact persisted profile. forwarded.extend([ "--resolution".to_owned(), normalized_float(values.resolution), @@ -332,6 +344,7 @@ impl HistoryBuildOptions { fn insert_current_engine_profile( profile: &mut BuildProfile, deep: bool, + excludes_hubs: bool, ) -> Result<(), HistoryError> { for (key, value) in [ ("compass_version", env!("CARGO_PKG_VERSION").to_owned()), @@ -428,8 +441,25 @@ fn insert_current_engine_profile( ), ("enabled_features", "workspace-default".to_owned()), ("direction", "native-source-semantics".to_owned()), - ("cluster_algorithm", "seeded-louvain/v1".to_owned()), - ("cluster_seed", "42".to_owned()), + ("cluster_algorithm", QUALITY_CLUSTER_ALGORITHM.to_owned()), + ("cluster_seed", COMPATIBILITY_CLUSTER_SEED_TEXT.to_owned()), + ("cluster_topology", QUALITY_CLUSTER_TOPOLOGY.to_owned()), + ("cluster_quality", QUALITY_CLUSTER_QUALITY.to_owned()), + ( + "cluster_selector", + COMPATIBILITY_CLUSTER_SELECTOR.to_owned(), + ), + ("cluster_resolution_policy", "fixed/v1".to_owned()), + ( + "cluster_hub_policy", + if excludes_hubs { + "exclude-percentile/v1" + } else { + "none/v1" + } + .to_owned(), + ), + ("cluster_limits_version", QUALITY_CLUSTER_LIMITS.to_owned()), ( "semantic_prompt_sha256", compass_semantic::extraction_prompt_sha256(deep), @@ -477,6 +507,12 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError | "direction" | "cluster_algorithm" | "cluster_seed" + | "cluster_topology" + | "cluster_quality" + | "cluster_selector" + | "cluster_resolution_policy" + | "cluster_hub_policy" + | "cluster_limits_version" | "gitignore" | "code_only" | "cargo" @@ -515,8 +551,11 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError ("program_provider_policy", "offline-artifacts-first"), ("enabled_features", "workspace-default"), ("direction", "native-source-semantics"), - ("cluster_algorithm", "seeded-louvain/v1"), - ("cluster_seed", "42"), + ("cluster_algorithm", QUALITY_CLUSTER_ALGORITHM), + ("cluster_seed", COMPATIBILITY_CLUSTER_SEED_TEXT), + ("cluster_topology", QUALITY_CLUSTER_TOPOLOGY), + ("cluster_quality", QUALITY_CLUSTER_QUALITY), + ("cluster_limits_version", QUALITY_CLUSTER_LIMITS), ] { if profile.value(key) != Some(expected) { return Err(HistoryError::InvalidFingerprint(format!( @@ -524,6 +563,29 @@ fn validate_persisted_profile(profile: &BuildProfile) -> Result<(), HistoryError ))); } } + if profile.value("cluster_resolution_policy") != Some("fixed/v1") { + return Err(HistoryError::InvalidFingerprint( + "persisted cluster_resolution_policy is incompatible with fixed/v1".to_owned(), + )); + } + if profile.value("cluster_selector") != Some(COMPATIBILITY_CLUSTER_SELECTOR) { + return Err(HistoryError::InvalidFingerprint(format!( + "persisted cluster_selector is incompatible with {COMPATIBILITY_CLUSTER_SELECTOR}" + ))); + } + let expected_hub_policy = if profile + .value("exclude_hubs") + .is_some_and(|value| value != "none") + { + "exclude-percentile/v1" + } else { + "none/v1" + }; + if profile.value("cluster_hub_policy") != Some(expected_hub_policy) { + return Err(HistoryError::InvalidFingerprint(format!( + "persisted cluster_hub_policy is incompatible with {expected_hub_policy}" + ))); + } for (key, expected) in [ ( "document_raw_bytes_limit", @@ -849,6 +911,7 @@ struct HistoryBuildValues { ocr_languages: Vec, token_budget: Option, resolution: f64, + resolution_explicit: bool, exclude_hubs: Option, gitignore: bool, excludes: Vec, @@ -872,6 +935,7 @@ impl Default for HistoryBuildValues { ocr_languages: Vec::new(), token_budget: None, resolution: 1.0, + resolution_explicit: false, exclude_hubs: None, gitignore: true, excludes: Vec::new(), @@ -956,7 +1020,10 @@ pub(crate) fn parse_build_command( values.ocr_profile = value.to_owned(); } "--token-budget" => values.token_budget = Some(positive_usize(name, value)?), - "--resolution" => values.resolution = positive_float(name, value)?, + "--resolution" => { + values.resolution = positive_float(name, value)?; + values.resolution_explicit = true; + } "--exclude-hubs" => values.exclude_hubs = Some(finite_float(name, value)?), "--format" => format = Some(value.to_owned()), "--profile-from" => profile_from = Some(nonempty(name, value)?.to_owned()), @@ -1418,6 +1485,7 @@ impl NativeCompleteGraphBuilder { .value("resolution") .and_then(|value| value.parse().ok()) .unwrap_or(1.0); + options.resolution_explicit = true; options.exclude_hubs = self .profile .value("exclude_hubs") @@ -1451,12 +1519,21 @@ impl NativeCompleteGraphBuilder { })?; let manifest = serde_json::from_slice(&manifest_bytes) .map_err(|error| MaterializeError::Builder(error.to_string()))?; - let artifacts = GraphArtifacts::from_trusted( + let mut artifacts = GraphArtifacts::from_trusted( retained.document, retained.program, retained.analysis, Some(manifest), )?; + artifacts.authoritative_sidecars.insert( + "community-quality.json".to_owned(), + fs::read(result.output_dir.join("community-quality.json")).map_err(|source| { + compass_files::FileError::Io { + path: result.output_dir.join("community-quality.json"), + source, + } + })?, + ); let code_files = result .detection .files diff --git a/crates/compass-cli/src/label_commands.rs b/crates/compass-cli/src/label_commands.rs index 213db76ed..c3377ff9e 100644 --- a/crates/compass-cli/src/label_commands.rs +++ b/crates/compass-cli/src/label_commands.rs @@ -25,6 +25,7 @@ struct LabelArguments { backend: Option, model: Option, resolution: f64, + resolution_explicit: bool, exclude_hubs: Option, max_concurrency: usize, batch_size: usize, @@ -100,6 +101,7 @@ pub(super) fn command_label(_frontend: Frontend, args: &[String]) -> Outcome { no_viz: parsed.no_viz, no_label: false, resolution: parsed.resolution, + resolution_explicit: parsed.resolution_explicit, exclude_hubs: parsed.exclude_hubs, min_community_size: parsed.min_community_size, }; @@ -418,6 +420,7 @@ fn parse_arguments(args: &[String]) -> Result { backend: None, model: None, resolution: 1.0, + resolution_explicit: false, exclude_hubs: None, max_concurrency: 4, batch_size: 100, @@ -437,7 +440,10 @@ fn parse_arguments(args: &[String]) -> Result { "--graph" => parsed.graph_override = Some(PathBuf::from(value)), "--backend" => parsed.backend = Some(value.clone()), "--model" => parsed.model = Some(value.clone()), - "--resolution" => parsed.resolution = parse_number(value, argument)?, + "--resolution" => { + parsed.resolution = parse_number(value, argument)?; + parsed.resolution_explicit = true; + } "--exclude-hubs" => parsed.exclude_hubs = Some(parse_number(value, argument)?), "--max-concurrency" => { parsed.max_concurrency = parse_positive(value, argument)? @@ -457,6 +463,7 @@ fn parse_arguments(args: &[String]) -> Result { value if value.starts_with("--model=") => parsed.model = Some(value[8..].to_owned()), value if value.starts_with("--resolution=") => { parsed.resolution = parse_number(&value[13..], "--resolution")?; + parsed.resolution_explicit = true; } value if value.starts_with("--exclude-hubs=") => { parsed.exclude_hubs = Some(parse_number(&value[15..], "--exclude-hubs")?); diff --git a/crates/compass-cli/src/lib.rs b/crates/compass-cli/src/lib.rs index 3c9130aac..87bceefb7 100644 --- a/crates/compass-cli/src/lib.rs +++ b/crates/compass-cli/src/lib.rs @@ -1347,6 +1347,7 @@ fn command_cluster_only(_frontend: Frontend, args: &[String]) -> Outcome { let mut no_label = false; let mut timing = false; let mut resolution = 1.0; + let mut resolution_explicit = false; let mut exclude_hubs = None; let mut min_community_size = 3_usize; let mut index = 0; @@ -1370,6 +1371,7 @@ fn command_cluster_only(_frontend: Frontend, args: &[String]) -> Outcome { return Outcome::failure("error: --resolution requires a number".to_owned()); }; resolution = value; + resolution_explicit = true; index += 1; } value if value.starts_with("--resolution=") => { @@ -1377,6 +1379,7 @@ fn command_cluster_only(_frontend: Frontend, args: &[String]) -> Outcome { return Outcome::failure("error: --resolution requires a number".to_owned()); }; resolution = parsed; + resolution_explicit = true; } "--exclude-hubs" => { let Some(argument) = args.get(index + 1) else { @@ -1403,7 +1406,7 @@ fn command_cluster_only(_frontend: Frontend, args: &[String]) -> Outcome { min_community_size = parsed; } "-h" | "--help" => { - return Outcome::success("Usage: compass cluster-only [PATH] [--graph PATH] [--no-viz] [--no-label] [--resolution N] [--exclude-hubs N] [--min-community-size=N]".to_owned()); + return Outcome::success("Usage: compass cluster-only [PATH] [--graph PATH] [--no-viz] [--no-label] [--resolution N] [--exclude-hubs N] [--min-community-size=N]\nCommunity resolution: omission uses fixed resolution 1; --resolution N uses exactly N. Automatic multi-resolution selection is qualification-only.".to_owned()); } value if value.starts_with('-') => { return Outcome::failure(format!( @@ -1449,6 +1452,7 @@ fn command_cluster_only(_frontend: Frontend, args: &[String]) -> Outcome { no_viz, no_label, resolution, + resolution_explicit, exclude_hubs, min_community_size, }) { @@ -1781,6 +1785,7 @@ fn command_build_with_validation_inner( let mut excludes = Vec::new(); let mut program_artifacts = Vec::new(); let mut resolution = 1.0; + let mut resolution_explicit = false; let mut exclude_hubs = None; let mut index = 0; while index < args.len() { @@ -2005,6 +2010,7 @@ fn command_build_with_validation_inner( Ok(value) => value, Err(error) => return extract_parse_failure(frontend, error), }; + resolution_explicit = true; index += 1; } value if value.starts_with("--resolution=") => { @@ -2012,6 +2018,7 @@ fn command_build_with_validation_inner( Ok(value) => value, Err(error) => return extract_parse_failure(frontend, error), }; + resolution_explicit = true; } "--exclude-hubs" if index + 1 < args.len() => { let Ok(value) = args[index + 1].parse::() else { @@ -2070,7 +2077,7 @@ fn command_build_with_validation_inner( extract_help() } else { format!( - "Usage: compass {} [path] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--max-source-bytes N] [--max-workers N] [--no-cluster] [--force] [--no-viz] [--timing]", + "Usage: compass {} [path] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--max-source-bytes N] [--max-workers N] [--no-cluster] [--force] [--no-viz] [--timing] [--resolution N]\nCommunity resolution: omission uses fixed resolution 1; --resolution N uses exactly N. Automatic multi-resolution selection is qualification-only.", operation.label() ) }); @@ -2145,6 +2152,7 @@ fn command_build_with_validation_inner( } options.extra_excludes = excludes; options.resolution = resolution; + options.resolution_explicit = resolution_explicit; options.exclude_hubs = exclude_hubs; options.code_only = code_only; options.purpose = if extract { @@ -3055,7 +3063,7 @@ fn executable_on_path(name: &str) -> bool { } fn extract_help() -> String { - "Usage: compass extract [PATH] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--code-only] [--cargo] [--google-workspace] [--postgres DSN] [--backend NAME] [--model MODEL] [--mode deep] [--ocr off|auto|always] [--ocr-profile NAME] [--ocr-language BCP47] [--token-budget N] [--max-concurrency N] [--max-workers N] [--max-source-bytes N] [--api-timeout SECONDS] [--allow-partial] [--dedup-llm] [--timing] [--out DIR] [--no-cluster] [--force] [--no-viz] [--no-gitignore] [--exclude PATTERN] [--resolution N] [--exclude-hubs N]\nProvider selection: --backend/--model override COMPASS_BACKEND/COMPASS_MODEL. Built-ins: claude, kimi, ollama, gemini, openai, deepseek, azure, bedrock, claude-cli. Set the selected provider's documented credential variable; custom providers use `compass provider add`. Credentials are never written to Compass artifacts.".to_owned() + "Usage: compass extract [PATH] [--program] [--program-artifact PATH] [--no-program] [--store json|sqlite] [--inference-level low|medium|high|max] [--code-only] [--cargo] [--google-workspace] [--postgres DSN] [--backend NAME] [--model MODEL] [--mode deep] [--ocr off|auto|always] [--ocr-profile NAME] [--ocr-language BCP47] [--token-budget N] [--max-concurrency N] [--max-workers N] [--max-source-bytes N] [--api-timeout SECONDS] [--allow-partial] [--dedup-llm] [--timing] [--out DIR] [--no-cluster] [--force] [--no-viz] [--no-gitignore] [--exclude PATTERN] [--resolution N] [--exclude-hubs N]\nCommunity resolution: omission uses fixed resolution 1; --resolution N uses exactly N. Automatic multi-resolution selection is qualification-only.\nProvider selection: --backend/--model override COMPASS_BACKEND/COMPASS_MODEL. Built-ins: claude, kimi, ollama, gemini, openai, deepseek, azure, bedrock, claude-cli. Set the selected provider's documented credential variable; custom providers use `compass provider add`. Credentials are never written to Compass artifacts.".to_owned() } fn saved_graph_root() -> Option { diff --git a/crates/compass-cli/tests/history_cli.rs b/crates/compass-cli/tests/history_cli.rs index 913075bd2..c9431bfad 100644 --- a/crates/compass-cli/tests/history_cli.rs +++ b/crates/compass-cli/tests/history_cli.rs @@ -51,8 +51,14 @@ fn current_history_profile() -> Result Result, + #[serde(default = "default_cluster_algorithm")] + pub cluster_algorithm: String, + #[serde(default = "default_cluster_topology")] + pub cluster_topology: String, + #[serde(default = "default_cluster_quality")] + pub cluster_quality: String, + #[serde(default = "default_cluster_selector")] + pub cluster_selector: String, + #[serde(default = "default_cluster_seed")] + pub cluster_seed: u32, + #[serde(default = "default_resolution_policy")] + pub cluster_resolution_policy: String, + #[serde(default = "default_hub_policy")] + pub cluster_hub_policy: String, + #[serde(default = "default_cluster_limits")] + pub cluster_limits_version: String, #[serde(default)] pub code_only: bool, pub program_analysis: bool, @@ -106,6 +122,38 @@ pub(crate) struct BuildProfile { pub document_processing_identity: String, } +fn default_cluster_algorithm() -> String { + compass_graph::COMPATIBILITY_CLUSTER_ALGORITHM.to_owned() +} + +fn default_cluster_topology() -> String { + compass_graph::COMPATIBILITY_CLUSTER_TOPOLOGY.to_owned() +} + +fn default_cluster_quality() -> String { + compass_graph::COMPATIBILITY_CLUSTER_QUALITY.to_owned() +} + +fn default_cluster_selector() -> String { + compass_graph::COMPATIBILITY_CLUSTER_SELECTOR.to_owned() +} + +const fn default_cluster_seed() -> u32 { + compass_graph::COMPATIBILITY_CLUSTER_SEED +} + +fn default_resolution_policy() -> String { + "fixed/v1".to_owned() +} + +fn default_hub_policy() -> String { + "none/v1".to_owned() +} + +fn default_cluster_limits() -> String { + compass_graph::COMPATIBILITY_CLUSTER_LIMITS.to_owned() +} + // Build-state schema 1 omitted the historical max profile. Keep interpreting // an absent field as max even though new builds default to low. New low // profiles serialize the field explicitly, so the first build after the @@ -298,6 +346,14 @@ mod tests { no_viz: true, resolution: 1.0, exclude_hubs: None, + cluster_algorithm: default_cluster_algorithm(), + cluster_topology: default_cluster_topology(), + cluster_quality: default_cluster_quality(), + cluster_selector: default_cluster_selector(), + cluster_seed: default_cluster_seed(), + cluster_resolution_policy: default_resolution_policy(), + cluster_hub_policy: default_hub_policy(), + cluster_limits_version: default_cluster_limits(), code_only: true, program_analysis: false, graph_storage: "json".to_owned(), @@ -333,6 +389,14 @@ mod tests { no_viz: true, resolution: 1.0, exclude_hubs: None, + cluster_algorithm: default_cluster_algorithm(), + cluster_topology: default_cluster_topology(), + cluster_quality: default_cluster_quality(), + cluster_selector: default_cluster_selector(), + cluster_seed: default_cluster_seed(), + cluster_resolution_policy: default_resolution_policy(), + cluster_hub_policy: default_hub_policy(), + cluster_limits_version: default_cluster_limits(), code_only: false, program_analysis: true, graph_storage: "json".to_owned(), diff --git a/crates/compass-core/src/cluster_existing.rs b/crates/compass-core/src/cluster_existing.rs index 9ac37bb7c..e8f4311da 100644 --- a/crates/compass-core/src/cluster_existing.rs +++ b/crates/compass-core/src/cluster_existing.rs @@ -1,13 +1,15 @@ -use std::collections::{BTreeMap, HashMap}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::fs; use std::path::{Path, PathBuf}; use std::time::{Duration, Instant}; use compass_files::{BuildGuard, write_atomic_with_digest, write_json_atomic, write_text_atomic}; use compass_graph::{ - ClusterOptions, Communities, GodNode, blind_spot_report, cluster, community_member_signatures, - god_nodes, label_communities_by_hub, remap_communities_to_previous, score_communities, - suggest_questions, surprising_connections, write_canonical_graph_json, + ClusterOptions, Communities, CommunityLimits, CommunityProfile, CommunityQualityArtifact, + CommunityRequest, GodNode, ResolutionPolicy, blind_spot_report, build_communities, cluster, + community_member_signatures, god_nodes, label_communities_by_hub, + remap_communities_to_previous, score_communities, suggest_questions, surprising_connections, + write_canonical_graph_json, }; use compass_model::GraphDocument; use compass_model::GraphError; @@ -31,6 +33,7 @@ pub struct ClusterExistingOptions { pub no_viz: bool, pub no_label: bool, pub resolution: f64, + pub resolution_explicit: bool, pub exclude_hubs: Option, pub min_community_size: usize, } @@ -158,17 +161,43 @@ where }) .collect::>(); let cluster_started = Instant::now(); - let fresh = cluster( - document, - ClusterOptions { - resolution: options.resolution, - exclude_hubs_percentile: options.exclude_hubs, - }, - ); - let communities = if previous.is_empty() { - fresh + let community_limits = CommunityLimits::default(); + let (communities, quality_evidence) = if let Some(typed) = typed_document.as_ref() { + let changed_sources = BTreeSet::new(); + let result = build_communities( + typed, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Fixed(options.resolution), + exclude_hubs_percentile: options.exclude_hubs, + previous: (!previous.is_empty()).then_some(&previous), + incremental: false, + changed_sources: &changed_sources, + limits: community_limits, + }, + )?; + ( + result.communities, + Some(( + typed.graph.build.generation_id.clone(), + result.identity, + result.quality, + )), + ) } else { - remap_communities_to_previous(&fresh, &previous) + let fresh = cluster( + document, + ClusterOptions { + resolution: options.resolution, + exclude_hubs_percentile: options.exclude_hubs, + }, + ); + let communities = if previous.is_empty() { + fresh + } else { + remap_communities_to_previous(&fresh, &previous) + }; + (communities, None) }; let cluster_elapsed = cluster_started.elapsed(); let analyze_started = Instant::now(); @@ -277,6 +306,7 @@ where }), true, )?; + let publishes_quality = quality_evidence.is_some(); let graph_path = staging.join("graph.json"); let graph_identity = if let Some(typed) = typed_document { let receipt = write_atomic_with_digest(&graph_path, |writer| { @@ -301,6 +331,16 @@ where )?; graph_artifact_identity(&graph_path)? }; + if let Some((generation, identity, quality)) = quality_evidence { + let artifact = CommunityQualityArtifact::new( + generation, + graph_identity.clone(), + identity, + community_limits, + quality, + )?; + write_json_atomic(staging.join("community-quality.json"), &artifact, true)?; + } orientation.evidence_status.artifact_set_identity = Some(graph_identity); let report = render_agent_report_markdown(&orientation, report_options.obsidian)?; let orientation_json = render_orientation_json(&orientation)?; @@ -345,6 +385,9 @@ where if html_written { artifacts.push("graph.html"); } + if publishes_quality { + artifacts.push("community-quality.json"); + } guard.commit_with_artifacts(&artifacts)?; BuildGuard::publish_root_artifacts( &output_container, @@ -357,6 +400,7 @@ where "graph-overview.json", "graph.html", "graph.json", + "community-quality.json", ], true, )?; @@ -474,7 +518,13 @@ mod tests { use std::error::Error; use std::fs::OpenOptions; + use compass_model::code_graph::{ + BuildMetadata, EdgeKind, EdgeRecord, ExtractionStatus, FileRecord, NodeKind, NodeRecord, + }; + use compass_model::identity::file_id; + use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin, Provenance, SourceAnchor}; use serde_json::Value; + use sha2::{Digest, Sha256}; use tempfile::TempDir; use super::*; @@ -562,6 +612,36 @@ mod tests { Ok(()) } + #[test] + fn typed_cluster_only_publishes_graph_bound_fixed_quality_evidence() + -> Result<(), Box> { + let fixture = managed_typed_graph_fixture()?; + cluster_existing_graph(&fixture.options)?; + let current = BuildGuard::resolve_current_snapshot_directory(&fixture.output)?; + let graph_bytes = fs::read(current.join("graph.json"))?; + let graph: V1GraphDocument = serde_json::from_slice(&graph_bytes)?; + let quality_bytes = fs::read(current.join("community-quality.json"))?; + assert_eq!( + quality_bytes, + fs::read(fixture.output.join("community-quality.json"))? + ); + let quality: CommunityQualityArtifact = serde_json::from_slice(&quality_bytes)?; + quality.validate_for_graph( + &graph.graph.build.generation_id, + &format!("sha256:{:x}", Sha256::digest(&graph_bytes)), + )?; + assert_eq!( + quality.identity.algorithm, + compass_graph::QUALITY_CLUSTER_ALGORITHM + ); + assert_eq!( + quality.identity.selector, + compass_graph::COMPATIBILITY_CLUSTER_SELECTOR + ); + assert_eq!(quality.partition.candidate_summaries.len(), 1); + Ok(()) + } + #[test] fn cluster_only_publishes_disambiguated_community_labels_in_graph_report() -> Result<(), Box> { @@ -702,6 +782,94 @@ mod tests { ) } + fn managed_typed_graph_fixture() -> Result> { + let digest = format!("sha256:{}", "0".repeat(64)); + let mut document = V1GraphDocument::empty_v1(BuildMetadata { + builder_version: "test".to_owned(), + schema_fingerprint: digest.clone(), + source_tree_digest: digest.clone(), + configuration_digest: digest.clone(), + generation_id: digest.clone(), + source_commit: None, + }); + document.graph.files.push(FileRecord { + id: file_id("src/lib.rs"), + path: "src/lib.rs".to_owned(), + language: Some("rust".to_owned()), + content_digest: digest.clone(), + byte_size: 2, + generated: false, + extraction_status: ExtractionStatus::Extracted, + extractor_versions: vec!["cluster-existing-test".to_owned()], + coverage: Vec::new(), + diagnostics: Vec::new(), + }); + let anchor = |index: u64| SourceAnchor { + file: "src/lib.rs".to_owned(), + start_byte: index, + end_byte: index + 1, + start_line: 1, + start_column: u32::try_from(index).unwrap_or_default(), + end_line: 1, + end_column: u32::try_from(index + 1).unwrap_or(u32::MAX), + }; + let evidence = |anchor: SourceAnchor| Provenance { + origin: EvidenceOrigin::Ast, + extractor: "cluster-existing-test".to_owned(), + confidence: EvidenceConfidence::Exact, + rule: None, + anchors: vec![anchor], + wiring_site: None, + score: None, + candidates: Vec::new(), + }; + document.nodes = ["a", "b"] + .into_iter() + .enumerate() + .map(|(index, id)| NodeRecord { + id: id.to_owned(), + kind: NodeKind::Function, + roles: Vec::new(), + name: id.to_owned(), + qualified_name: id.to_owned(), + language: Some("rust".to_owned()), + framework: None, + source: Some(anchor(u64::try_from(index).unwrap_or(u64::MAX - 1))), + details: None, + evidence: vec![evidence(anchor( + u64::try_from(index).unwrap_or(u64::MAX - 1), + ))], + coverage: Vec::new(), + diagnostics: Vec::new(), + community: None, + }) + .collect(); + let relationship_site = anchor(0); + let edge_id = compass_model::identity::edge_id( + "a", + EdgeKind::Calls, + "b", + Some(&relationship_site), + None, + ); + document.links.push(EdgeRecord { + id: edge_id.clone(), + key: edge_id, + source: "a".to_owned(), + target: "b".to_owned(), + kind: EdgeKind::Calls, + occurrence_rule: None, + relationship_site: Some(relationship_site.clone()), + details: None, + evidence: vec![evidence(relationship_site)], + weight: Some(1.0), + context: None, + deferred: false, + diagnostics: Vec::new(), + }); + managed_graph_fixture_with_json(&serde_json::to_string(&document)?) + } + fn managed_graph_fixture_with_json( graph_json: &str, ) -> Result> { @@ -719,6 +887,7 @@ mod tests { no_viz: true, no_label: true, resolution: 1.0, + resolution_explicit: false, exclude_hubs: None, min_community_size: 1, }; diff --git a/crates/compass-core/src/pipeline.rs b/crates/compass-core/src/pipeline.rs index 98b139145..7ff7abe63 100644 --- a/crates/compass-core/src/pipeline.rs +++ b/crates/compass-core/src/pipeline.rs @@ -15,15 +15,16 @@ use compass_files::{ write_text_atomic, }; use compass_graph::{ - BuildEvidence, ClusterOptions, EntityTiebreaker, GRAPH_DIAGNOSTICS_EXTENSION, + BuildEvidence, CommunityExecution, CommunityLimits, CommunityProfile, CommunityQualityArtifact, + CommunityRequest, CommunityResult, EntityTiebreaker, GRAPH_DIAGNOSTICS_EXTENSION, GRAPH_JSON_DELTA_MAX_SOURCE_BYTES, GRAPH_SNAPSHOT_MAX_OBJECTS, - GRAPH_SNAPSHOT_SELECTOR_SCHEMA_V1, GraphSnapshotBuilder, GraphSnapshotGcStats, - IncrementalClusterLimits, InferenceLevel, InventoryEvidence, PublicationOmissions, - SnapshotSelector, SourceDigest, apply_inference_level, + GRAPH_SNAPSHOT_SELECTOR_SCHEMA_V1, GraphSnapshotBuilder, GraphSnapshotGcStats, InferenceLevel, + InventoryEvidence, PublicationOmissions, ResolutionPolicy, SnapshotSelector, SourceDigest, + apply_inference_level, build_communities, build_owned_with_tiebreaker_at_inference as build_document, canonical_edge_kind, - canonical_raw_edge_sites, cluster_incremental, deduped_node_count, extraction_from_v1, + canonical_raw_edge_sites, deduped_node_count, extraction_from_v1, garbage_collect_graph_snapshots, graph_insights_with_blind_spots, graph_snapshot_needs_gc, - label_communities_by_hub, normalize_document_v1_with_evidence_best_effort_owned_at_inference, + normalize_document_v1_with_evidence_best_effort_owned_at_inference, normalize_document_v1_with_inventory_and_source_digests_best_effort_owned_at_inference, normalize_document_v1_with_inventory_best_effort_at_inference, score_communities, write_canonical_graph_json, write_fact_neutral_graph_json_delta_prevalidated, @@ -97,7 +98,7 @@ const PARALLEL_AST_FACT_DIGEST_MIN_FILES: usize = 32; const SHARED_STORE_GC_MANIFEST_THRESHOLD: usize = 8; const STORE_SNAPSHOT_EXCLUSIONS: [&str; 3] = [STORE_FILE_NAME, "store.sqlite3-wal", "store.sqlite3-shm"]; -const ROOT_ARTIFACTS: [&str; 7] = [ +const ROOT_ARTIFACTS: [&str; 8] = [ "GRAPH_REPORT.md", "orientation.json", "graph-overview.json", @@ -105,6 +106,7 @@ const ROOT_ARTIFACTS: [&str; 7] = [ "manifest.json", "program.json", "graph.json", + "community-quality.json", ]; #[derive(Clone, Debug)] @@ -136,6 +138,8 @@ pub struct BuildOptions { pub extra_excludes: Vec, pub scope: BuildScope, pub resolution: f64, + /// Whether the user explicitly requested a fixed detector resolution. + pub resolution_explicit: bool, pub exclude_hubs: Option, pub google_workspace: bool, /// Restrict structural extraction to files classified as code. @@ -400,6 +404,7 @@ impl BuildOptions { extra_excludes: Vec::new(), scope: BuildScope::default(), resolution: 1.0, + resolution_explicit: false, exclude_hubs: None, google_workspace: false, code_only: false, @@ -2026,6 +2031,7 @@ fn publish_fact_neutral_incremental( let clustered = !options.no_cluster; if !clustered { remove_if_exists(&output_dir.join(GRAPH_OVERVIEW_FILE))?; + remove_if_exists(&output_dir.join("community-quality.json"))?; } save_output_stats( &output_dir, @@ -2195,6 +2201,10 @@ pub enum CoreError { #[error(transparent)] Dedup(#[from] compass_graph::DedupError), #[error(transparent)] + Community(#[from] compass_graph::CommunityError), + #[error(transparent)] + CommunityQualityArtifact(#[from] compass_graph::CommunityQualityArtifactError), + #[error(transparent)] Output(#[from] compass_output::OutputError), #[error("invalid cached AST extraction for {path}: {source}")] InvalidCache { @@ -3949,19 +3959,9 @@ fn build_graph_inner_unscoped( .edges .saturating_add(resolution_omitted_candidates); let report_health = current_orientation_health(options, omissions); - // Legacy clustering and report code needs a compatibility projection, but - // retaining it beside the complete typed authority doubles the dominant - // graph working set. Move records into the projection and reconstruct the - // strict authority after those consumers finish instead. - let document = published.document.into_legacy_document()?; - // A history realization must depend only on the target commit and build // profile. Prior community numbering is current-worktree operational state // and cannot influence the content-addressed result. - let cluster_options = ClusterOptions { - resolution: options.resolution, - exclude_hubs_percentile: options.exclude_hubs, - }; let previous_started = Instant::now(); let history_build = std::env::var_os("COMPASS_HISTORY_BUILD").is_some(); let previous = if history_build { @@ -3976,27 +3976,45 @@ fn build_graph_inner_unscoped( .map(|path| relative_fact_path(path, &root)) .collect::>(); let cluster_started = Instant::now(); - let clustered = cluster_incremental( - &document, - &previous, - &changed_sources, - cluster_options, - IncrementalClusterLimits::default(), - ); + let community_limits = CommunityLimits::default(); + let clustered = build_communities( + &published.document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + // The bounded three-candidate selector remains qualification-only: + // its measured clustering overhead exceeds the production gate. + resolution: ResolutionPolicy::Fixed(options.resolution), + exclude_hubs_percentile: options.exclude_hubs, + previous: (!previous.is_empty()).then_some(&previous), + incremental: !history_build, + changed_sources: &changed_sources, + limits: community_limits, + }, + )?; let cluster_elapsed = cluster_started.elapsed(); profile_internal_duration( - if clustered.used_incremental { + if matches!(clustered.execution, CommunityExecution::Incremental { .. }) { "bounded incremental clustering" } else { - "Louvain clustering" + "Leiden community detection" }, cluster_elapsed, ); internal_started = Instant::now(); - let communities = clustered.communities; + let CommunityResult { + communities, + base_labels: labels, + quality: community_quality, + identity: community_identity, + .. + } = clustered; + // Legacy report code still needs a compatibility projection, but the + // complete community operation above consumes the typed authority first. + // Move records into the projection and reconstruct the strict authority + // after those consumers finish so two complete views are not retained. + let document = published.document.into_legacy_document()?; timings.graph_assembly += stage_started.elapsed(); stage_started = Instant::now(); - let labels = label_communities_by_hub(&document, &communities); profile_internal("community labeling", &mut internal_started); let graph_analyses = || @@ -4210,6 +4228,23 @@ fn build_graph_inner_unscoped( if let Some(metrics) = store_metrics { record_store_metrics(&mut timings, metrics); } + let graph_seal_for_quality = graph_seal.as_ref().ok_or_else(|| { + CoreError::InvalidBuildState( + "graph artifact seal is unavailable for community quality evidence".to_owned(), + ) + })?; + let quality_artifact = CommunityQualityArtifact::new( + published_document.graph.build.generation_id.clone(), + format!("sha256:{}", graph_seal_for_quality.sha256), + community_identity, + community_limits, + community_quality, + )?; + write_json_atomic( + output_dir.join("community-quality.json"), + &quality_artifact, + true, + )?; if options.purpose == BuildPurpose::Update { write_prepared_graph_overview(overview_model, &output_dir)?; } @@ -4454,6 +4489,19 @@ fn build_profile(options: &BuildOptions) -> BuildProfile { no_viz: options.no_viz, resolution: options.resolution, exclude_hubs: options.exclude_hubs, + cluster_algorithm: compass_graph::QUALITY_CLUSTER_ALGORITHM.to_owned(), + cluster_topology: compass_graph::QUALITY_CLUSTER_TOPOLOGY.to_owned(), + cluster_quality: compass_graph::QUALITY_CLUSTER_QUALITY.to_owned(), + cluster_selector: compass_graph::COMPATIBILITY_CLUSTER_SELECTOR.to_owned(), + cluster_seed: compass_graph::COMPATIBILITY_CLUSTER_SEED, + cluster_resolution_policy: "fixed/v1".to_owned(), + cluster_hub_policy: if options.exclude_hubs.is_some() { + "exclude-percentile/v1" + } else { + "none/v1" + } + .to_owned(), + cluster_limits_version: compass_graph::QUALITY_CLUSTER_LIMITS.to_owned(), code_only: options.code_only, program_analysis: options.program_analysis, graph_storage: match options.graph_storage { @@ -4558,11 +4606,13 @@ fn publish_build_state( output_dir.join("labels.json"), output_dir.join("GRAPH_REPORT.md"), output_dir.join("orientation.json"), + output_dir.join("community-quality.json"), ]); } } BuildPurpose::Extract if !options.no_cluster => { required.push(output_dir.join("analysis.json")); + required.push(output_dir.join("community-quality.json")); } BuildPurpose::Extract => {} } @@ -7523,6 +7573,32 @@ mod tests { ); } + #[test] + fn production_community_profile_stays_fixed_for_omitted_and_explicit_resolution() { + let omitted = BuildOptions::new("."); + let omitted_profile = build_profile(&omitted); + assert_eq!( + omitted_profile.cluster_algorithm, + compass_graph::QUALITY_CLUSTER_ALGORITHM + ); + assert_eq!( + omitted_profile.cluster_selector, + compass_graph::COMPATIBILITY_CLUSTER_SELECTOR + ); + assert_eq!(omitted_profile.cluster_resolution_policy, "fixed/v1"); + + let mut explicit = omitted; + explicit.resolution = 1.25; + explicit.resolution_explicit = true; + let explicit_profile = build_profile(&explicit); + assert_eq!( + explicit_profile.cluster_selector, + compass_graph::COMPATIBILITY_CLUSTER_SELECTOR + ); + assert_eq!(explicit_profile.cluster_resolution_policy, "fixed/v1"); + assert_eq!(explicit_profile.resolution, 1.25); + } + #[test] fn build_inference_levels_publish_nested_coherent_graphs() -> Result<(), Box> { let directory = tempfile::tempdir()?; diff --git a/crates/compass-graph/examples/community_quality_qualification.rs b/crates/compass-graph/examples/community_quality_qualification.rs new file mode 100644 index 000000000..28f5ee8b0 --- /dev/null +++ b/crates/compass-graph/examples/community_quality_qualification.rs @@ -0,0 +1,671 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::error::Error; +use std::path::Path; + +use compass_graph::{ + Communities, CommunityLimits, CommunityProfile, CommunityRequest, CommunityResult, + ResolutionPolicy, adjusted_mutual_information, adjusted_rand_index, build_communities, +}; +use compass_model::code_graph::{BuildMetadata, EdgeKind, EdgeRecord, NodeKind, NodeRecord}; +use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin, Provenance, SourceAnchor}; +use serde::Serialize; + +const REPORT_SCHEMA: &str = "compass.community-quality-qualification/1"; + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Report { + schema: &'static str, + fixture_count: usize, + acceptance: Acceptance, + fixtures: Vec, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct Acceptance { + zero_disconnected_quality_communities: bool, + deterministic_repeat_and_permutation: bool, + exact_recovery_for_required_fixtures: bool, + no_adjusted_rand_regression_over_point_zero_two: bool, + resolution_limit_improved: bool, + articulation_improved: bool, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct FixtureReport { + name: String, + nodes: usize, + edges: usize, + exact_recovery_required: bool, + deterministic_repeat: bool, + permutation_equal: bool, + compatibility: ProfileReport, + quality: ProfileReport, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ProfileReport { + algorithm: String, + topology: String, + selector: String, + selected_candidate_resolution: f64, + communities: usize, + disconnected_communities: usize, + modularity: f64, + weighted_mean_conductance: f64, + worst_conductance: f64, + largest_community_fraction: f64, + non_isolate_singletons: usize, + quality_visits: usize, + adjusted_rand_index: Option, + adjusted_mutual_information: Option, + exact_recovery: Option, + false_merges: Option, + false_splits: Option, +} + +struct Fixture { + name: String, + document: compass_model::code_graph::GraphDocument, + planted: Option, + exact_recovery_required: bool, +} + +#[derive(Clone, Copy)] +struct EdgeSpec { + source: usize, + target: usize, + kind: EdgeKind, + confidence: EvidenceConfidence, +} + +fn main() -> Result<(), Box> { + let arguments = std::env::args().skip(1).collect::>(); + if arguments + .first() + .is_some_and(|value| value == "graph-profile") + { + return graph_profile(&arguments); + } + if let Some((profile, fixed_resolution)) = arguments.first().and_then(|argument| match argument + .as_str() + { + "benchmark-compatibility" => Some((CommunityProfile::CompatibilityV1, None)), + "benchmark-quality" => Some((CommunityProfile::QualityV1, None)), + "benchmark-quality-fixed" => Some((CommunityProfile::QualityV1, Some(1.0))), + _ => None, + }) { + return benchmark(profile, fixed_resolution); + } + let selected = arguments.first(); + let fixed_resolution = arguments + .get(1) + .map(|value| value.parse::()) + .transpose()?; + let fixtures = fixtures() + .into_iter() + .filter(|fixture| selected.is_none_or(|name| fixture.name == *name)) + .collect::>(); + if fixtures.is_empty() { + return Err("no community qualification fixture matched".into()); + } + let mut reports = Vec::with_capacity(fixtures.len()); + for fixture in fixtures { + reports.push(qualify_fixture(fixture, fixed_resolution)?); + } + let report = Report { + schema: REPORT_SCHEMA, + fixture_count: reports.len(), + acceptance: Acceptance { + zero_disconnected_quality_communities: reports + .iter() + .all(|fixture| fixture.quality.disconnected_communities == 0), + deterministic_repeat_and_permutation: reports + .iter() + .all(|fixture| fixture.deterministic_repeat && fixture.permutation_equal), + exact_recovery_for_required_fixtures: reports.iter().all(|fixture| { + !fixture.exact_recovery_required || fixture.quality.exact_recovery == Some(true) + }), + no_adjusted_rand_regression_over_point_zero_two: reports.iter().all(|fixture| { + match ( + fixture.compatibility.adjusted_rand_index, + fixture.quality.adjusted_rand_index, + ) { + (Some(compatibility), Some(quality)) => quality + 0.02 >= compatibility, + _ => true, + } + }), + resolution_limit_improved: fixture_improved(&reports, "ring-of-cliques"), + articulation_improved: fixture_improved(&reports, "articulation"), + }, + fixtures: reports, + }; + println!("{}", serde_json::to_string_pretty(&report)?); + Ok(()) +} + +fn graph_profile(arguments: &[String]) -> Result<(), Box> { + let path = arguments + .get(1) + .ok_or("graph-profile requires a graph path")?; + let profile = arguments.get(2).map(String::as_str).unwrap_or("fixed"); + if arguments.len() > 3 { + return Err("graph-profile accepts only a graph path and profile".into()); + } + let document = compass_model::code_graph::GraphDocument::load_for_recluster(Path::new(path))?; + let result = match profile { + "compatibility" => detect(&document, CommunityProfile::CompatibilityV1)?, + "fixed" => detect_quality(&document, Some(1.0))?, + "automatic" => detect_quality(&document, None)?, + _ => return Err("graph-profile must be compatibility, fixed, or automatic".into()), + }; + println!( + "{}", + serde_json::to_string_pretty(&profile_report(&result, None))? + ); + Ok(()) +} + +fn benchmark( + profile: CommunityProfile, + fixed_resolution: Option, +) -> Result<(), Box> { + let fixtures = fixtures(); + let mut observed = 0usize; + for _ in 0..50 { + for fixture in &fixtures { + let result = if profile == CommunityProfile::QualityV1 { + detect_quality(&fixture.document, fixed_resolution)? + } else { + detect(&fixture.document, profile)? + }; + observed = observed.saturating_add(result.communities.len()); + } + } + println!("{observed}"); + Ok(()) +} +fn fixture_improved(reports: &[FixtureReport], name: &str) -> bool { + reports.iter().any(|fixture| { + fixture.name == name + && matches!( + ( + fixture.compatibility.adjusted_rand_index, + fixture.quality.adjusted_rand_index, + ), + (Some(compatibility), Some(quality)) if quality > compatibility + 0.02 + ) + }) +} + +fn qualify_fixture( + fixture: Fixture, + fixed_resolution: Option, +) -> Result> { + let compatibility = detect(&fixture.document, CommunityProfile::CompatibilityV1)?; + let quality = detect_quality(&fixture.document, fixed_resolution)?; + let repeated = detect_quality(&fixture.document, fixed_resolution)?; + let mut permuted = fixture.document.clone(); + permuted.nodes.reverse(); + permuted.links.reverse(); + let permuted = detect_quality(&permuted, fixed_resolution)?; + Ok(FixtureReport { + name: fixture.name, + nodes: fixture.document.nodes.len(), + edges: fixture.document.links.len(), + exact_recovery_required: fixture.exact_recovery_required, + deterministic_repeat: quality.communities == repeated.communities + && quality.quality == repeated.quality, + permutation_equal: quality.communities == permuted.communities + && quality.quality == permuted.quality, + compatibility: profile_report(&compatibility, fixture.planted.as_ref()), + quality: profile_report(&quality, fixture.planted.as_ref()), + }) +} + +fn detect_quality( + document: &compass_model::code_graph::GraphDocument, + fixed_resolution: Option, +) -> Result { + let changed_sources = BTreeSet::new(); + build_communities( + document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: fixed_resolution.map_or( + ResolutionPolicy::Auto { base: 1.0 }, + ResolutionPolicy::Fixed, + ), + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits: CommunityLimits::default(), + }, + ) +} + +fn detect( + document: &compass_model::code_graph::GraphDocument, + profile: CommunityProfile, +) -> Result { + let changed_sources = BTreeSet::new(); + build_communities( + document, + &CommunityRequest { + profile, + resolution: if profile == CommunityProfile::QualityV1 { + ResolutionPolicy::Auto { base: 1.0 } + } else { + ResolutionPolicy::Fixed(1.0) + }, + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits: CommunityLimits::default(), + }, + ) +} + +fn profile_report(result: &CommunityResult, planted: Option<&Communities>) -> ProfileReport { + let (ari, ami, exact, false_merges, false_splits) = + planted.map_or((None, None, None, None, None), |planted| { + let (merges, splits) = merge_split_counts(&result.communities, planted); + ( + Some(adjusted_rand_index(&result.communities, planted)), + adjusted_mutual_information(&result.communities, planted).ok(), + Some(canonical_memberships(&result.communities) == canonical_memberships(planted)), + Some(merges), + Some(splits), + ) + }); + ProfileReport { + algorithm: result.identity.algorithm.clone(), + topology: result.identity.topology.clone(), + selector: result.identity.selector.clone(), + selected_candidate_resolution: result + .quality + .candidate_summaries + .iter() + .find(|candidate| candidate.selected) + .map_or(result.quality.resolution, |candidate| candidate.resolution), + communities: result.communities.len(), + disconnected_communities: result.quality.disconnected_community_count, + modularity: result.quality.modularity, + weighted_mean_conductance: result.quality.weighted_mean_conductance, + worst_conductance: result.quality.worst_conductance, + largest_community_fraction: result.quality.largest_community_fraction, + non_isolate_singletons: result.quality.non_isolate_singleton_count, + quality_visits: result.quality.quality_visit_count, + adjusted_rand_index: ari, + adjusted_mutual_information: ami, + exact_recovery: exact, + false_merges, + false_splits, + } +} + +fn canonical_memberships(communities: &Communities) -> Vec> { + let mut memberships = communities.values().cloned().collect::>(); + for members in &mut memberships { + members.sort(); + } + memberships.sort(); + memberships +} + +fn merge_split_counts(detected: &Communities, planted: &Communities) -> (usize, usize) { + let planted_assignment = assignment(planted); + let detected_assignment = assignment(detected); + let false_merges = detected + .values() + .filter(|members| { + members + .iter() + .filter_map(|member| planted_assignment.get(member)) + .collect::>() + .len() + > 1 + }) + .count(); + let false_splits = planted + .values() + .filter(|members| { + members + .iter() + .filter_map(|member| detected_assignment.get(member)) + .collect::>() + .len() + > 1 + }) + .count(); + (false_merges, false_splits) +} + +fn assignment(communities: &Communities) -> BTreeMap { + communities + .iter() + .flat_map(|(community, members)| { + members + .iter() + .map(move |member| (member.clone(), *community)) + }) + .collect() +} + +fn fixtures() -> Vec { + let mut output = vec![ + dense_bridge(), + ring_of_cliques(), + articulation(), + stars_and_hubs(), + directed_fan(), + reciprocal_and_one_way(), + parallel_confidence(), + containment_dominated(), + isolates_and_subsystem(), + large_weak_community(), + layered_code(), + tests_and_documentation(), + ]; + output.extend([0, 2, 4].map(lfr_style)); + output +} + +fn dense_bridge() -> Fixture { + let mut edges = clique(0, 5, EdgeKind::Calls); + edges.extend(clique(5, 10, EdgeKind::Calls)); + edges.push(exact(4, 5, EdgeKind::Calls)); + graph_fixture("dense-groups-one-bridge", 10, edges, groups(&[5, 5]), true) +} + +fn ring_of_cliques() -> Fixture { + let mut edges = Vec::new(); + let group_count = 12; + for group in 0..group_count { + edges.extend(clique(group * 3, group * 3 + 3, EdgeKind::Calls)); + edges.push(exact( + group * 3 + 2, + ((group + 1) % group_count) * 3, + EdgeKind::References, + )); + } + graph_fixture( + "ring-of-cliques", + group_count * 3, + edges, + groups(&[3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3]), + true, + ) +} + +fn articulation() -> Fixture { + let mut edges = clique(0, 4, EdgeKind::Calls); + edges.extend(clique(4, 8, EdgeKind::Calls)); + edges.extend([ + exact(3, 8, EdgeKind::References), + exact(8, 4, EdgeKind::Calls), + ]); + graph_fixture("articulation", 9, edges, groups(&[4, 5]), true) +} + +fn stars_and_hubs() -> Fixture { + let mut edges = (1..8) + .map(|node| exact(0, node, EdgeKind::Calls)) + .collect::>(); + edges.extend((9..16).map(|node| exact(8, node, EdgeKind::Imports))); + edges.push(exact(0, 8, EdgeKind::References)); + graph_fixture("stars-and-multi-hubs", 16, edges, None, false) +} + +fn directed_fan() -> Fixture { + let mut edges = (1..6) + .map(|node| exact(0, node, EdgeKind::Publishes)) + .collect::>(); + edges.extend((7..12).map(|node| exact(node, 6, EdgeKind::Subscribes))); + graph_fixture("directed-fan-in-out", 12, edges, None, false) +} + +fn reciprocal_and_one_way() -> Fixture { + graph_fixture( + "reciprocal-versus-one-way", + 6, + vec![ + exact(0, 1, EdgeKind::Calls), + exact(1, 0, EdgeKind::Calls), + exact(1, 2, EdgeKind::Calls), + exact(3, 4, EdgeKind::Calls), + exact(4, 5, EdgeKind::Calls), + ], + groups(&[3, 3]), + false, + ) +} + +fn parallel_confidence() -> Fixture { + let mut edges = vec![ + exact(0, 1, EdgeKind::Calls), + inferred(0, 1, EdgeKind::Calls), + ]; + edges.extend((0..5).map(|_| exact(1, 2, EdgeKind::Calls))); + edges.push(EdgeSpec { + source: 2, + target: 3, + kind: EdgeKind::Calls, + confidence: EvidenceConfidence::Ambiguous, + }); + graph_fixture("parallel-confidence-evidence", 4, edges, None, false) +} + +fn containment_dominated() -> Fixture { + let mut edges = (1..10) + .map(|node| exact(0, node, EdgeKind::Contains)) + .collect::>(); + edges.extend(clique(1, 5, EdgeKind::Calls)); + edges.extend(clique(5, 10, EdgeKind::Calls)); + graph_fixture( + "containment-dominated-files", + 10, + edges, + groups(&[1, 4, 5]), + false, + ) +} + +fn isolates_and_subsystem() -> Fixture { + graph_fixture( + "isolates-and-connected-subsystems", + 9, + clique(0, 6, EdgeKind::Calls), + groups(&[6, 1, 1, 1]), + false, + ) +} + +fn large_weak_community() -> Fixture { + let edges = (0..39) + .map(|node| exact(node, node + 1, EdgeKind::References)) + .collect(); + graph_fixture("large-weak-community", 40, edges, groups(&[40]), false) +} + +fn layered_code() -> Fixture { + let mut edges = clique(0, 4, EdgeKind::RoutesTo); + edges.extend(clique(4, 8, EdgeKind::Calls)); + edges.extend(clique(8, 12, EdgeKind::Reads)); + for node in 0..4 { + edges.push(exact(node, node + 4, EdgeKind::Calls)); + edges.push(exact(node + 4, node + 8, EdgeKind::Calls)); + } + graph_fixture( + "layered-handler-domain-repository", + 12, + edges, + groups(&[4, 4, 4]), + false, + ) +} + +fn tests_and_documentation() -> Fixture { + let mut edges = clique(0, 6, EdgeKind::Calls); + edges.extend((6..9).map(|node| exact(node, node - 6, EdgeKind::Tests))); + edges.extend((9..12).map(|node| exact(node, node - 9, EdgeKind::Documents))); + graph_fixture("tests-and-documentation", 12, edges, groups(&[12]), false) +} + +fn lfr_style(mixing: usize) -> Fixture { + let group_size = 8; + let group_count = 4; + let mut edges = Vec::new(); + for group in 0..group_count { + let start = group * group_size; + for left in start..start + group_size { + for right in left + 1..start + group_size { + if (left * 31 + right * 17 + group) % 3 != 0 { + edges.push(exact(left, right, EdgeKind::Calls)); + } + } + } + for offset in 0..mixing { + edges.push(inferred( + start + offset, + ((group + 1) % group_count) * group_size + offset, + EdgeKind::DependsOn, + )); + } + } + graph_fixture( + &format!("lfr-style-mixing-{mixing}"), + group_size * group_count, + edges, + groups(&[group_size, group_size, group_size, group_size]), + mixing == 0, + ) +} + +fn clique(start: usize, end: usize, kind: EdgeKind) -> Vec { + (start..end) + .flat_map(|left| (left + 1..end).map(move |right| exact(left, right, kind))) + .collect() +} + +fn exact(source: usize, target: usize, kind: EdgeKind) -> EdgeSpec { + EdgeSpec { + source, + target, + kind, + confidence: EvidenceConfidence::Exact, + } +} + +fn inferred(source: usize, target: usize, kind: EdgeKind) -> EdgeSpec { + EdgeSpec { + source, + target, + kind, + confidence: EvidenceConfidence::Inferred, + } +} + +fn groups(sizes: &[usize]) -> Option { + let mut start = 0; + Some( + sizes + .iter() + .enumerate() + .map(|(community, size)| { + let members = (start..start + size) + .map(|node| format!("node-{node:03}")) + .collect(); + start += size; + (community, members) + }) + .collect(), + ) +} + +fn graph_fixture( + name: &str, + node_count: usize, + edges: Vec, + planted: Option, + exact_recovery_required: bool, +) -> Fixture { + let mut document = compass_model::code_graph::GraphDocument::empty_v1(BuildMetadata { + builder_version: "community-quality-qualification".to_owned(), + schema_fingerprint: "fixture-v1".to_owned(), + source_tree_digest: name.to_owned(), + configuration_digest: "quality-v1".to_owned(), + generation_id: format!("fixture:{name}"), + source_commit: None, + }); + document.nodes = (0..node_count) + .map(|index| NodeRecord { + id: format!("node-{index:03}"), + kind: NodeKind::Function, + roles: Vec::new(), + name: format!("node_{index}"), + qualified_name: format!("fixture::{name}::node_{index}"), + language: Some("rust".to_owned()), + framework: None, + source: Some(anchor(name, index)), + details: None, + evidence: Vec::new(), + coverage: Vec::new(), + diagnostics: Vec::new(), + community: None, + }) + .collect(); + document.links = edges + .into_iter() + .enumerate() + .map(|(index, edge)| EdgeRecord { + id: format!("edge-{index:04}"), + key: format!("edge-{index:04}"), + source: format!("node-{:03}", edge.source), + target: format!("node-{:03}", edge.target), + kind: edge.kind, + occurrence_rule: None, + relationship_site: Some(anchor(name, index + node_count)), + details: None, + evidence: vec![Provenance { + origin: EvidenceOrigin::Ast, + extractor: "community-quality-fixture".to_owned(), + confidence: edge.confidence, + rule: None, + anchors: Vec::new(), + wiring_site: None, + score: None, + candidates: Vec::new(), + }], + weight: Some(1.0), + context: None, + deferred: false, + diagnostics: Vec::new(), + }) + .collect(); + Fixture { + name: name.to_owned(), + document, + planted, + exact_recovery_required, + } +} + +fn anchor(name: &str, index: usize) -> SourceAnchor { + let byte = u64::try_from(index).unwrap_or(u64::MAX - 1); + let line = u32::try_from(index.saturating_add(1)).unwrap_or(u32::MAX); + SourceAnchor { + file: format!("fixtures/{name}.rs"), + start_byte: byte, + end_byte: byte.saturating_add(1), + start_line: line, + start_column: 0, + end_line: line, + end_column: 1, + } +} diff --git a/crates/compass-graph/src/analyze.rs b/crates/compass-graph/src/analyze.rs index 3dad66ac6..cb0aba675 100644 --- a/crates/compass-graph/src/analyze.rs +++ b/crates/compass-graph/src/analyze.rs @@ -8,7 +8,7 @@ use rayon::prelude::*; use serde::Serialize; use sha2::{Digest, Sha256}; -use crate::cluster::{Communities, PythonRandom}; +use crate::cluster::{Communities, PythonRandom, score_communities}; const BUILTIN_NOISE_LABELS: &[&str] = &[ "str", @@ -334,7 +334,9 @@ fn suggest_questions_in( top_n: usize, ) -> (Vec, BlindSpotReport) { let node_community = invert_communities(communities); - let cohesion = community_cohesion_scores(graph, communities, &node_community); + let cohesion = score_communities(graph.document, communities) + .into_iter() + .collect::>(); let mut questions = Vec::new(); for edge in &graph.edges { if edge_string(edge.record, "confidence") != "AMBIGUOUS" { @@ -1692,6 +1694,7 @@ fn relation_edge_sort_key( } struct AnalysisGraph<'a> { + document: &'a GraphDocument, nodes: Vec<&'a NodeRecord>, positions: HashMap<&'a str, usize>, edges: Vec>, @@ -1752,6 +1755,7 @@ impl<'a> AnalysisGraph<'a> { } } Self { + document, nodes, positions, edges, @@ -1823,34 +1827,6 @@ fn invert_communities(communities: &Communities) -> HashMap { .collect() } -fn community_cohesion_scores( - graph: &AnalysisGraph<'_>, - communities: &Communities, - node_community: &HashMap, -) -> HashMap { - let mut internal_edges = HashMap::::new(); - for edge in &graph.edges { - let left = node_community.get(&graph.nodes[edge.left].id); - if let Some(community) = left - && node_community.get(&graph.nodes[edge.right].id) == Some(community) - { - *internal_edges.entry(*community).or_default() += 1; - } - } - communities - .iter() - .map(|(community, members)| { - let count = members.len(); - let possible = count.saturating_mul(count.saturating_sub(1)) / 2; - let score = if possible == 0 { - 1.0 - } else { - internal_edges.get(community).copied().unwrap_or_default() as f64 / possible as f64 - }; - (*community, score) - }) - .collect() -} fn is_concept_node(node: &NodeRecord) -> bool { let source = attribute(node, "source_file").unwrap_or_default(); source.is_empty() || !source.rsplit('/').next().unwrap_or_default().contains('.') diff --git a/crates/compass-graph/src/cluster.rs b/crates/compass-graph/src/cluster.rs index be4b39632..517eae43c 100644 --- a/crates/compass-graph/src/cluster.rs +++ b/crates/compass-graph/src/cluster.rs @@ -4,9 +4,12 @@ use std::time::Instant; use ahash::{AHashMap as HashMap, AHashSet as HashSet}; use compass_model::{EdgeRecord, GraphDocument, NodeRecord}; use rayon::prelude::*; +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; use sha2::{Digest, Sha256}; +use crate::community::compatibility_density_scores; + const MAX_COMMUNITY_FRACTION: f64 = 0.25; const MIN_SPLIT_SIZE: usize = 10; const COHESION_SPLIT_THRESHOLD: f64 = 0.05; @@ -17,7 +20,8 @@ const LOUVAIN_MAX_LEVEL: usize = 10; pub type Communities = BTreeMap>; /// Bounds for a topology-changing incremental community update. -#[derive(Clone, Copy, Debug, PartialEq)] +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] pub struct IncrementalClusterLimits { /// Absolute ceiling for nodes admitted to the local reclustering region. pub max_affected_nodes: usize, @@ -615,9 +619,10 @@ pub fn community_member_signatures(communities: &Communities) -> BTreeMap f64 { - let graph = WeightedGraph::from_document(document); - let positions = graph.position_map(); - cohesion_score_graph(&graph, &positions, members) + compatibility_density_scores(document, &BTreeMap::from([(0, members.to_vec())])) + .get(&0) + .copied() + .unwrap_or(1.0) } #[must_use] @@ -625,54 +630,7 @@ pub fn score_communities( document: &GraphDocument, communities: &Communities, ) -> BTreeMap { - let positions = document - .nodes - .iter() - .enumerate() - .map(|(index, node)| (node.id.as_str(), index)) - .collect::>(); - let mut node_community = vec![None; document.nodes.len()]; - let mut internal_edges = HashMap::::new(); - for (community, members) in communities { - for member in members { - if let Some(position) = positions.get(member.as_str()) { - node_community[*position] = Some(*community); - } - } - } - let mut seen = HashSet::<(usize, usize)>::new(); - for edge in &document.links { - let (Some(&left), Some(&right)) = ( - positions.get(edge.source.as_str()), - positions.get(edge.target.as_str()), - ) else { - continue; - }; - let pair = if left <= right { - (left, right) - } else { - (right, left) - }; - if seen.insert(pair) - && let Some(community) = node_community[left] - && node_community[right] == Some(community) - { - *internal_edges.entry(community).or_default() += 1; - } - } - communities - .iter() - .map(|(community, members)| { - let count = members.len(); - let possible = count.saturating_mul(count.saturating_sub(1)) / 2; - let score = if possible == 0 { - 1.0 - } else { - internal_edges.get(community).copied().unwrap_or_default() as f64 / possible as f64 - }; - (*community, score) - }) - .collect() + compatibility_density_scores(document, communities) } #[must_use] @@ -743,7 +701,7 @@ pub fn remap_communities_to_previous( .collect() } -fn excluded_hubs(graph: &WeightedGraph, percentile: Option) -> HashSet { +pub(crate) fn excluded_hubs(graph: &WeightedGraph, percentile: Option) -> HashSet { let Some(percentile) = percentile else { return HashSet::new(); }; @@ -764,7 +722,11 @@ fn excluded_hubs(graph: &WeightedGraph, percentile: Option) -> HashSet, raw: &mut Vec>) { +pub(crate) fn reattach_hubs( + graph: &WeightedGraph, + hubs: &HashSet, + raw: &mut Vec>, +) { let mut node_community = raw .iter() .enumerate() @@ -1063,14 +1025,14 @@ fn aggregate_graph(graph: &WeightedGraph, communities: &[BTreeSet]) -> We } #[derive(Clone)] -struct WeightedGraph { - ids: Vec, - members: Vec>, +pub(crate) struct WeightedGraph { + pub(crate) ids: Vec, + pub(crate) members: Vec>, adjacency: Vec>, } impl WeightedGraph { - fn new(ids: Vec, members: Vec>) -> Self { + pub(crate) fn new(ids: Vec, members: Vec>) -> Self { let adjacency = vec![Vec::new(); ids.len()]; Self { ids, @@ -1079,7 +1041,7 @@ impl WeightedGraph { } } - fn from_document(document: &GraphDocument) -> Self { + pub(crate) fn from_document(document: &GraphDocument) -> Self { let mut ids = document .nodes .iter() @@ -1131,19 +1093,19 @@ impl WeightedGraph { graph } - fn len(&self) -> usize { + pub(crate) fn len(&self) -> usize { self.ids.len() } - fn is_empty(&self) -> bool { + pub(crate) fn is_empty(&self) -> bool { self.ids.is_empty() } - fn edge_count(&self) -> usize { + pub(crate) fn edge_count(&self) -> usize { self.edges().count() } - fn position_map(&self) -> HashMap<&String, usize> { + pub(crate) fn position_map(&self) -> HashMap<&String, usize> { self.ids .iter() .enumerate() @@ -1151,14 +1113,14 @@ impl WeightedGraph { .collect() } - fn degree_unweighted(&self, node: usize) -> usize { + pub(crate) fn degree_unweighted(&self, node: usize) -> usize { self.adjacency[node] .iter() .map(|(neighbor, _)| if *neighbor == node { 2 } else { 1 }) .sum() } - fn degree_weighted(&self, node: usize) -> f64 { + pub(crate) fn degree_weighted(&self, node: usize) -> f64 { self.adjacency[node] .iter() .map(|(neighbor, weight)| { @@ -1171,7 +1133,7 @@ impl WeightedGraph { .sum() } - fn total_weight(&self) -> f64 { + pub(crate) fn total_weight(&self) -> f64 { self.edges().map(|(_, _, weight)| weight).sum() } @@ -1196,7 +1158,7 @@ impl WeightedGraph { } } - fn add_edge(&mut self, left: usize, right: usize, weight: f64) { + pub(crate) fn add_edge(&mut self, left: usize, right: usize, weight: f64) { if let Some((_, existing)) = self.adjacency[left] .iter_mut() .find(|(neighbor, _)| *neighbor == right) @@ -1217,7 +1179,7 @@ impl WeightedGraph { } } - fn edges(&self) -> impl Iterator + '_ { + pub(crate) fn edges(&self) -> impl Iterator + '_ { self.adjacency .iter() .enumerate() @@ -1229,7 +1191,11 @@ impl WeightedGraph { }) } - fn subgraph(&self, selected: &[usize]) -> Self { + pub(crate) fn neighbors(&self, node: usize) -> &[(usize, f64)] { + &self.adjacency[node] + } + + pub(crate) fn subgraph(&self, selected: &[usize]) -> Self { let positions = selected .iter() .enumerate() @@ -1369,7 +1335,7 @@ impl PythonRandom { } } - fn shuffle(&mut self, values: &mut [T]) { + pub(crate) fn shuffle(&mut self, values: &mut [T]) { for index in (1..values.len()).rev() { let replacement = self.below(index + 1); values.swap(index, replacement); diff --git a/crates/compass-graph/src/community/artifact.rs b/crates/compass-graph/src/community/artifact.rs new file mode 100644 index 000000000..ac6c07569 --- /dev/null +++ b/crates/compass-graph/src/community/artifact.rs @@ -0,0 +1,351 @@ +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use super::build::{CommunityIdentity, CommunityLimits}; +use super::quality::PartitionQuality; + +pub const COMMUNITY_QUALITY_SCHEMA: &str = "compass.community-quality/1"; + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommunityQualityArtifact { + pub schema: String, + pub graph_generation: String, + pub graph_digest: String, + pub identity: CommunityIdentity, + pub limits: CommunityLimits, + pub partition: PartitionQuality, + pub result_digest: String, +} + +#[derive(Debug, Error)] +pub enum CommunityQualityArtifactError { + #[error("unsupported community quality schema `{0}`")] + UnsupportedSchema(String), + #[error("community quality artifact is missing graph identity")] + MissingGraphIdentity, + #[error("community quality profile identity does not match partition evidence")] + IdentityMismatch, + #[error("community quality result digest mismatch")] + DigestMismatch, + #[error("community quality graph identity does not match the selected graph")] + GraphIdentityMismatch, + #[error("invalid community quality evidence: {0}")] + InvalidEvidence(String), + #[error("could not encode community quality evidence: {0}")] + Encode(#[from] serde_json::Error), +} + +impl CommunityQualityArtifact { + pub fn new( + graph_generation: String, + graph_digest: String, + identity: CommunityIdentity, + limits: CommunityLimits, + partition: PartitionQuality, + ) -> Result { + let mut artifact = Self { + schema: COMMUNITY_QUALITY_SCHEMA.to_owned(), + graph_generation, + graph_digest, + identity, + limits, + partition, + result_digest: String::new(), + }; + artifact.result_digest = artifact.calculate_digest()?; + artifact.validate()?; + Ok(artifact) + } + + pub fn validate(&self) -> Result<(), CommunityQualityArtifactError> { + if self.schema != COMMUNITY_QUALITY_SCHEMA { + return Err(CommunityQualityArtifactError::UnsupportedSchema( + self.schema.clone(), + )); + } + if self.graph_generation.is_empty() || !is_sha256_identity(&self.graph_digest) { + return Err(CommunityQualityArtifactError::MissingGraphIdentity); + } + if self.identity.algorithm != self.partition.algorithm + || self.identity.topology != self.partition.topology + || self.identity.quality != self.partition.quality + || self.identity.selector != self.partition.selector + || self.identity.seed != self.partition.seed + || self.identity.limits != self.partition.limits + { + return Err(CommunityQualityArtifactError::IdentityMismatch); + } + self.validate_evidence()?; + if self.result_digest != self.calculate_digest()? { + return Err(CommunityQualityArtifactError::DigestMismatch); + } + Ok(()) + } + + fn validate_evidence(&self) -> Result<(), CommunityQualityArtifactError> { + let invalid = + |message: &str| CommunityQualityArtifactError::InvalidEvidence(message.to_owned()); + if self.identity.algorithm.is_empty() + || self.identity.topology.is_empty() + || self.identity.quality.is_empty() + || self.identity.selector.is_empty() + || self.identity.limits.is_empty() + { + return Err(invalid("profile identity contains an empty field")); + } + if !self.limits.max_total_weight.is_finite() || self.limits.max_total_weight < 0.0 { + return Err(invalid("maxTotalWeight must be finite and non-negative")); + } + if self.partition.quality_visit_limit != self.limits.max_quality_visits + || self.partition.quality_visit_count > self.partition.quality_visit_limit + { + return Err(invalid("quality visit accounting does not match limits")); + } + if self.partition.witness_limit != self.limits.witness_limit { + return Err(invalid("witness limit does not match profile limits")); + } + let candidates = &self.partition.candidate_summaries; + if candidates.is_empty() || candidates.len() > self.limits.max_candidates { + return Err(invalid("candidate count is outside profile limits")); + } + if candidates + .iter() + .filter(|candidate| candidate.selected) + .count() + != 1 + { + return Err(invalid( + "candidate evidence must select exactly one partition", + )); + } + if !self.partition.resolution.is_finite() || self.partition.resolution <= 0.0 { + return Err(invalid("partition resolution must be finite and positive")); + } + if candidates.iter().any(|candidate| { + !candidate.resolution.is_finite() + || candidate.resolution <= 0.0 + || !candidate.modularity.is_finite() + || !candidate.weighted_mean_conductance.is_finite() + }) { + return Err(invalid( + "candidate metrics must be finite with positive resolution", + )); + } + if self.partition.communities.len() != self.partition.community_count + || self + .partition + .communities + .iter() + .any(|(community, metric)| { + metric.community != *community + || metric.witness_node_ids.len() > self.limits.witness_limit + || metric.witness_edge_ids.len() > self.limits.witness_limit + || !metric.internal_weight.is_finite() + || !metric.boundary_weight.is_finite() + || !metric.volume.is_finite() + || !metric.density.is_finite() + || !metric.conductance.is_finite() + || !metric.modularity_contribution.is_finite() + }) + { + return Err(invalid("per-community evidence is inconsistent")); + } + let assigned = self + .partition + .communities + .values() + .map(|metric| metric.member_count) + .fold(0usize, usize::saturating_add); + let disconnected = self + .partition + .communities + .values() + .filter(|metric| !metric.isolate && metric.connected_component_count > 1) + .count(); + let omitted_nodes = self + .partition + .communities + .values() + .map(|metric| metric.omitted_witness_node_count) + .fold(0usize, usize::saturating_add); + let omitted_edges = self + .partition + .communities + .values() + .map(|metric| metric.omitted_witness_edge_count) + .fold(0usize, usize::saturating_add); + if assigned != self.partition.assigned_node_count + || disconnected != self.partition.disconnected_community_count + || omitted_nodes != self.partition.omitted_witness_node_count + || omitted_edges != self.partition.omitted_witness_edge_count + || !self.partition.modularity.is_finite() + || !self.partition.weighted_mean_conductance.is_finite() + || !self.partition.worst_conductance.is_finite() + || !self.partition.largest_community_fraction.is_finite() + { + return Err(invalid("partition aggregates are inconsistent")); + } + if let Some(topology) = &self.partition.topology_evidence + && (topology.retained_occurrence_count > topology.input_edge_count + || topology.projected_pair_count > self.limits.max_projected_pairs + || !topology.input_weight_sum.is_finite() + || !topology.retained_total_weight.is_finite() + || topology.retained_total_weight > self.limits.max_total_weight) + { + return Err(invalid("topology evidence is inconsistent with limits")); + } + Ok(()) + } + + pub fn validate_for_graph( + &self, + generation: &str, + graph_digest: &str, + ) -> Result<(), CommunityQualityArtifactError> { + self.validate()?; + if self.graph_generation != generation || self.graph_digest != graph_digest { + return Err(CommunityQualityArtifactError::GraphIdentityMismatch); + } + Ok(()) + } + + fn calculate_digest(&self) -> Result { + let mut canonical = self.clone(); + canonical.result_digest.clear(); + let bytes = serde_json::to_vec(&canonical)?; + Ok(format!("sha256:{:x}", Sha256::digest(bytes))) + } +} + +fn is_sha256_identity(value: &str) -> bool { + value.strip_prefix("sha256:").is_some_and(|digest| { + digest.len() == 64 + && digest + .as_bytes() + .iter() + .all(|byte| byte.is_ascii_hexdigit() && !byte.is_ascii_uppercase()) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + COMPATIBILITY_CLUSTER_ALGORITHM, COMPATIBILITY_CLUSTER_LIMITS, + COMPATIBILITY_CLUSTER_QUALITY, COMPATIBILITY_CLUSTER_SEED, COMPATIBILITY_CLUSTER_SELECTOR, + COMPATIBILITY_CLUSTER_TOPOLOGY, + }; + use std::collections::BTreeMap; + + fn identity() -> CommunityIdentity { + CommunityIdentity { + algorithm: COMPATIBILITY_CLUSTER_ALGORITHM.to_owned(), + topology: COMPATIBILITY_CLUSTER_TOPOLOGY.to_owned(), + quality: COMPATIBILITY_CLUSTER_QUALITY.to_owned(), + selector: COMPATIBILITY_CLUSTER_SELECTOR.to_owned(), + seed: COMPATIBILITY_CLUSTER_SEED, + limits: COMPATIBILITY_CLUSTER_LIMITS.to_owned(), + } + } + + fn partition() -> PartitionQuality { + PartitionQuality { + assigned_node_count: 0, + omitted_node_count: 0, + community_count: 0, + non_isolate_singleton_count: 0, + disconnected_community_count: 0, + modularity: 0.0, + weighted_mean_conductance: 0.0, + worst_conductance: 0.0, + largest_community_fraction: 0.0, + resolution: 1.0, + algorithm: COMPATIBILITY_CLUSTER_ALGORITHM.to_owned(), + topology: COMPATIBILITY_CLUSTER_TOPOLOGY.to_owned(), + quality: COMPATIBILITY_CLUSTER_QUALITY.to_owned(), + selector: COMPATIBILITY_CLUSTER_SELECTOR.to_owned(), + seed: COMPATIBILITY_CLUSTER_SEED, + limits: COMPATIBILITY_CLUSTER_LIMITS.to_owned(), + quality_visit_count: 0, + quality_visit_limit: super::super::quality::DEFAULT_MAX_QUALITY_VISITS, + witness_limit: 8, + omitted_witness_node_count: 0, + omitted_witness_edge_count: 0, + candidate_summaries: vec![super::super::quality::CommunityCandidateSummary { + resolution: 1.0, + modularity: 0.0, + weighted_mean_conductance: 0.0, + disconnected_community_count: 0, + size_violation_count: 0, + non_isolate_singleton_count: 0, + partition_digest: "fixture".to_owned(), + selected: true, + rejection_reason: None, + }], + candidate_agreement: Vec::new(), + selection_reason: "test".to_owned(), + topology_evidence: None, + communities: BTreeMap::new(), + } + } + + #[test] + fn artifact_digest_detects_mutation() -> Result<(), CommunityQualityArtifactError> { + let mut artifact = CommunityQualityArtifact::new( + "generation".to_owned(), + format!("sha256:{}", "0".repeat(64)), + identity(), + CommunityLimits::default(), + partition(), + )?; + artifact.partition.selection_reason = "mutated".to_owned(); + assert!(matches!( + artifact.validate(), + Err(CommunityQualityArtifactError::DigestMismatch) + )); + Ok(()) + } + + #[test] + fn artifact_rejects_unknown_major_and_wrong_graph() -> Result<(), CommunityQualityArtifactError> + { + let artifact = CommunityQualityArtifact::new( + "generation".to_owned(), + format!("sha256:{}", "0".repeat(64)), + identity(), + CommunityLimits::default(), + partition(), + )?; + assert!(matches!( + artifact.validate_for_graph("other", &format!("sha256:{}", "0".repeat(64))), + Err(CommunityQualityArtifactError::GraphIdentityMismatch) + )); + let mut unknown = artifact; + unknown.schema = "compass.community-quality/2".to_owned(); + assert!(matches!( + unknown.validate(), + Err(CommunityQualityArtifactError::UnsupportedSchema(_)) + )); + Ok(()) + } + + #[test] + fn artifact_rejects_unknown_fields() -> Result<(), Box> { + let artifact = CommunityQualityArtifact::new( + "generation".to_owned(), + format!("sha256:{}", "0".repeat(64)), + identity(), + CommunityLimits::default(), + partition(), + )?; + let mut value = serde_json::to_value(artifact)?; + value + .as_object_mut() + .ok_or_else(|| std::io::Error::other("artifact must be an object"))? + .insert("futureField".to_owned(), serde_json::Value::Bool(true)); + assert!(serde_json::from_value::(value).is_err()); + Ok(()) + } +} diff --git a/crates/compass-graph/src/community/build.rs b/crates/compass-graph/src/community/build.rs new file mode 100644 index 000000000..31cf3a638 --- /dev/null +++ b/crates/compass-graph/src/community/build.rs @@ -0,0 +1,1174 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use thiserror::Error; + +use super::identity::{ + COMPATIBILITY_CLUSTER_ALGORITHM, COMPATIBILITY_CLUSTER_LIMITS, COMPATIBILITY_CLUSTER_QUALITY, + COMPATIBILITY_CLUSTER_SEED, COMPATIBILITY_CLUSTER_SELECTOR, COMPATIBILITY_CLUSTER_TOPOLOGY, + QUALITY_CLUSTER_ALGORITHM, QUALITY_CLUSTER_LIMITS, QUALITY_CLUSTER_QUALITY, + QUALITY_CLUSTER_SELECTOR, QUALITY_CLUSTER_TOPOLOGY, +}; +use super::incremental::{ + IncrementalPreparation, IncrementalPreparationFallback, baseline_partition, + prepare_anchored_topology, +}; +use super::leiden::{CommunityDetectorError, leiden}; +use super::quality::{ + CandidateAgreement, CommunityCandidateSummary, CommunityQualityError, PartitionQuality, + QualityEvaluationOptions, QualityIdentity, adjusted_rand_index, evaluate_graph_partition, + evaluate_partition_quality, +}; +use super::topology::{CommunityTopologyError, TopologyLimits, from_typed_document}; +use crate::cluster::{ + ClusterOptions, Communities, IncrementalClusterLimits, cluster, cluster_incremental, + community_member_signatures, excluded_hubs, label_communities_by_hub, reattach_hubs, + remap_communities_to_previous, +}; + +pub type PreviousCommunities = HashMap; + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub enum ResolutionPolicy { + Auto { base: f64 }, + Fixed(f64), +} + +impl ResolutionPolicy { + fn base(self) -> f64 { + match self { + Self::Auto { base } | Self::Fixed(base) => base, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum CommunityProfile { + CompatibilityV1, + QualityV1, +} + +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommunityLimits { + pub incremental: IncrementalClusterLimits, + pub max_nodes: usize, + pub max_edges: usize, + pub max_projected_pairs: usize, + pub max_total_weight: f64, + pub max_candidates: usize, + pub max_levels: usize, + pub max_moves: usize, + pub max_quality_visits: usize, + pub witness_limit: usize, +} + +impl Default for CommunityLimits { + fn default() -> Self { + Self { + incremental: IncrementalClusterLimits::default(), + max_nodes: 2_000_000, + max_edges: 8_000_000, + max_projected_pairs: 8_000_000, + max_total_weight: 512_000_000.0, + max_candidates: 3, + max_levels: 10, + max_moves: 100_000_000, + max_quality_visits: super::quality::DEFAULT_MAX_QUALITY_VISITS, + witness_limit: super::quality::DEFAULT_QUALITY_WITNESS_LIMIT, + } + } +} + +#[derive(Clone, Debug)] +pub struct CommunityRequest<'a> { + pub profile: CommunityProfile, + pub resolution: ResolutionPolicy, + pub exclude_hubs_percentile: Option, + pub previous: Option<&'a PreviousCommunities>, + pub incremental: bool, + pub changed_sources: &'a BTreeSet, + pub limits: CommunityLimits, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum FallbackReason { + CompatibilityGuard, + InvalidIncrementalLimits, + RemovedNode, + AffectedRegionLimit, + HubPolicy, + FrozenAnchorsWouldMerge, + QualityRegression, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum CommunityExecution { + Full, + Incremental { + affected_nodes: usize, + }, + FullFallback { + affected_nodes: usize, + reason: FallbackReason, + }, +} + +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommunityIdentity { + pub algorithm: String, + pub topology: String, + pub quality: String, + pub selector: String, + pub seed: u32, + pub limits: String, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommunityResult { + pub communities: Communities, + pub base_labels: BTreeMap, + pub signatures: BTreeMap, + pub quality: PartitionQuality, + pub execution: CommunityExecution, + pub identity: CommunityIdentity, +} + +#[derive(Debug, Error)] +pub enum CommunityError { + #[error("invalid community profile: {reason}")] + InvalidProfile { reason: &'static str }, + #[error("invalid community resolution {resolution}; expected a finite positive value")] + InvalidResolution { resolution: f64 }, + #[error("could not adapt the typed graph for community detection: {0}")] + GraphAdapter(#[from] compass_model::GraphError), + #[error(transparent)] + Quality(#[from] CommunityQualityError), + #[error(transparent)] + Topology(#[from] CommunityTopologyError), + #[error(transparent)] + Detector(#[from] CommunityDetectorError), + #[error( + "community selector candidates requires {required} candidates, exceeds limit {limit} after {processed} candidates" + )] + CandidateLimitExceeded { + required: usize, + limit: usize, + processed: usize, + }, +} + +/// Build the complete compatibility community result from the typed Base Graph +/// authority. This is the migration seam for all production callers. +pub fn build_communities( + document: &compass_model::code_graph::GraphDocument, + request: &CommunityRequest<'_>, +) -> Result { + let resolution = request.resolution.base(); + if !resolution.is_finite() || resolution <= 0.0 { + return Err(CommunityError::InvalidResolution { resolution }); + } + if !request.limits.max_total_weight.is_finite() || request.limits.max_total_weight < 0.0 { + return Err(CommunityError::InvalidProfile { + reason: "max_total_weight must be finite and non-negative", + }); + } + let legacy = document.to_legacy_document()?; + if request.profile == CommunityProfile::QualityV1 { + return build_quality_communities(document, &legacy, request, resolution); + } + let options = ClusterOptions { + resolution, + exclude_hubs_percentile: request.exclude_hubs_percentile, + }; + let (communities, execution) = if request.incremental + && let Some(previous) = request.previous + { + let incremental = cluster_incremental( + &legacy, + previous, + request.changed_sources, + options, + request.limits.incremental, + ); + let execution = if incremental.used_incremental { + CommunityExecution::Incremental { + affected_nodes: incremental.affected_nodes, + } + } else { + CommunityExecution::FullFallback { + affected_nodes: incremental.affected_nodes, + reason: FallbackReason::CompatibilityGuard, + } + }; + (incremental.communities, execution) + } else { + (cluster(&legacy, options), CommunityExecution::Full) + }; + let communities = match request.previous { + Some(previous) => remap_communities_to_previous(&communities, previous), + None => communities, + }; + let base_labels = label_communities_by_hub(&legacy, &communities); + let signatures = community_member_signatures(&communities); + let quality = evaluate_partition_quality(&legacy, &communities, resolution)?; + let identity = CommunityIdentity { + algorithm: COMPATIBILITY_CLUSTER_ALGORITHM.to_owned(), + topology: COMPATIBILITY_CLUSTER_TOPOLOGY.to_owned(), + quality: COMPATIBILITY_CLUSTER_QUALITY.to_owned(), + selector: COMPATIBILITY_CLUSTER_SELECTOR.to_owned(), + seed: COMPATIBILITY_CLUSTER_SEED, + limits: COMPATIBILITY_CLUSTER_LIMITS.to_owned(), + }; + Ok(CommunityResult { + communities, + base_labels, + signatures, + quality, + execution, + identity, + }) +} + +fn build_quality_communities( + document: &compass_model::code_graph::GraphDocument, + legacy: &compass_model::GraphDocument, + request: &CommunityRequest<'_>, + base_resolution: f64, +) -> Result { + let topology = from_typed_document( + document, + TopologyLimits { + max_nodes: request.limits.max_nodes, + max_edges: request.limits.max_edges, + max_projected_pairs: request.limits.max_projected_pairs, + max_total_weight: request.limits.max_total_weight, + }, + )?; + let resolutions = match request.resolution { + ResolutionPolicy::Fixed(value) => vec![value], + ResolutionPolicy::Auto { base } => vec![base * 0.75, base, base * (4.0 / 3.0)], + }; + if let Some(resolution) = resolutions + .iter() + .copied() + .find(|resolution| !resolution.is_finite() || *resolution <= 0.0) + { + return Err(CommunityError::InvalidResolution { resolution }); + } + if resolutions.len() > request.limits.max_candidates { + return Err(CommunityError::CandidateLimitExceeded { + required: resolutions.len(), + limit: request.limits.max_candidates, + processed: 0, + }); + } + let selector_identity = if resolutions.len() == 1 { + COMPATIBILITY_CLUSTER_SELECTOR + } else { + QUALITY_CLUSTER_SELECTOR + }; + let identity = QualityIdentity { + algorithm: QUALITY_CLUSTER_ALGORITHM, + topology: QUALITY_CLUSTER_TOPOLOGY, + quality: QUALITY_CLUSTER_QUALITY, + selector: selector_identity, + seed: COMPATIBILITY_CLUSTER_SEED, + limits: QUALITY_CLUSTER_LIMITS, + }; + let preparation = request + .incremental + .then_some(request.previous) + .flatten() + .map(|previous| { + prepare_anchored_topology( + document, + &topology.graph, + previous, + request.changed_sources, + request.limits.incremental, + request.exclude_hubs_percentile, + ) + }); + let mut execution = match &preparation { + Some(IncrementalPreparation::Ready(anchored)) => CommunityExecution::Incremental { + affected_nodes: anchored.affected_nodes, + }, + Some(IncrementalPreparation::Unchanged(_)) => { + CommunityExecution::Incremental { affected_nodes: 0 } + } + Some(IncrementalPreparation::Fallback { + affected_nodes, + reason, + }) => CommunityExecution::FullFallback { + affected_nodes: *affected_nodes, + reason: map_preparation_fallback(*reason), + }, + None => CommunityExecution::Full, + }; + let mut candidates = Vec::<(f64, Communities, PartitionQuality, String)>::new(); + let mut quality_guards = Vec::new(); + if matches!( + preparation, + None | Some(IncrementalPreparation::Fallback { .. }) + ) { + candidates = resolutions + .par_iter() + .map(|resolution| { + full_candidate(&topology, *resolution, base_resolution, request, identity) + }) + .collect::, CommunityError>>()?; + quality_guards.resize(candidates.len(), true); + } else { + for resolution in &resolutions { + let incremental = match &preparation { + Some(IncrementalPreparation::Ready(anchored)) => anchored.cluster( + *resolution, + request.limits.max_levels, + request.limits.max_moves, + )?, + Some(IncrementalPreparation::Unchanged(communities)) => Some(communities.clone()), + Some(IncrementalPreparation::Fallback { .. }) | None => None, + }; + if matches!(&preparation, Some(IncrementalPreparation::Ready(_))) + && incremental.is_none() + { + execution = CommunityExecution::FullFallback { + affected_nodes: match &preparation { + Some(IncrementalPreparation::Ready(anchored)) => anchored.affected_nodes, + _ => 0, + }, + reason: FallbackReason::FrozenAnchorsWouldMerge, + }; + } + let communities = incremental.unwrap_or_default(); + let quality = evaluate_graph_partition( + &topology.graph, + &communities, + base_resolution, + QualityEvaluationOptions { + identity: QualityIdentity { ..identity }, + topology_evidence: Some(topology.evidence.clone()), + pair_evidence: Some(&topology.pair_evidence), + max_quality_visits: request.limits.max_quality_visits, + witness_limit: request.limits.witness_limit, + }, + )?; + let digest = partition_digest(&communities); + let quality_guard = if let Some(previous) = request.previous + && matches!(execution, CommunityExecution::Incremental { .. }) + { + let baseline = baseline_partition(&topology.graph, previous); + let baseline_quality = evaluate_graph_partition( + &topology.graph, + &baseline, + base_resolution, + QualityEvaluationOptions { + identity, + topology_evidence: None, + pair_evidence: Some(&topology.pair_evidence), + max_quality_visits: request.limits.max_quality_visits, + witness_limit: request.limits.witness_limit, + }, + )?; + quality.modularity + 1e-4 >= baseline_quality.modularity + && (quality.largest_community_fraction <= 0.25 + || baseline_quality.largest_community_fraction > 0.25) + && quality.weighted_mean_conductance + <= baseline_quality.weighted_mean_conductance + 1e-4 + } else { + true + }; + quality_guards.push(quality_guard); + candidates.push((*resolution, communities, quality, digest)); + } + } + let anchors_would_merge = matches!( + execution, + CommunityExecution::FullFallback { + reason: FallbackReason::FrozenAnchorsWouldMerge, + .. + } + ); + if anchors_would_merge + || (matches!(execution, CommunityExecution::Incremental { .. }) + && !quality_guards.iter().any(|guard| *guard)) + { + let affected_nodes = match &execution { + CommunityExecution::Incremental { affected_nodes } => *affected_nodes, + _ => 0, + }; + if !anchors_would_merge { + execution = CommunityExecution::FullFallback { + affected_nodes, + reason: FallbackReason::QualityRegression, + }; + } + candidates.clear(); + quality_guards.clear(); + candidates = resolutions + .par_iter() + .map(|resolution| { + full_candidate(&topology, *resolution, base_resolution, request, identity) + }) + .collect::, CommunityError>>()?; + quality_guards.resize(candidates.len(), true); + } + let best_modularity = candidates + .iter() + .filter(|(_, _, quality, _)| quality.disconnected_community_count == 0) + .map(|(_, _, quality, _)| quality.modularity) + .fold(f64::NEG_INFINITY, f64::max); + let tolerance = 1e-4; + let mut eligible = candidates + .iter() + .enumerate() + .filter(|(index, (_, _, quality, _))| { + quality_guards[*index] + && quality.assigned_node_count == topology.graph.len() + && quality.disconnected_community_count == 0 + && quality.modularity + tolerance >= best_modularity + }) + .map(|(index, _)| index) + .collect::>(); + eligible.sort_by(|left, right| { + let left_candidate = &candidates[*left]; + let right_candidate = &candidates[*right]; + candidate_size_violations(&left_candidate.2) + .cmp(&candidate_size_violations(&right_candidate.2)) + .then_with(|| { + left_candidate + .2 + .weighted_mean_conductance + .total_cmp(&right_candidate.2.weighted_mean_conductance) + }) + .then_with(|| { + left_candidate + .2 + .non_isolate_singleton_count + .cmp(&right_candidate.2.non_isolate_singleton_count) + }) + .then_with(|| left_candidate.3.cmp(&right_candidate.3)) + }); + let Some(selected_index) = eligible.first().copied() else { + if candidates + .iter() + .any(|(_, _, quality, _)| quality.assigned_node_count != topology.graph.len()) + { + return Err(CommunityDetectorError::IncompletePartition.into()); + } + return Err(CommunityDetectorError::DisconnectedPartition.into()); + }; + let candidate_agreement = candidate_agreement(&candidates); + let summaries = candidates + .iter() + .enumerate() + .map(|(index, (resolution, _, quality, digest))| { + let rejection_reason = if !quality_guards[index] { + Some("incremental quality regression".to_owned()) + } else if quality.assigned_node_count != topology.graph.len() { + Some("partition incomplete".to_owned()) + } else if quality.disconnected_community_count != 0 { + Some("partition disconnected".to_owned()) + } else if quality.modularity + tolerance < best_modularity { + Some("outside modularity tolerance".to_owned()) + } else if index != selected_index { + Some("lost deterministic quality tie-break".to_owned()) + } else { + None + }; + CommunityCandidateSummary { + resolution: *resolution, + modularity: quality.modularity, + weighted_mean_conductance: quality.weighted_mean_conductance, + disconnected_community_count: quality.disconnected_community_count, + size_violation_count: candidate_size_violations(quality), + non_isolate_singleton_count: quality.non_isolate_singleton_count, + partition_digest: digest.clone(), + selected: index == selected_index, + rejection_reason, + } + }) + .collect(); + let (_, selected_communities, selected_quality, _) = candidates.swap_remove(selected_index); + let communities = match request.previous { + Some(previous) => remap_communities_to_previous(&selected_communities, previous), + None => selected_communities, + }; + let mut quality = if request.previous.is_none() { + selected_quality + } else { + evaluate_graph_partition( + &topology.graph, + &communities, + base_resolution, + QualityEvaluationOptions { + identity, + topology_evidence: Some(topology.evidence.clone()), + pair_evidence: Some(&topology.pair_evidence), + max_quality_visits: request.limits.max_quality_visits, + witness_limit: request.limits.witness_limit, + }, + )? + }; + quality.candidate_summaries = summaries; + quality.candidate_agreement = candidate_agreement; + quality.selection_reason = if quality.candidate_summaries.len() == 1 { + "fixed resolution".to_owned() + } else { + "modularity plateau, size, conductance, fragmentation, partition digest".to_owned() + }; + let base_labels = label_communities_by_hub(legacy, &communities); + let signatures = community_member_signatures(&communities); + let identity = CommunityIdentity { + algorithm: QUALITY_CLUSTER_ALGORITHM.to_owned(), + topology: QUALITY_CLUSTER_TOPOLOGY.to_owned(), + quality: QUALITY_CLUSTER_QUALITY.to_owned(), + selector: selector_identity.to_owned(), + seed: COMPATIBILITY_CLUSTER_SEED, + limits: QUALITY_CLUSTER_LIMITS.to_owned(), + }; + Ok(CommunityResult { + communities, + base_labels, + signatures, + quality, + execution, + identity, + }) +} + +fn full_candidate( + topology: &super::topology::CommunityTopology, + resolution: f64, + base_resolution: f64, + request: &CommunityRequest<'_>, + identity: QualityIdentity<'_>, +) -> Result<(f64, Communities, PartitionQuality, String), CommunityError> { + let communities = quality_partition( + &topology.graph, + resolution, + request.exclude_hubs_percentile, + request.limits.max_levels, + request.limits.max_moves, + )? + .into_iter() + .enumerate() + .collect::(); + let quality = evaluate_graph_partition( + &topology.graph, + &communities, + base_resolution, + QualityEvaluationOptions { + identity, + topology_evidence: Some(topology.evidence.clone()), + pair_evidence: Some(&topology.pair_evidence), + max_quality_visits: request.limits.max_quality_visits, + witness_limit: request.limits.witness_limit, + }, + )?; + let digest = partition_digest(&communities); + Ok((resolution, communities, quality, digest)) +} + +fn map_preparation_fallback(reason: IncrementalPreparationFallback) -> FallbackReason { + match reason { + IncrementalPreparationFallback::InvalidLimits => FallbackReason::InvalidIncrementalLimits, + IncrementalPreparationFallback::RemovedNode => FallbackReason::RemovedNode, + IncrementalPreparationFallback::AffectedLimit => FallbackReason::AffectedRegionLimit, + IncrementalPreparationFallback::HubPolicy => FallbackReason::HubPolicy, + } +} + +fn quality_partition( + graph: &crate::cluster::WeightedGraph, + resolution: f64, + exclude_hubs_percentile: Option, + max_levels: usize, + max_moves: usize, +) -> Result>, CommunityDetectorError> { + let hubs = excluded_hubs(graph, exclude_hubs_percentile); + let isolates = (0..graph.len()) + .filter(|node| graph.degree_unweighted(*node) == 0 && !hubs.contains(node)) + .collect::>(); + let selected = (0..graph.len()) + .filter(|node| graph.degree_unweighted(*node) > 0 && !hubs.contains(node)) + .collect::>(); + let connected = graph.subgraph(&selected); + let mut communities = leiden(&connected, resolution, max_levels, max_moves)?; + communities.extend( + isolates + .into_iter() + .map(|node| vec![graph.ids[node].clone()]), + ); + reattach_hubs(graph, &hubs, &mut communities); + for members in &mut communities { + members.sort(); + } + communities.sort_by(|left, right| right.len().cmp(&left.len()).then_with(|| left.cmp(right))); + Ok(communities) +} + +fn candidate_agreement( + candidates: &[(f64, Communities, PartitionQuality, String)], +) -> Vec { + let mut agreement = Vec::new(); + for left in 0..candidates.len() { + for right in left + 1..candidates.len() { + agreement.push(CandidateAgreement { + left_resolution: candidates[left].0, + right_resolution: candidates[right].0, + adjusted_rand_index: adjusted_rand_index(&candidates[left].1, &candidates[right].1), + exact_membership: candidates[left].3 == candidates[right].3, + }); + } + } + agreement +} + +fn candidate_size_violations(quality: &PartitionQuality) -> usize { + quality + .communities + .values() + .filter(|community| { + community.member_count >= 10 + && (community.member_count as f64 / quality.assigned_node_count.max(1) as f64) + > 0.25 + }) + .count() +} + +fn partition_digest(communities: &Communities) -> String { + let mut hasher = Sha256::new(); + for members in communities.values() { + let mut members = members.clone(); + members.sort(); + for member in members { + hasher.update(member.as_bytes()); + hasher.update([0]); + } + hasher.update([0xff]); + } + format!("{:x}", hasher.finalize()) +} + +#[cfg(test)] +mod tests { + use super::*; + use compass_model::code_graph::{BuildMetadata, EdgeKind, EdgeRecord, NodeKind, NodeRecord}; + use compass_model::provenance::SourceAnchor; + + fn node(id: &str) -> NodeRecord { + NodeRecord { + id: id.to_owned(), + kind: NodeKind::Function, + roles: Vec::new(), + name: id.to_owned(), + qualified_name: format!("crate::{id}"), + language: Some("rust".to_owned()), + framework: None, + source: Some(SourceAnchor { + file: format!("src/{id}.rs"), + start_byte: 0, + end_byte: 1, + start_line: 1, + start_column: 0, + end_line: 1, + end_column: 1, + }), + details: None, + evidence: Vec::new(), + coverage: Vec::new(), + diagnostics: Vec::new(), + community: None, + } + } + + fn typed_graph() -> compass_model::code_graph::GraphDocument { + let mut document = compass_model::code_graph::GraphDocument::empty_v1(BuildMetadata { + builder_version: "test".to_owned(), + schema_fingerprint: "test".to_owned(), + source_tree_digest: "test".to_owned(), + configuration_digest: "test".to_owned(), + generation_id: "test".to_owned(), + source_commit: None, + }); + document.nodes = ["a", "b", "c", "d"].into_iter().map(node).collect(); + document.links = [("a", "b"), ("b", "c"), ("c", "d")] + .into_iter() + .enumerate() + .map(|(index, (source, target))| EdgeRecord { + id: format!("edge-{index}"), + key: format!("edge-{index}"), + source: source.to_owned(), + target: target.to_owned(), + kind: EdgeKind::Calls, + occurrence_rule: None, + relationship_site: None, + details: None, + evidence: Vec::new(), + weight: Some(1.0), + context: None, + deferred: false, + diagnostics: Vec::new(), + }) + .collect(); + document + } + + fn planted_graph() -> compass_model::code_graph::GraphDocument { + let mut document = compass_model::code_graph::GraphDocument::empty_v1(BuildMetadata { + builder_version: "test".to_owned(), + schema_fingerprint: "test".to_owned(), + source_tree_digest: "test".to_owned(), + configuration_digest: "test".to_owned(), + generation_id: "test".to_owned(), + source_commit: None, + }); + document.nodes = ["a", "b", "c", "d", "w", "x", "y", "z"] + .into_iter() + .map(node) + .collect(); + let pairs = [ + ("a", "b"), + ("a", "c"), + ("a", "d"), + ("b", "c"), + ("b", "d"), + ("c", "d"), + ("w", "x"), + ("w", "y"), + ("w", "z"), + ("x", "y"), + ("x", "z"), + ("y", "z"), + ("d", "w"), + ]; + document.links = pairs + .into_iter() + .enumerate() + .map(|(index, (source, target))| EdgeRecord { + id: format!("edge-{index:02}"), + key: format!("edge-{index:02}"), + source: source.to_owned(), + target: target.to_owned(), + kind: EdgeKind::Calls, + occurrence_rule: None, + relationship_site: None, + details: None, + evidence: Vec::new(), + weight: Some(1.0), + context: None, + deferred: false, + diagnostics: Vec::new(), + }) + .collect(); + document + } + + fn previous_from(result: &CommunityResult) -> PreviousCommunities { + result + .communities + .iter() + .flat_map(|(community, members)| { + members + .iter() + .map(move |member| (member.clone(), *community)) + }) + .collect() + } + + fn canonical_memberships(communities: &Communities) -> Vec> { + let mut memberships = communities.values().cloned().collect::>(); + for members in &mut memberships { + members.sort(); + } + memberships.sort(); + memberships + } + + #[test] + fn facade_preserves_compatibility_membership_and_adds_evidence() + -> Result<(), Box> { + let document = typed_graph(); + let legacy = document.to_legacy_document()?; + let expected = cluster(&legacy, ClusterOptions::default()); + let changed_sources = BTreeSet::new(); + let result = build_communities( + &document, + &CommunityRequest { + profile: CommunityProfile::CompatibilityV1, + resolution: ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits: CommunityLimits::default(), + }, + )?; + + assert_eq!(result.communities, expected); + assert_eq!(result.execution, CommunityExecution::Full); + assert_eq!(result.quality.assigned_node_count, document.nodes.len()); + assert_eq!(result.identity.algorithm, COMPATIBILITY_CLUSTER_ALGORITHM); + assert_eq!(result.signatures.len(), result.communities.len()); + assert_eq!(result.base_labels.len(), result.communities.len()); + Ok(()) + } + + #[test] + fn quality_profile_uses_typed_leiden_and_bounded_selection() + -> Result<(), Box> { + let document = typed_graph(); + let changed_sources = BTreeSet::new(); + let result = build_communities( + &document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Auto { base: 1.0 }, + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits: CommunityLimits::default(), + }, + ); + + let result = result?; + assert_eq!(result.identity.algorithm, QUALITY_CLUSTER_ALGORITHM); + assert_eq!(result.identity.topology, QUALITY_CLUSTER_TOPOLOGY); + assert_eq!(result.identity.selector, QUALITY_CLUSTER_SELECTOR); + assert_eq!(result.quality.candidate_summaries.len(), 3); + assert_eq!(result.quality.candidate_agreement.len(), 3); + assert_eq!( + result + .quality + .candidate_summaries + .iter() + .filter(|candidate| candidate.selected) + .count(), + 1 + ); + assert!( + result + .quality + .candidate_summaries + .iter() + .filter(|candidate| !candidate.selected) + .all(|candidate| candidate.rejection_reason.is_some()) + ); + assert_eq!(result.quality.disconnected_community_count, 0); + assert!(result.quality.quality_visit_count > 0); + assert!( + result + .quality + .topology_evidence + .as_ref() + .is_some_and(|evidence| !evidence.retained_relationship_counts.is_empty()) + ); + Ok(()) + } + + #[test] + fn quality_profile_fails_closed_at_the_quality_visit_limit() { + let document = typed_graph(); + let changed_sources = BTreeSet::new(); + let limits = CommunityLimits { + max_quality_visits: 0, + ..CommunityLimits::default() + }; + assert!(matches!( + build_communities( + &document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits, + }, + ), + Err(CommunityError::Quality( + CommunityQualityError::QualityLimitExceeded { limit: 0, .. } + )) + )); + } + + #[test] + fn quality_profile_rejects_invalid_limits_and_overflowed_auto_candidates() { + let document = typed_graph(); + let changed_sources = BTreeSet::new(); + let limits = CommunityLimits { + max_total_weight: f64::NAN, + ..CommunityLimits::default() + }; + assert!(matches!( + build_communities( + &document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits, + }, + ), + Err(CommunityError::InvalidProfile { .. }) + )); + assert!(matches!( + build_communities( + &document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Auto { base: f64::MAX }, + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits: CommunityLimits::default(), + }, + ), + Err(CommunityError::InvalidResolution { .. }) + )); + } + + #[test] + fn quality_incremental_run_preserves_frozen_assignments() + -> Result<(), Box> { + let document = typed_graph(); + let no_changes = BTreeSet::new(); + let initial = build_communities( + &document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &no_changes, + limits: CommunityLimits::default(), + }, + )?; + let previous = initial + .communities + .iter() + .flat_map(|(community, members)| { + members + .iter() + .map(move |member| (member.clone(), *community)) + }) + .collect::(); + let changed_sources = BTreeSet::from(["src/a.rs".to_owned()]); + let mut limits = CommunityLimits::default(); + limits.incremental.max_affected_fraction = 1.0; + let incremental = build_communities( + &document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous: Some(&previous), + incremental: true, + changed_sources: &changed_sources, + limits, + }, + )?; + + assert!(matches!( + incremental.execution, + CommunityExecution::Incremental { affected_nodes } if affected_nodes > 0 + )); + for node in ["c", "d"] { + let retained = incremental + .communities + .iter() + .find_map(|(community, members)| { + members.contains(&node.to_owned()).then_some(*community) + }); + assert_eq!(retained, previous.get(node).copied()); + } + assert_eq!(incremental.quality.disconnected_community_count, 0); + Ok(()) + } + + #[test] + fn quality_profile_recovers_planted_groups_and_is_permutation_invariant() + -> Result<(), Box> { + let document = planted_graph(); + let changes = BTreeSet::new(); + let request = CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Auto { base: 1.0 }, + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changes, + limits: CommunityLimits::default(), + }; + let expected = build_communities(&document, &request)?; + assert_eq!( + expected.communities.values().cloned().collect::>(), + vec![ + vec![ + "a".to_owned(), + "b".to_owned(), + "c".to_owned(), + "d".to_owned() + ], + vec![ + "w".to_owned(), + "x".to_owned(), + "y".to_owned(), + "z".to_owned() + ], + ] + ); + + let mut permuted = document; + permuted.nodes.reverse(); + permuted.links.reverse(); + let actual = build_communities(&permuted, &request)?; + assert_eq!(actual.communities, expected.communities); + assert_eq!( + actual + .quality + .candidate_summaries + .iter() + .map(|candidate| (&candidate.partition_digest, candidate.selected)) + .collect::>(), + expected + .quality + .candidate_summaries + .iter() + .map(|candidate| (&candidate.partition_digest, candidate.selected)) + .collect::>() + ); + Ok(()) + } + + #[test] + fn quality_incremental_edit_revert_rename_delete_sequence_is_stable() + -> Result<(), Box> { + let original = planted_graph(); + let no_changes = BTreeSet::new(); + let limits = CommunityLimits { + incremental: IncrementalClusterLimits { + max_affected_nodes: 4_096, + max_affected_fraction: 1.0, + }, + ..CommunityLimits::default() + }; + let build = |document: &compass_model::code_graph::GraphDocument, + previous: Option<&PreviousCommunities>, + changed_sources: &BTreeSet| + -> Result { + build_communities( + document, + &CommunityRequest { + profile: CommunityProfile::QualityV1, + resolution: ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous, + incremental: previous.is_some(), + changed_sources, + limits, + }, + ) + }; + + let initial = build(&original, None, &no_changes)?; + let initial_memberships = canonical_memberships(&initial.communities); + + let mut edited_graph = original.clone(); + let edited_edge = edited_graph + .links + .iter_mut() + .find(|edge| edge.source == "a" && edge.target == "b") + .ok_or_else(|| std::io::Error::other("missing editable edge"))?; + edited_edge.kind = EdgeKind::References; + let edited_sources = BTreeSet::from(["src/a.rs".to_owned(), "src/b.rs".to_owned()]); + let edited = build( + &edited_graph, + Some(&previous_from(&initial)), + &edited_sources, + )?; + assert_eq!(edited.quality.disconnected_community_count, 0); + + let reverted = build(&original, Some(&previous_from(&edited)), &edited_sources)?; + assert_eq!( + canonical_memberships(&reverted.communities), + initial_memberships + ); + + let mut renamed_graph = original.clone(); + let renamed_node = renamed_graph + .nodes + .iter_mut() + .find(|node| node.id == "a") + .ok_or_else(|| std::io::Error::other("missing rename node"))?; + renamed_node.id = "a2".to_owned(); + renamed_node.name = "a2".to_owned(); + renamed_node.qualified_name = "crate::a2".to_owned(); + if let Some(source) = renamed_node.source.as_mut() { + source.file = "src/a2.rs".to_owned(); + } + for edge in &mut renamed_graph.links { + if edge.source == "a" { + edge.source = "a2".to_owned(); + } + if edge.target == "a" { + edge.target = "a2".to_owned(); + } + } + let renamed_sources = BTreeSet::from(["src/a.rs".to_owned(), "src/a2.rs".to_owned()]); + let renamed = build( + &renamed_graph, + Some(&previous_from(&reverted)), + &renamed_sources, + )?; + assert!(matches!( + renamed.execution, + CommunityExecution::FullFallback { + reason: FallbackReason::RemovedNode, + .. + } + )); + assert_eq!(renamed.quality.disconnected_community_count, 0); + + let mut deleted_graph = renamed_graph; + deleted_graph.nodes.retain(|node| node.id != "a2"); + deleted_graph + .links + .retain(|edge| edge.source != "a2" && edge.target != "a2"); + let deleted = build( + &deleted_graph, + Some(&previous_from(&renamed)), + &BTreeSet::from(["src/a2.rs".to_owned()]), + )?; + assert!(matches!( + deleted.execution, + CommunityExecution::FullFallback { + reason: FallbackReason::RemovedNode, + .. + } + )); + assert_eq!( + deleted.quality.assigned_node_count, + deleted_graph.nodes.len() + ); + assert_eq!(deleted.quality.disconnected_community_count, 0); + Ok(()) + } +} diff --git a/crates/compass-graph/src/community/identity.rs b/crates/compass-graph/src/community/identity.rs new file mode 100644 index 000000000..12cf93e91 --- /dev/null +++ b/crates/compass-graph/src/community/identity.rs @@ -0,0 +1,22 @@ +//! Stable identities for the currently shipped compatibility detector. + +/// Native seeded Louvain implementation used by the compatibility profile. +pub const COMPATIBILITY_CLUSTER_ALGORITHM: &str = "seeded-louvain/v1"; +/// Undirected weighted projection used by the compatibility profile. +pub const COMPATIBILITY_CLUSTER_TOPOLOGY: &str = "legacy-undirected/v1"; +/// Partition evidence contract used before the quality-profile cutover. +pub const COMPATIBILITY_CLUSTER_QUALITY: &str = "density/v1"; +/// Fixed-resolution selection policy used by the compatibility profile. +pub const COMPATIBILITY_CLUSTER_SELECTOR: &str = "fixed-resolution/v1"; +/// Seed retained for deterministic compatibility with the historical detector. +pub const COMPATIBILITY_CLUSTER_SEED: u32 = 42; +/// Canonical profile encoding of [`COMPATIBILITY_CLUSTER_SEED`]. +pub const COMPATIBILITY_CLUSTER_SEED_TEXT: &str = "42"; +/// Work-limit policy for the compatibility detector. +pub const COMPATIBILITY_CLUSTER_LIMITS: &str = "community-limits/v1"; + +pub const QUALITY_CLUSTER_ALGORITHM: &str = "seeded-leiden-modularity/v1"; +pub const QUALITY_CLUSTER_TOPOLOGY: &str = "typed-evidence-undirected/v1"; +pub const QUALITY_CLUSTER_QUALITY: &str = "community-quality/v1"; +pub const QUALITY_CLUSTER_SELECTOR: &str = "bounded-multiresolution/v1"; +pub const QUALITY_CLUSTER_LIMITS: &str = "community-limits/v1"; diff --git a/crates/compass-graph/src/community/incremental.rs b/crates/compass-graph/src/community/incremental.rs new file mode 100644 index 000000000..990942490 --- /dev/null +++ b/crates/compass-graph/src/community/incremental.rs @@ -0,0 +1,322 @@ +use std::collections::{BTreeMap, BTreeSet, HashMap}; + +use compass_model::code_graph::GraphDocument; + +use super::leiden::{CommunityDetectorError, leiden_anchored}; +use crate::cluster::{Communities, IncrementalClusterLimits, WeightedGraph}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum IncrementalPreparationFallback { + InvalidLimits, + RemovedNode, + AffectedLimit, + HubPolicy, +} + +pub(crate) enum IncrementalPreparation { + Ready(Box), + Unchanged(Communities), + Fallback { + affected_nodes: usize, + reason: IncrementalPreparationFallback, + }, +} + +pub(crate) struct AnchoredTopology { + graph: WeightedGraph, + anchor_positions: BTreeSet, + anchor_communities: BTreeMap, + frozen: Communities, + used_ids: BTreeSet, + previous: HashMap, + next_id: usize, + pub affected_nodes: usize, +} + +impl AnchoredTopology { + pub fn cluster( + &self, + resolution: f64, + max_levels: usize, + max_moves: usize, + ) -> Result, CommunityDetectorError> { + let local = leiden_anchored( + &self.graph, + resolution, + &self.anchor_positions, + max_levels, + max_moves, + )?; + let mut output = self.frozen.clone(); + let mut used = self.used_ids.clone(); + let mut next = self.next_id; + for members in local { + let anchors = members + .iter() + .filter_map(|member| self.anchor_communities.get(member).copied()) + .collect::>(); + if anchors.len() > 1 { + return Ok(None); + } + let real_members = members + .into_iter() + .filter(|member| !self.anchor_communities.contains_key(member)) + .collect::>(); + if real_members.is_empty() { + continue; + } + let community = if let Some(anchor) = anchors.first().copied() { + anchor + } else { + let mut overlaps = real_members + .iter() + .filter_map(|member| self.previous.get(member).copied()) + .fold(BTreeMap::::new(), |mut counts, community| { + *counts.entry(community).or_default() += 1; + counts + }) + .into_iter() + .collect::>(); + overlaps.sort_by_key(|(community, count)| (std::cmp::Reverse(*count), *community)); + if let Some(reused) = overlaps + .into_iter() + .map(|(community, _)| community) + .find(|community| used.insert(*community)) + { + reused + } else { + while used.contains(&next) { + next = next.saturating_add(1); + } + let assigned = next; + used.insert(assigned); + next = next.saturating_add(1); + assigned + } + }; + output.entry(community).or_default().extend(real_members); + } + output.retain(|_, members| !members.is_empty()); + for members in output.values_mut() { + members.sort(); + members.dedup(); + } + Ok(Some(output)) + } +} + +pub(crate) fn prepare_anchored_topology( + document: &GraphDocument, + graph: &WeightedGraph, + previous: &HashMap, + changed_sources: &BTreeSet, + limits: IncrementalClusterLimits, + exclude_hubs_percentile: Option, +) -> IncrementalPreparation { + if exclude_hubs_percentile.is_some() { + return IncrementalPreparation::Fallback { + affected_nodes: graph.len(), + reason: IncrementalPreparationFallback::HubPolicy, + }; + } + if limits.max_affected_nodes == 0 + || !limits.max_affected_fraction.is_finite() + || limits.max_affected_fraction <= 0.0 + { + return IncrementalPreparation::Fallback { + affected_nodes: graph.len(), + reason: IncrementalPreparationFallback::InvalidLimits, + }; + } + let graph_positions = graph.position_map(); + if previous + .keys() + .any(|node| !graph_positions.contains_key(node)) + { + return IncrementalPreparation::Fallback { + affected_nodes: graph.len(), + reason: IncrementalPreparationFallback::RemovedNode, + }; + } + let sources = document + .nodes + .iter() + .map(|node| { + ( + node.id.as_str(), + node.source + .as_ref() + .map(|source| source.file.replace('\\', "/")), + ) + }) + .collect::>(); + let mut affected = graph + .ids + .iter() + .enumerate() + .filter_map(|(position, id)| { + let changed = sources + .get(id.as_str()) + .and_then(|source| source.as_ref()) + .is_some_and(|source| changed_sources.contains(source)); + (!previous.contains_key(id) || changed).then_some(position) + }) + .collect::>(); + if affected.is_empty() { + return IncrementalPreparation::Unchanged(communities_from_previous(graph, previous)); + } + let touched = affected + .iter() + .filter_map(|position| previous.get(&graph.ids[*position]).copied()) + .collect::>(); + for (position, id) in graph.ids.iter().enumerate() { + if previous + .get(id) + .is_some_and(|community| touched.contains(community)) + { + affected.insert(position); + } + } + let fraction_limit = + ((graph.len() as f64 * limits.max_affected_fraction).ceil() as usize).max(1); + let affected_limit = limits.max_affected_nodes.min(fraction_limit); + if affected.len() > affected_limit { + return IncrementalPreparation::Fallback { + affected_nodes: affected.len(), + reason: IncrementalPreparationFallback::AffectedLimit, + }; + } + + let mut adjacent_frozen = BTreeSet::::new(); + for (left, right, _) in graph.edges() { + if affected.contains(&left) && !affected.contains(&right) { + if let Some(community) = previous.get(&graph.ids[right]) { + adjacent_frozen.insert(*community); + } + } else if affected.contains(&right) + && !affected.contains(&left) + && let Some(community) = previous.get(&graph.ids[left]) + { + adjacent_frozen.insert(*community); + } + } + let mut ids = affected + .iter() + .map(|position| graph.ids[*position].clone()) + .collect::>(); + let anchor_ids = adjacent_frozen + .iter() + .map(|community| { + ( + *community, + format!("\0compass-community-anchor:{community}"), + ) + }) + .collect::>(); + ids.extend(anchor_ids.values().cloned()); + let members = ids.iter().cloned().map(|id| BTreeSet::from([id])).collect(); + let mut local = WeightedGraph::new(ids.clone(), members); + let local_positions = ids + .iter() + .enumerate() + .map(|(position, id)| (id.as_str(), position)) + .collect::>(); + let full_to_local = affected + .iter() + .map(|position| (*position, local_positions[graph.ids[*position].as_str()])) + .collect::>(); + for (left, right, weight) in graph.edges() { + match (full_to_local.get(&left), full_to_local.get(&right)) { + (Some(local_left), Some(local_right)) => { + local.add_edge(*local_left, *local_right, weight); + } + (Some(local_node), None) => { + if let Some(community) = previous.get(&graph.ids[right]) + && let Some(anchor) = anchor_ids.get(community) + { + local.add_edge(*local_node, local_positions[anchor.as_str()], weight); + } + } + (None, Some(local_node)) => { + if let Some(community) = previous.get(&graph.ids[left]) + && let Some(anchor) = anchor_ids.get(community) + { + local.add_edge(local_positions[anchor.as_str()], *local_node, weight); + } + } + (None, None) => {} + } + } + let anchor_communities = anchor_ids + .into_iter() + .map(|(community, id)| (id, community)) + .collect::>(); + let anchor_positions = anchor_communities + .keys() + .map(|id| local_positions[id.as_str()]) + .collect(); + let frozen = previous + .iter() + .filter_map(|(id, community)| { + let position = graph_positions.get(id)?; + (!affected.contains(position)).then_some((*community, id.clone())) + }) + .fold(Communities::new(), |mut output, (community, id)| { + output.entry(community).or_default().push(id); + output + }); + let used_ids = frozen.keys().copied().collect::>(); + let next_id = previous + .values() + .copied() + .max() + .map_or(0, |maximum| maximum.saturating_add(1)); + IncrementalPreparation::Ready(Box::new(AnchoredTopology { + graph: local, + anchor_positions, + anchor_communities, + frozen, + used_ids, + previous: previous.clone(), + next_id, + affected_nodes: affected.len(), + })) +} + +pub(crate) fn baseline_partition( + graph: &WeightedGraph, + previous: &HashMap, +) -> Communities { + let mut output = communities_from_previous(graph, previous); + let mut used = output.keys().copied().collect::>(); + let mut next = previous + .values() + .copied() + .max() + .map_or(0, |maximum| maximum.saturating_add(1)); + for id in &graph.ids { + if previous.contains_key(id) { + continue; + } + while used.contains(&next) { + next = next.saturating_add(1); + } + output.insert(next, vec![id.clone()]); + used.insert(next); + next = next.saturating_add(1); + } + output +} + +fn communities_from_previous( + graph: &WeightedGraph, + previous: &HashMap, +) -> Communities { + let mut output = Communities::new(); + for id in &graph.ids { + if let Some(community) = previous.get(id) { + output.entry(*community).or_default().push(id.clone()); + } + } + output +} diff --git a/crates/compass-graph/src/community/leiden.rs b/crates/compass-graph/src/community/leiden.rs new file mode 100644 index 000000000..3cd0b9963 --- /dev/null +++ b/crates/compass-graph/src/community/leiden.rs @@ -0,0 +1,470 @@ +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use thiserror::Error; + +use crate::cluster::{PythonRandom, WeightedGraph}; + +const MODULARITY_THRESHOLD: f64 = 1e-4; + +#[derive(Clone, Copy)] +struct MovePolicy<'a> { + coarse: Option<&'a [usize]>, + locked: Option<&'a BTreeSet>, + preserve_source_connectivity: bool, +} + +#[derive(Clone, Debug, Error, PartialEq)] +pub enum CommunityDetectorError { + #[error( + "community detector local_moves requires {required} moves, exceeds limit {limit} after {processed} moves" + )] + MoveLimitExceeded { + required: usize, + limit: usize, + processed: usize, + }, + #[error( + "community detector levels requires {required} levels, exceeds limit {limit} after {processed} levels" + )] + LevelLimitExceeded { + required: usize, + limit: usize, + processed: usize, + }, + #[error("community detector produced a disconnected refined community")] + DisconnectedPartition, + #[error("community detector produced an incomplete partition")] + IncompletePartition, +} + +pub(crate) fn leiden( + graph: &WeightedGraph, + resolution: f64, + max_levels: usize, + max_moves: usize, +) -> Result>, CommunityDetectorError> { + if graph.edge_count() == 0 { + return Ok(graph.ids.iter().cloned().map(|id| vec![id]).collect()); + } + if max_levels == 0 { + return Err(CommunityDetectorError::LevelLimitExceeded { + required: 1, + limit: 0, + processed: 0, + }); + } + let mut current = graph.clone(); + let mut random = PythonRandom::seeded(42); + let mut moves = 0usize; + let mut best_modularity = f64::NEG_INFINITY; + let mut best = current + .members + .iter() + .map(|members| members.iter().cloned().collect::>()) + .collect::>(); + + for level in 0..max_levels { + let coarse = local_move( + ¤t, + resolution, + MovePolicy { + coarse: None, + locked: None, + preserve_source_connectivity: false, + }, + &mut random, + &mut moves, + max_moves, + )?; + let coarse_assignment = assignment(&coarse, current.len()); + let refined = local_move( + ¤t, + resolution, + MovePolicy { + coarse: Some(&coarse_assignment), + locked: None, + preserve_source_connectivity: true, + }, + &mut random, + &mut moves, + max_moves, + )?; + if !partition_connected(¤t, &refined) { + return Err(CommunityDetectorError::DisconnectedPartition); + } + let next_modularity = partition_modularity(¤t, &refined, resolution); + best = original_members(¤t, &refined); + if next_modularity - best_modularity <= MODULARITY_THRESHOLD + || refined.len() == current.len() + { + break; + } + if level + 1 == max_levels { + return Err(CommunityDetectorError::LevelLimitExceeded { + required: max_levels.saturating_add(1), + limit: max_levels, + processed: max_levels, + }); + } + best_modularity = next_modularity; + current = aggregate(¤t, &refined); + if current.len() <= 1 { + break; + } + } + for members in &mut best { + members.sort(); + } + best.sort_by(|left, right| right.len().cmp(&left.len()).then_with(|| left.cmp(right))); + Ok(best) +} + +pub(crate) fn leiden_anchored( + graph: &WeightedGraph, + resolution: f64, + anchors: &BTreeSet, + max_levels: usize, + max_moves: usize, +) -> Result>, CommunityDetectorError> { + if graph.edge_count() == 0 { + return Ok(graph.ids.iter().cloned().map(|id| vec![id]).collect()); + } + if max_levels == 0 { + return Err(CommunityDetectorError::LevelLimitExceeded { + required: 1, + limit: 0, + processed: 0, + }); + } + let mut random = PythonRandom::seeded(42); + let mut moves = 0usize; + let coarse = local_move( + graph, + resolution, + MovePolicy { + coarse: None, + locked: Some(anchors), + preserve_source_connectivity: false, + }, + &mut random, + &mut moves, + max_moves, + )?; + let coarse_assignment = assignment(&coarse, graph.len()); + let refined = local_move( + graph, + resolution, + MovePolicy { + coarse: Some(&coarse_assignment), + locked: Some(anchors), + preserve_source_connectivity: true, + }, + &mut random, + &mut moves, + max_moves, + )?; + if !partition_connected(graph, &refined) { + return Err(CommunityDetectorError::DisconnectedPartition); + } + let mut output = original_members(graph, &refined); + for members in &mut output { + members.sort(); + } + output.sort_by(|left, right| right.len().cmp(&left.len()).then_with(|| left.cmp(right))); + Ok(output) +} + +fn local_move( + graph: &WeightedGraph, + resolution: f64, + policy: MovePolicy<'_>, + random: &mut PythonRandom, + moves: &mut usize, + max_moves: usize, +) -> Result>, CommunityDetectorError> { + let total_weight = graph.total_weight(); + let denominator = 2.0 * total_weight.powi(2); + let degrees = (0..graph.len()) + .map(|node| graph.degree_weighted(node)) + .collect::>(); + let mut node_to_community = (0..graph.len()).collect::>(); + let mut members = (0..graph.len()) + .map(|node| BTreeSet::from([node])) + .collect::>(); + let mut totals = degrees.clone(); + let mut nodes = (0..graph.len()).collect::>(); + random.shuffle(&mut nodes); + let mut previous_modularity = partition_modularity(graph, &members, resolution); + loop { + let previous_members = members.clone(); + let mut pass_moves = 0usize; + for node in &nodes { + if policy.locked.is_some_and(|locked| locked.contains(node)) { + continue; + } + let old = node_to_community[*node]; + // Refinement grows connected subcommunities by merging only a + // singleton source into a neighboring group inside its coarse + // community. The joining edge proves the destination remains + // connected, while removing a singleton cannot disconnect its + // source. This is both the Leiden refinement invariant and avoids + // an O(VE) articulation search for every proposed move. + if policy.preserve_source_connectivity && members[old].len() != 1 { + continue; + } + let degree = degrees[*node]; + let mut neighbor_weights = BTreeMap::::new(); + for (neighbor, weight) in graph.neighbors(*node) { + if neighbor == node { + continue; + } + if policy + .coarse + .is_some_and(|partition| partition[*neighbor] != partition[*node]) + { + continue; + } + *neighbor_weights + .entry(node_to_community[*neighbor]) + .or_default() += weight; + } + totals[old] -= degree; + let old_weight = neighbor_weights.get(&old).copied().unwrap_or_default(); + let remove_cost = + -old_weight / total_weight + resolution * totals[old] * degree / denominator; + let mut best = old; + let mut best_gain = 0.0; + for (candidate, weight) in neighbor_weights { + if candidate == old { + continue; + } + let gain = remove_cost + weight / total_weight + - resolution * totals[candidate] * degree / denominator; + if gain > best_gain { + best = candidate; + best_gain = gain; + } + } + totals[best] += degree; + if best != old { + if *moves == max_moves { + return Err(CommunityDetectorError::MoveLimitExceeded { + required: (*moves).saturating_add(1), + limit: max_moves, + processed: *moves, + }); + } + members[old].remove(node); + members[best].insert(*node); + node_to_community[*node] = best; + *moves += 1; + pass_moves += 1; + } + } + if pass_moves == 0 { + break; + } + let modularity = partition_modularity(graph, &members, resolution); + if modularity <= previous_modularity + f64::EPSILON { + members = previous_members; + break; + } + previous_modularity = modularity; + } + let mut retained = members + .into_iter() + .filter(|community| !community.is_empty()) + .collect::>(); + retained.sort_by_key(|community| community.first().copied().unwrap_or_default()); + Ok(retained) +} + +fn assignment(partition: &[BTreeSet], node_count: usize) -> Vec { + let mut assignments = vec![usize::MAX; node_count]; + for (community, members) in partition.iter().enumerate() { + for member in members { + assignments[*member] = community; + } + } + assignments +} + +fn partition_connected(graph: &WeightedGraph, partition: &[BTreeSet]) -> bool { + partition.iter().all(|community| { + let Some(start) = community.first().copied() else { + return true; + }; + let mut visited = BTreeSet::from([start]); + let mut queue = VecDeque::from([start]); + while let Some(node) = queue.pop_front() { + for (neighbor, _) in graph.neighbors(node) { + if community.contains(neighbor) && visited.insert(*neighbor) { + queue.push_back(*neighbor); + } + } + } + visited.len() == community.len() + }) +} + +fn partition_modularity( + graph: &WeightedGraph, + partition: &[BTreeSet], + resolution: f64, +) -> f64 { + let total_weight = graph.total_weight(); + if total_weight == 0.0 { + return 0.0; + } + partition + .iter() + .map(|community| { + let internal = graph + .edges() + .filter(|(left, right, _)| community.contains(left) && community.contains(right)) + .map(|(_, _, weight)| weight) + .sum::(); + let volume = community + .iter() + .map(|node| graph.degree_weighted(*node)) + .sum::(); + internal / total_weight - resolution * (volume / (2.0 * total_weight)).powi(2) + }) + .sum() +} + +fn original_members(graph: &WeightedGraph, partition: &[BTreeSet]) -> Vec> { + partition + .iter() + .map(|community| { + community + .iter() + .flat_map(|node| graph.members[*node].iter().cloned()) + .collect() + }) + .collect() +} + +fn aggregate(graph: &WeightedGraph, partition: &[BTreeSet]) -> WeightedGraph { + let assignments = assignment(partition, graph.len()); + let members = partition + .iter() + .map(|community| { + community + .iter() + .flat_map(|node| graph.members[*node].iter().cloned()) + .collect() + }) + .collect::>(); + let ids = (0..partition.len()).map(|id| id.to_string()).collect(); + let mut output = WeightedGraph::new(ids, members); + for (left, right, weight) in graph.edges() { + output.add_edge(assignments[left], assignments[right], weight); + } + output +} + +#[cfg(test)] +mod tests { + use super::*; + + fn graph(edges: &[(usize, usize)]) -> WeightedGraph { + let ids = (0..8).map(|node| node.to_string()).collect::>(); + let members = ids.iter().cloned().map(|id| BTreeSet::from([id])).collect(); + let mut graph = WeightedGraph::new(ids, members); + for (left, right) in edges { + graph.add_edge(*left, *right, 1.0); + } + graph + } + + #[test] + fn separates_dense_groups_and_is_deterministic() -> Result<(), CommunityDetectorError> { + let graph = graph(&[ + (0, 1), + (0, 2), + (0, 3), + (1, 2), + (1, 3), + (2, 3), + (4, 5), + (4, 6), + (4, 7), + (5, 6), + (5, 7), + (6, 7), + (3, 4), + ]); + let first = leiden(&graph, 1.0, 10, 10_000)?; + let second = leiden(&graph, 1.0, 10, 10_000)?; + assert_eq!(first, second); + assert_eq!(first.len(), 2); + assert_eq!(first[0], ["0", "1", "2", "3"]); + assert_eq!(first[1], ["4", "5", "6", "7"]); + Ok(()) + } + + #[test] + fn respects_the_move_limit() { + let graph = graph(&[(0, 1), (1, 2), (2, 3)]); + assert!(matches!( + leiden(&graph, 1.0, 10, 0), + Err(CommunityDetectorError::MoveLimitExceeded { limit: 0, .. }) + )); + } + + #[test] + fn fails_closed_when_another_level_is_required() { + let graph = graph(&[ + (0, 1), + (0, 2), + (0, 3), + (1, 2), + (1, 3), + (2, 3), + (4, 5), + (4, 6), + (4, 7), + (5, 6), + (5, 7), + (6, 7), + (3, 4), + ]); + assert!(matches!( + leiden(&graph, 1.0, 1, 10_000), + Err(CommunityDetectorError::LevelLimitExceeded { + required: 2, + limit: 1, + processed: 1, + }) + )); + } + + #[test] + fn articulation_fixture_never_returns_a_disconnected_community() + -> Result<(), CommunityDetectorError> { + let graph = graph(&[ + (0, 1), + (0, 2), + (1, 2), + (2, 3), + (3, 4), + (4, 5), + (4, 6), + (5, 6), + ]); + let partition = leiden(&graph, 0.75, 10, 10_000)?; + let positions = graph.position_map(); + let indexed = partition + .iter() + .map(|members| { + members + .iter() + .map(|member| positions[member]) + .collect::>() + }) + .collect::>(); + assert!(partition_connected(&graph, &indexed)); + Ok(()) + } +} diff --git a/crates/compass-graph/src/community/mod.rs b/crates/compass-graph/src/community/mod.rs new file mode 100644 index 000000000..480185cb3 --- /dev/null +++ b/crates/compass-graph/src/community/mod.rs @@ -0,0 +1,31 @@ +//! Versioned community detection identities and quality evidence. + +mod artifact; +mod build; +mod identity; +mod incremental; +mod leiden; +mod quality; +mod topology; + +pub use artifact::{ + COMMUNITY_QUALITY_SCHEMA, CommunityQualityArtifact, CommunityQualityArtifactError, +}; +pub use build::{ + CommunityError, CommunityExecution, CommunityIdentity, CommunityLimits, CommunityProfile, + CommunityRequest, CommunityResult, FallbackReason, PreviousCommunities, ResolutionPolicy, + build_communities, +}; +pub use identity::{ + COMPATIBILITY_CLUSTER_ALGORITHM, COMPATIBILITY_CLUSTER_LIMITS, COMPATIBILITY_CLUSTER_QUALITY, + COMPATIBILITY_CLUSTER_SEED, COMPATIBILITY_CLUSTER_SEED_TEXT, COMPATIBILITY_CLUSTER_SELECTOR, + COMPATIBILITY_CLUSTER_TOPOLOGY, QUALITY_CLUSTER_ALGORITHM, QUALITY_CLUSTER_LIMITS, + QUALITY_CLUSTER_QUALITY, QUALITY_CLUSTER_SELECTOR, QUALITY_CLUSTER_TOPOLOGY, +}; +pub use leiden::CommunityDetectorError; +pub(crate) use quality::compatibility_density_scores; +pub use quality::{ + CandidateAgreement, CommunityCandidateSummary, CommunityQuality, CommunityQualityError, + PartitionQuality, adjusted_mutual_information, adjusted_rand_index, evaluate_partition_quality, +}; +pub use topology::{CommunityTopologyError, TopologyEvidence}; diff --git a/crates/compass-graph/src/community/quality.rs b/crates/compass-graph/src/community/quality.rs new file mode 100644 index 000000000..233b818b9 --- /dev/null +++ b/crates/compass-graph/src/community/quality.rs @@ -0,0 +1,806 @@ +use std::collections::{BTreeMap, BTreeSet, VecDeque}; + +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use super::identity::{ + COMPATIBILITY_CLUSTER_ALGORITHM, COMPATIBILITY_CLUSTER_LIMITS, COMPATIBILITY_CLUSTER_QUALITY, + COMPATIBILITY_CLUSTER_SEED, COMPATIBILITY_CLUSTER_SELECTOR, COMPATIBILITY_CLUSTER_TOPOLOGY, +}; +use super::topology::{ProjectedPairEvidence, TopologyEvidence}; +use crate::cluster::{Communities, WeightedGraph}; +use compass_model::GraphDocument; + +pub const DEFAULT_MAX_QUALITY_VISITS: usize = 50_000_000; +pub const DEFAULT_QUALITY_WITNESS_LIMIT: usize = 8; + +/// Quality evidence for one detected community. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommunityQuality { + pub community: usize, + pub member_count: usize, + pub isolate: bool, + pub connected_component_count: usize, + pub internal_edge_count: usize, + pub internal_weight: f64, + pub boundary_edge_count: usize, + pub boundary_weight: f64, + pub volume: f64, + pub density: f64, + pub conductance: f64, + pub modularity_contribution: f64, + pub relationship_mix: BTreeMap, + pub relation_strength_mix: BTreeMap, + pub evidence_confidence_mix: BTreeMap, + pub witness_node_ids: Vec, + pub witness_edge_ids: Vec, + pub omitted_witness_node_count: usize, + pub omitted_witness_edge_count: usize, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CommunityCandidateSummary { + pub resolution: f64, + pub modularity: f64, + pub weighted_mean_conductance: f64, + pub disconnected_community_count: usize, + pub size_violation_count: usize, + pub non_isolate_singleton_count: usize, + pub partition_digest: String, + pub selected: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub rejection_reason: Option, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct CandidateAgreement { + pub left_resolution: f64, + pub right_resolution: f64, + pub adjusted_rand_index: f64, + pub exact_membership: bool, +} + +/// Complete evidence for a partition evaluated on one named topology. +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct PartitionQuality { + pub assigned_node_count: usize, + pub omitted_node_count: usize, + pub community_count: usize, + pub non_isolate_singleton_count: usize, + pub disconnected_community_count: usize, + pub modularity: f64, + pub weighted_mean_conductance: f64, + pub worst_conductance: f64, + pub largest_community_fraction: f64, + pub resolution: f64, + pub algorithm: String, + pub topology: String, + pub quality: String, + pub selector: String, + pub seed: u32, + pub limits: String, + pub quality_visit_count: usize, + pub quality_visit_limit: usize, + pub witness_limit: usize, + pub omitted_witness_node_count: usize, + pub omitted_witness_edge_count: usize, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub candidate_summaries: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub candidate_agreement: Vec, + pub selection_reason: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub topology_evidence: Option, + pub communities: BTreeMap, +} + +#[derive(Clone, Copy)] +pub(crate) struct QualityIdentity<'a> { + pub algorithm: &'a str, + pub topology: &'a str, + pub quality: &'a str, + pub selector: &'a str, + pub seed: u32, + pub limits: &'a str, +} + +/// A partition cannot be evaluated when it does not unambiguously assign the +/// detector topology. +#[derive(Clone, Debug, Error, PartialEq)] +pub enum CommunityQualityError { + #[error("community {community} contains unknown node `{node}`")] + UnknownMember { community: usize, node: String }, + #[error("node `{node}` is assigned to communities {first} and {second}")] + DuplicateMember { + node: String, + first: usize, + second: usize, + }, + #[error("community quality resolution must be finite and positive, got {resolution}")] + InvalidResolution { resolution: f64 }, + #[error( + "community quality {stage} requires {required} visits, exceeds limit {limit} after {processed}" + )] + QualityLimitExceeded { + stage: &'static str, + required: usize, + limit: usize, + processed: usize, + }, +} + +/// Evaluate a complete or partial partition on the exact topology consumed by +/// the compatibility detector. +pub fn evaluate_partition_quality( + document: &GraphDocument, + communities: &Communities, + resolution: f64, +) -> Result { + if !resolution.is_finite() || resolution <= 0.0 { + return Err(CommunityQualityError::InvalidResolution { resolution }); + } + let graph = WeightedGraph::from_document(document); + evaluate_graph_partition( + &graph, + communities, + resolution, + QualityEvaluationOptions { + identity: QualityIdentity { + algorithm: COMPATIBILITY_CLUSTER_ALGORITHM, + topology: COMPATIBILITY_CLUSTER_TOPOLOGY, + quality: COMPATIBILITY_CLUSTER_QUALITY, + selector: COMPATIBILITY_CLUSTER_SELECTOR, + seed: COMPATIBILITY_CLUSTER_SEED, + limits: COMPATIBILITY_CLUSTER_LIMITS, + }, + topology_evidence: None, + pair_evidence: None, + max_quality_visits: DEFAULT_MAX_QUALITY_VISITS, + witness_limit: DEFAULT_QUALITY_WITNESS_LIMIT, + }, + ) +} + +/// Preserve the public cohesion projection while sourcing its edge inventory +/// from the same canonical topology as clustering and richer quality evidence. +pub(crate) fn compatibility_density_scores( + document: &GraphDocument, + communities: &Communities, +) -> BTreeMap { + let graph = WeightedGraph::from_document(document); + density_scores(&graph, communities) +} + +fn density_scores(graph: &WeightedGraph, communities: &Communities) -> BTreeMap { + let positions = graph.position_map(); + let mut assignments = vec![None; graph.len()]; + for (community, members) in communities { + for member in members { + if let Some(position) = positions.get(member) { + assignments[*position] = Some(*community); + } + } + } + let mut internal_edges = BTreeMap::::new(); + for (left, right, _) in graph.edges() { + if let Some(community) = assignments[left] + && assignments[right] == Some(community) + { + *internal_edges.entry(community).or_default() += 1; + } + } + communities + .iter() + .map(|(community, members)| { + let possible = members + .len() + .saturating_mul(members.len().saturating_sub(1)) + / 2; + let density = if possible == 0 { + 1.0 + } else { + internal_edges.get(community).copied().unwrap_or_default() as f64 / possible as f64 + }; + (*community, density) + }) + .collect() +} + +pub(crate) struct QualityEvaluationOptions<'a> { + pub(crate) identity: QualityIdentity<'a>, + pub(crate) topology_evidence: Option, + pub(crate) pair_evidence: Option<&'a [ProjectedPairEvidence]>, + pub(crate) max_quality_visits: usize, + pub(crate) witness_limit: usize, +} + +pub(crate) fn evaluate_graph_partition( + graph: &WeightedGraph, + communities: &Communities, + resolution: f64, + options: QualityEvaluationOptions<'_>, +) -> Result { + let QualityEvaluationOptions { + identity, + topology_evidence, + pair_evidence, + max_quality_visits, + witness_limit, + } = options; + let assigned_members = communities.values().map(Vec::len).sum::(); + let quality_visit_count = graph + .len() + .saturating_add(assigned_members.saturating_mul(2)) + .saturating_add(graph.edge_count().saturating_mul(4)); + if quality_visit_count > max_quality_visits { + return Err(CommunityQualityError::QualityLimitExceeded { + stage: "partition_metrics", + required: quality_visit_count, + limit: max_quality_visits, + processed: 0, + }); + } + let positions = graph.position_map(); + let mut assignments = vec![None; graph.len()]; + for (community, members) in communities { + for member in members { + let Some(position) = positions.get(member) else { + return Err(CommunityQualityError::UnknownMember { + community: *community, + node: member.clone(), + }); + }; + if let Some(first) = assignments[*position] { + return Err(CommunityQualityError::DuplicateMember { + node: member.clone(), + first, + second: *community, + }); + } + assignments[*position] = Some(*community); + } + } + + let total_weight = graph.total_weight(); + let total_volume = 2.0 * total_weight; + let densities = density_scores(graph, communities); + let mut metrics = communities + .iter() + .map(|(community, members)| { + ( + *community, + CommunityQuality { + community: *community, + member_count: members.len(), + isolate: false, + connected_component_count: 0, + internal_edge_count: 0, + internal_weight: 0.0, + boundary_edge_count: 0, + boundary_weight: 0.0, + volume: 0.0, + density: 1.0, + conductance: 0.0, + modularity_contribution: 0.0, + relationship_mix: BTreeMap::new(), + relation_strength_mix: BTreeMap::new(), + evidence_confidence_mix: BTreeMap::new(), + witness_node_ids: Vec::new(), + witness_edge_ids: Vec::new(), + omitted_witness_node_count: 0, + omitted_witness_edge_count: 0, + }, + ) + }) + .collect::>(); + + for (node, assignment) in assignments.iter().enumerate() { + if let Some(community) = assignment + && let Some(metric) = metrics.get_mut(community) + { + metric.volume += graph.degree_weighted(node); + } + } + for (left, right, weight) in graph.edges() { + let left_community = assignments[left]; + let right_community = assignments[right]; + if let Some(community) = left_community + && right_community == Some(community) + && let Some(metric) = metrics.get_mut(&community) + { + metric.internal_edge_count += 1; + metric.internal_weight += weight; + } else { + if let Some(community) = left_community + && let Some(metric) = metrics.get_mut(&community) + { + metric.boundary_edge_count += 1; + metric.boundary_weight += weight; + } + if right != left + && let Some(community) = right_community + && let Some(metric) = metrics.get_mut(&community) + { + metric.boundary_edge_count += 1; + metric.boundary_weight += weight; + } + } + } + let mut witness_edges = BTreeMap::>::new(); + if let Some(pair_evidence) = pair_evidence { + for pair in pair_evidence { + let left_community = assignments[pair.left]; + let right_community = assignments[pair.right]; + let mut admitted = BTreeSet::new(); + if let Some(community) = left_community { + admitted.insert(community); + } + if let Some(community) = right_community { + admitted.insert(community); + } + for community in admitted { + if let Some(metric) = metrics.get_mut(&community) { + merge_counts(&mut metric.relationship_mix, &pair.relationship_counts); + merge_counts(&mut metric.relation_strength_mix, &pair.strength_counts); + merge_counts(&mut metric.evidence_confidence_mix, &pair.confidence_counts); + if left_community != right_community { + witness_edges + .entry(community) + .or_default() + .extend(pair.edge_ids.iter().cloned()); + } + } + } + } + } + + for (community, members) in communities { + let member_positions = members + .iter() + .filter_map(|member| positions.get(member).copied()) + .collect::>(); + let component_count = connected_component_count(graph, &member_positions); + if let Some(metric) = metrics.get_mut(community) { + metric.connected_component_count = component_count; + metric.isolate = metric.member_count == 1 && metric.volume == 0.0; + metric.density = densities.get(community).copied().unwrap_or(1.0); + let conductance_denominator = metric.volume.min(total_volume - metric.volume); + metric.conductance = if conductance_denominator > 0.0 { + metric.boundary_weight / conductance_denominator + } else { + 0.0 + }; + metric.modularity_contribution = if total_weight > 0.0 { + metric.internal_weight / total_weight + - resolution * (metric.volume / total_volume).powi(2) + } else { + 0.0 + }; + let fraction = metric.member_count as f64 / graph.len().max(1) as f64; + let degraded = component_count > 1 + || (metric.member_count == 1 && !metric.isolate) + || metric.conductance >= 0.5 + || fraction > 0.25; + if degraded { + let mut node_ids = members.clone(); + node_ids.sort(); + metric.omitted_witness_node_count = node_ids.len().saturating_sub(witness_limit); + node_ids.truncate(witness_limit); + metric.witness_node_ids = node_ids; + let mut edge_ids = witness_edges + .remove(community) + .unwrap_or_default() + .into_iter() + .collect::>(); + metric.omitted_witness_edge_count = edge_ids.len().saturating_sub(witness_limit); + edge_ids.truncate(witness_limit); + metric.witness_edge_ids = edge_ids; + } + } + } + + let assigned_node_count = assignments.iter().filter(|value| value.is_some()).count(); + let disconnected_community_count = metrics + .values() + .filter(|metric| !metric.isolate && metric.connected_component_count > 1) + .count(); + let non_isolate_singleton_count = metrics + .values() + .filter(|metric| metric.member_count == 1 && !metric.isolate) + .count(); + let modularity = metrics + .values() + .map(|metric| metric.modularity_contribution) + .sum(); + let retained_volume = metrics.values().map(|metric| metric.volume).sum::(); + let weighted_mean_conductance = if retained_volume > 0.0 { + metrics + .values() + .map(|metric| metric.conductance * metric.volume) + .sum::() + / retained_volume + } else { + 0.0 + }; + let worst_conductance = metrics + .values() + .map(|metric| metric.conductance) + .fold(0.0, f64::max); + let largest_community_fraction = if graph.is_empty() { + 0.0 + } else { + metrics + .values() + .map(|metric| metric.member_count) + .max() + .unwrap_or_default() as f64 + / graph.len() as f64 + }; + let omitted_witness_node_count = metrics + .values() + .map(|metric| metric.omitted_witness_node_count) + .sum(); + let omitted_witness_edge_count = metrics + .values() + .map(|metric| metric.omitted_witness_edge_count) + .sum(); + + Ok(PartitionQuality { + assigned_node_count, + omitted_node_count: graph.len().saturating_sub(assigned_node_count), + community_count: metrics.len(), + non_isolate_singleton_count, + disconnected_community_count, + modularity, + weighted_mean_conductance, + worst_conductance, + largest_community_fraction, + resolution, + algorithm: identity.algorithm.to_owned(), + topology: identity.topology.to_owned(), + quality: identity.quality.to_owned(), + selector: identity.selector.to_owned(), + seed: identity.seed, + limits: identity.limits.to_owned(), + quality_visit_count, + quality_visit_limit: max_quality_visits, + witness_limit, + omitted_witness_node_count, + omitted_witness_edge_count, + candidate_summaries: Vec::new(), + candidate_agreement: Vec::new(), + selection_reason: "fixed resolution".to_owned(), + topology_evidence, + communities: metrics, + }) +} + +fn merge_counts(target: &mut BTreeMap, source: &BTreeMap) { + for (key, count) in source { + *target.entry(key.clone()).or_default() += count; + } +} + +/// Adjusted Rand agreement for two complete partitions of the same node set. +#[must_use] +pub fn adjusted_rand_index(left: &Communities, right: &Communities) -> f64 { + let left_assignments = assignments_by_id(left); + let right_assignments = assignments_by_id(right); + if left_assignments.keys().ne(right_assignments.keys()) { + return 0.0; + } + let mut contingency = BTreeMap::<(usize, usize), usize>::new(); + let mut left_counts = BTreeMap::::new(); + let mut right_counts = BTreeMap::::new(); + for (node, left_community) in &left_assignments { + let Some(right_community) = right_assignments.get(node) else { + return 0.0; + }; + *contingency + .entry((*left_community, *right_community)) + .or_default() += 1; + *left_counts.entry(*left_community).or_default() += 1; + *right_counts.entry(*right_community).or_default() += 1; + } + let pairs = combination_two(left_assignments.len()); + if pairs == 0.0 { + return 1.0; + } + let index = contingency + .values() + .map(|count| combination_two(*count)) + .sum::(); + let left_index = left_counts + .values() + .map(|count| combination_two(*count)) + .sum::(); + let right_index = right_counts + .values() + .map(|count| combination_two(*count)) + .sum::(); + let expected = left_index * right_index / pairs; + let maximum = 0.5 * (left_index + right_index); + if (maximum - expected).abs() <= f64::EPSILON { + return if left_assignments == right_assignments { + 1.0 + } else { + 0.0 + }; + } + (index - expected) / (maximum - expected) +} + +/// Adjusted mutual information using the arithmetic-mean entropy normalizer. +/// Returns zero when the partitions do not cover the same node IDs. +pub fn adjusted_mutual_information( + left: &Communities, + right: &Communities, +) -> Result { + let left_assignments = assignments_by_id(left); + let right_assignments = assignments_by_id(right); + if left_assignments.keys().ne(right_assignments.keys()) { + return Ok(0.0); + } + let node_count = left_assignments.len(); + if node_count <= 1 { + return Ok(1.0); + } + let mut contingency = BTreeMap::<(usize, usize), usize>::new(); + let mut left_counts = BTreeMap::::new(); + let mut right_counts = BTreeMap::::new(); + for (node, left_community) in &left_assignments { + let Some(right_community) = right_assignments.get(node) else { + return Ok(0.0); + }; + *contingency + .entry((*left_community, *right_community)) + .or_default() += 1; + *left_counts.entry(*left_community).or_default() += 1; + *right_counts.entry(*right_community).or_default() += 1; + } + let total = node_count as f64; + let mutual_information = contingency + .iter() + .filter(|(_, count)| **count > 0) + .map(|((left_community, right_community), count)| { + let count = *count as f64; + let left_count = left_counts[left_community] as f64; + let right_count = right_counts[right_community] as f64; + count / total * (total * count / (left_count * right_count)).ln() + }) + .sum::(); + let expected = expected_mutual_information( + node_count, + left_counts.values().copied(), + right_counts.values().copied(), + DEFAULT_MAX_QUALITY_VISITS, + )?; + let left_entropy = entropy(total, left_counts.values().copied()); + let right_entropy = entropy(total, right_counts.values().copied()); + let normalizer = 0.5 * (left_entropy + right_entropy); + if (normalizer - expected).abs() <= 1e-12 { + return Ok(if left_assignments == right_assignments { + 1.0 + } else { + 0.0 + }); + } + Ok((mutual_information - expected) / (normalizer - expected)) +} + +fn entropy(total: f64, counts: impl Iterator) -> f64 { + counts + .filter(|count| *count > 0) + .map(|count| { + let probability = count as f64 / total; + -probability * probability.ln() + }) + .sum() +} + +fn expected_mutual_information( + node_count: usize, + left_counts: impl Iterator + Clone, + right_counts: impl Iterator + Clone, + max_visits: usize, +) -> Result { + let total = node_count as f64; + let log_factorials = (0..=node_count) + .scan(0.0, |sum, value| { + if value > 1 { + *sum += (value as f64).ln(); + } + Some(*sum) + }) + .collect::>(); + let log_choose = |whole: usize, selected: usize| { + log_factorials[whole] + - log_factorials[selected] + - log_factorials[whole.saturating_sub(selected)] + }; + let mut expected = 0.0; + let mut visits = 0usize; + for left_count in left_counts { + for right_count in right_counts.clone() { + let lower = left_count + .saturating_add(right_count) + .saturating_sub(node_count) + .max(1); + let upper = left_count.min(right_count); + for overlap in lower..=upper { + visits = visits.saturating_add(1); + if visits > max_visits { + return Err(CommunityQualityError::QualityLimitExceeded { + stage: "adjusted_mutual_information", + required: visits, + limit: max_visits, + processed: visits.saturating_sub(1), + }); + } + let probability = (log_choose(left_count, overlap) + + log_choose(node_count - left_count, right_count - overlap) + - log_choose(node_count, right_count)) + .exp(); + let overlap = overlap as f64; + expected += probability * overlap / total + * (total * overlap / (left_count as f64 * right_count as f64)).ln(); + } + } + } + Ok(expected) +} + +fn assignments_by_id(communities: &Communities) -> BTreeMap<&str, usize> { + communities + .iter() + .flat_map(|(community, members)| { + members + .iter() + .map(move |member| (member.as_str(), *community)) + }) + .collect() +} + +fn combination_two(count: usize) -> f64 { + count.saturating_mul(count.saturating_sub(1)) as f64 / 2.0 +} + +fn connected_component_count(graph: &WeightedGraph, members: &BTreeSet) -> usize { + let mut remaining = members.clone(); + let mut components = 0usize; + while let Some(start) = remaining.pop_first() { + components += 1; + let mut queue = VecDeque::from([start]); + while let Some(node) = queue.pop_front() { + for (neighbor, _) in graph.neighbors(node) { + if remaining.remove(neighbor) { + queue.push_back(*neighbor); + } + } + } + } + components +} + +#[cfg(test)] +mod tests { + use super::*; + use compass_model::{EdgeRecord, GraphDocument, NodeRecord}; + use serde_json::json; + + fn node(id: &str) -> NodeRecord { + NodeRecord { + id: id.to_owned(), + attributes: serde_json::Map::from_iter([("label".to_owned(), json!(id))]), + } + } + + fn edge(source: &str, target: &str, weight: f64) -> EdgeRecord { + EdgeRecord { + source: source.to_owned(), + target: target.to_owned(), + attributes: serde_json::Map::from_iter([("weight".to_owned(), json!(weight))]), + } + } + + #[test] + fn evaluates_reference_partition() -> Result<(), CommunityQualityError> { + let document = GraphDocument { + directed: true, + multigraph: true, + graph: serde_json::Map::new(), + nodes: ["a", "b", "c", "d"].into_iter().map(node).collect(), + links: vec![ + edge("a", "b", 2.0), + edge("b", "c", 1.0), + edge("c", "d", 2.0), + ], + extras: BTreeMap::new(), + }; + let communities = BTreeMap::from([ + (0, vec!["a".to_owned(), "b".to_owned()]), + (1, vec!["c".to_owned(), "d".to_owned()]), + ]); + + let quality = evaluate_partition_quality(&document, &communities, 1.0)?; + + assert_eq!(quality.assigned_node_count, 4); + assert_eq!(quality.omitted_node_count, 0); + assert_eq!(quality.disconnected_community_count, 0); + assert!((quality.modularity - 0.3).abs() < 1e-12); + assert!((quality.weighted_mean_conductance - 0.2).abs() < 1e-12); + assert_eq!(quality.communities[&0].internal_edge_count, 1); + assert_eq!(quality.communities[&0].internal_weight, 2.0); + assert_eq!(quality.communities[&0].boundary_edge_count, 1); + assert_eq!(quality.communities[&0].boundary_weight, 1.0); + assert_eq!(quality.communities[&0].density, 1.0); + assert_eq!(quality.communities[&0].connected_component_count, 1); + Ok(()) + } + + #[test] + fn reports_disconnected_and_omitted_nodes() -> Result<(), CommunityQualityError> { + let document = GraphDocument { + directed: false, + multigraph: false, + graph: serde_json::Map::new(), + nodes: ["a", "b", "c"].into_iter().map(node).collect(), + links: vec![edge("a", "c", 1.0)], + extras: BTreeMap::new(), + }; + let communities = BTreeMap::from([(0, vec!["a".to_owned(), "b".to_owned()])]); + + let quality = evaluate_partition_quality(&document, &communities, 1.0)?; + + assert_eq!(quality.assigned_node_count, 2); + assert_eq!(quality.omitted_node_count, 1); + assert_eq!(quality.disconnected_community_count, 1); + assert_eq!(quality.communities[&0].connected_component_count, 2); + assert_eq!(quality.communities[&0].conductance, 1.0); + Ok(()) + } + + #[test] + fn rejects_ambiguous_partition_membership() { + let document = GraphDocument { + directed: false, + multigraph: false, + graph: serde_json::Map::new(), + nodes: vec![node("a")], + links: Vec::new(), + extras: BTreeMap::new(), + }; + let communities = BTreeMap::from([(2, vec!["a".to_owned()]), (3, vec!["a".to_owned()])]); + + assert_eq!( + evaluate_partition_quality(&document, &communities, 1.0), + Err(CommunityQualityError::DuplicateMember { + node: "a".to_owned(), + first: 2, + second: 3, + }) + ); + } + + #[test] + fn adjusted_partition_metrics_match_identity_and_disagreement() + -> Result<(), CommunityQualityError> { + let planted = BTreeMap::from([ + (0, vec!["a".to_owned(), "b".to_owned()]), + (1, vec!["c".to_owned(), "d".to_owned()]), + ]); + let crossed = BTreeMap::from([ + (0, vec!["a".to_owned(), "c".to_owned()]), + (1, vec!["b".to_owned(), "d".to_owned()]), + ]); + assert!((adjusted_rand_index(&planted, &planted) - 1.0).abs() < 1e-12); + assert!((adjusted_mutual_information(&planted, &planted)? - 1.0).abs() < 1e-12); + assert!(adjusted_rand_index(&planted, &crossed) < 0.0); + assert!(adjusted_mutual_information(&planted, &crossed)? < 0.0); + Ok(()) + } +} diff --git a/crates/compass-graph/src/community/topology.rs b/crates/compass-graph/src/community/topology.rs new file mode 100644 index 000000000..7793dd6b6 --- /dev/null +++ b/crates/compass-graph/src/community/topology.rs @@ -0,0 +1,573 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use compass_model::code_graph::{EdgeKind, GraphDocument}; +use compass_model::provenance::{EvidenceConfidence, effective_confidence}; +use serde::{Deserialize, Serialize}; +use thiserror::Error; + +use crate::cluster::WeightedGraph; + +pub(crate) const MAX_OCCURRENCES_PER_PAIR_KIND: usize = 4; +pub(crate) const MAX_PAIR_WEIGHT: f64 = 64.0; + +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +enum Strength { + Weak, + Medium, + Strong, +} + +impl Strength { + const fn weight(self) -> f64 { + match self { + Self::Weak => 1.0, + Self::Medium => 2.0, + Self::Strong => 4.0, + } + } + + const fn name(self) -> &'static str { + match self { + Self::Weak => "weak", + Self::Medium => "medium", + Self::Strong => "strong", + } + } +} + +const fn relationship_strength(kind: EdgeKind) -> Strength { + match kind { + EdgeKind::Calls + | EdgeKind::RoutesTo + | EdgeKind::Reads + | EdgeKind::Writes + | EdgeKind::Handles + | EdgeKind::Publishes + | EdgeKind::Subscribes + | EdgeKind::Produces + | EdgeKind::Consumes + | EdgeKind::Schedules + | EdgeKind::Triggers + | EdgeKind::Renders => Strength::Strong, + EdgeKind::Imports + | EdgeKind::DependsOn + | EdgeKind::Instantiates + | EdgeKind::Registers + | EdgeKind::Extends + | EdgeKind::Implements + | EdgeKind::MixesIn + | EdgeKind::Overrides + | EdgeKind::Decorates + | EdgeKind::TypeOf + | EdgeKind::Returns => Strength::Medium, + EdgeKind::Contains + | EdgeKind::Embeds + | EdgeKind::Exports + | EdgeKind::References + | EdgeKind::Aliases + | EdgeKind::Tests + | EdgeKind::Documents + | EdgeKind::MapsTo => Strength::Weak, + } +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase", deny_unknown_fields)] +pub struct TopologyEvidence { + pub input_edge_count: usize, + pub input_weight_sum: f64, + pub retained_occurrence_count: usize, + pub retained_total_weight: f64, + pub projected_pair_count: usize, + pub omitted_ambiguous_count: usize, + pub omitted_capped_count: usize, + pub omitted_duplicate_occurrence_count: usize, + pub omitted_pair_weight_cap_count: usize, + pub forward_occurrence_count: usize, + pub reverse_occurrence_count: usize, + pub relationship_counts: BTreeMap, + pub strength_counts: BTreeMap, + pub confidence_counts: BTreeMap, + pub retained_relationship_counts: BTreeMap, + pub retained_strength_counts: BTreeMap, + pub retained_confidence_counts: BTreeMap, +} + +#[derive(Clone, Copy, Debug, PartialEq)] +pub(crate) struct TopologyLimits { + pub max_nodes: usize, + pub max_edges: usize, + pub max_projected_pairs: usize, + pub max_total_weight: f64, +} + +#[derive(Clone, Debug, Error, PartialEq)] +pub enum CommunityTopologyError { + #[error("typed community topology contains duplicate node id `{node}`")] + DuplicateNode { node: String }, + #[error("typed community topology edge `{edge}` references missing endpoint `{endpoint}`")] + DanglingEndpoint { edge: String, endpoint: String }, + #[error("typed community topology edge `{edge}` has invalid weight {weight}")] + InvalidEdgeWeight { edge: String, weight: f64 }, + #[error( + "community topology {stage} requires {required} items, exceeds limit {limit} after {processed}" + )] + LimitExceeded { + stage: &'static str, + required: usize, + limit: usize, + processed: usize, + }, + #[error( + "community topology total_weight requires {required}, exceeds limit {limit} after {processed} projected pairs" + )] + TotalWeightLimitExceeded { + required: f64, + limit: f64, + processed: usize, + }, +} + +pub(crate) struct CommunityTopology { + pub graph: WeightedGraph, + pub evidence: TopologyEvidence, + pub pair_evidence: Vec, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct ProjectedPairEvidence { + pub left: usize, + pub right: usize, + pub edge_ids: Vec, + pub relationship_counts: BTreeMap, + pub strength_counts: BTreeMap, + pub confidence_counts: BTreeMap, +} + +#[derive(Clone)] +struct Contribution { + edge_id: String, + occurrence_key: String, + forward: bool, + weight: f64, + strength: &'static str, + confidence: &'static str, +} + +pub(crate) fn from_typed_document( + document: &GraphDocument, + limits: TopologyLimits, +) -> Result { + if document.nodes.len() > limits.max_nodes { + return Err(CommunityTopologyError::LimitExceeded { + stage: "nodes", + required: document.nodes.len(), + limit: limits.max_nodes, + processed: 0, + }); + } + if document.links.len() > limits.max_edges { + return Err(CommunityTopologyError::LimitExceeded { + stage: "edges", + required: document.links.len(), + limit: limits.max_edges, + processed: 0, + }); + } + let mut ids = document + .nodes + .iter() + .map(|node| node.id.clone()) + .collect::>(); + ids.sort(); + for duplicate in ids.windows(2) { + if duplicate[0] == duplicate[1] { + return Err(CommunityTopologyError::DuplicateNode { + node: duplicate[0].clone(), + }); + } + } + let positions = ids + .iter() + .enumerate() + .map(|(position, id)| (id.as_str(), position)) + .collect::>(); + let mut grouped = BTreeMap::<(usize, usize, String), Vec>::new(); + let mut projected_pairs = BTreeSet::<(usize, usize)>::new(); + let mut evidence = TopologyEvidence { + input_edge_count: document.links.len(), + ..TopologyEvidence::default() + }; + for (processed, edge) in document.links.iter().enumerate() { + let Some(&source) = positions.get(edge.source.as_str()) else { + return Err(CommunityTopologyError::DanglingEndpoint { + edge: edge.id.clone(), + endpoint: edge.source.clone(), + }); + }; + let Some(&target) = positions.get(edge.target.as_str()) else { + return Err(CommunityTopologyError::DanglingEndpoint { + edge: edge.id.clone(), + endpoint: edge.target.clone(), + }); + }; + let input_weight = edge.weight.unwrap_or(1.0); + if !input_weight.is_finite() || input_weight <= 0.0 { + return Err(CommunityTopologyError::InvalidEdgeWeight { + edge: edge.id.clone(), + weight: input_weight, + }); + } + evidence.input_weight_sum += input_weight; + if !evidence.input_weight_sum.is_finite() { + return Err(CommunityTopologyError::InvalidEdgeWeight { + edge: edge.id.clone(), + weight: evidence.input_weight_sum, + }); + } + let confidence = + effective_confidence(&edge.evidence).unwrap_or(EvidenceConfidence::Inferred); + *evidence + .relationship_counts + .entry(edge.kind.as_str().to_owned()) + .or_default() += 1; + let strength = relationship_strength(edge.kind); + *evidence + .strength_counts + .entry(strength.name().to_owned()) + .or_default() += 1; + *evidence + .confidence_counts + .entry(confidence.as_str().to_owned()) + .or_default() += 1; + if confidence == EvidenceConfidence::Ambiguous { + evidence.omitted_ambiguous_count += 1; + continue; + } + let (left, right, forward) = if source <= target { + (source, target, true) + } else { + (target, source, false) + }; + let relation = edge.kind.as_str().to_owned(); + let pair_is_new = !projected_pairs.contains(&(left, right)); + if pair_is_new && projected_pairs.len() == limits.max_projected_pairs { + return Err(CommunityTopologyError::LimitExceeded { + stage: "projected_pairs", + required: projected_pairs.len().saturating_add(1), + limit: limits.max_projected_pairs, + processed, + }); + } + projected_pairs.insert((left, right)); + let confidence_factor = match confidence { + EvidenceConfidence::Exact => 1.0, + EvidenceConfidence::Inferred => 0.5, + EvidenceConfidence::Ambiguous => 0.0, + }; + grouped + .entry((left, right, relation)) + .or_default() + .push(Contribution { + edge_id: edge.id.clone(), + occurrence_key: occurrence_key(edge), + forward, + weight: strength.weight() * confidence_factor * input_weight, + strength: strength.name(), + confidence: confidence.as_str(), + }); + } + + let members = ids.iter().cloned().map(|id| BTreeSet::from([id])).collect(); + let mut graph = WeightedGraph::new(ids, members); + let mut pair_weights = BTreeMap::<(usize, usize), f64>::new(); + let mut pair_evidence = BTreeMap::<(usize, usize), ProjectedPairEvidence>::new(); + for ((left, right, relation), contributions) in &mut grouped { + let before_deduplication = contributions.len(); + contributions.sort_by(|left, right| { + left.occurrence_key + .cmp(&right.occurrence_key) + .then_with(|| left.forward.cmp(&right.forward)) + .then_with(|| left.edge_id.cmp(&right.edge_id)) + }); + contributions.dedup_by(|left, right| { + left.occurrence_key == right.occurrence_key && left.forward == right.forward + }); + evidence.omitted_duplicate_occurrence_count += + before_deduplication.saturating_sub(contributions.len()); + evidence.omitted_capped_count += contributions + .len() + .saturating_sub(MAX_OCCURRENCES_PER_PAIR_KIND); + for contribution in contributions.iter().take(MAX_OCCURRENCES_PER_PAIR_KIND) { + evidence.retained_occurrence_count += 1; + if contribution.forward { + evidence.forward_occurrence_count += 1; + } else { + evidence.reverse_occurrence_count += 1; + } + *evidence + .retained_relationship_counts + .entry(relation.clone()) + .or_default() += 1; + *evidence + .retained_strength_counts + .entry(contribution.strength.to_owned()) + .or_default() += 1; + *evidence + .retained_confidence_counts + .entry(contribution.confidence.to_owned()) + .or_default() += 1; + let pair = + pair_evidence + .entry((*left, *right)) + .or_insert_with(|| ProjectedPairEvidence { + left: *left, + right: *right, + edge_ids: Vec::new(), + relationship_counts: BTreeMap::new(), + strength_counts: BTreeMap::new(), + confidence_counts: BTreeMap::new(), + }); + pair.edge_ids.push(contribution.edge_id.clone()); + *pair + .relationship_counts + .entry(relation.clone()) + .or_default() += 1; + *pair + .strength_counts + .entry(contribution.strength.to_owned()) + .or_default() += 1; + *pair + .confidence_counts + .entry(contribution.confidence.to_owned()) + .or_default() += 1; + *pair_weights.entry((*left, *right)).or_default() += contribution.weight; + } + } + evidence.projected_pair_count = pair_weights.len(); + for (processed, ((left, right), weight)) in pair_weights.into_iter().enumerate() { + if weight > MAX_PAIR_WEIGHT { + evidence.omitted_pair_weight_cap_count += 1; + } + let retained_weight = weight.min(MAX_PAIR_WEIGHT); + let required = evidence.retained_total_weight + retained_weight; + if !required.is_finite() || required > limits.max_total_weight { + return Err(CommunityTopologyError::TotalWeightLimitExceeded { + required, + limit: limits.max_total_weight, + processed, + }); + } + evidence.retained_total_weight = required; + graph.add_edge(left, right, retained_weight); + } + for pair in pair_evidence.values_mut() { + pair.edge_ids.sort(); + pair.edge_ids.dedup(); + } + Ok(CommunityTopology { + graph, + evidence, + pair_evidence: pair_evidence.into_values().collect(), + }) +} + +fn occurrence_key(edge: &compass_model::code_graph::EdgeRecord) -> String { + let mut anchors = edge + .relationship_site + .iter() + .chain(edge.evidence.iter().flat_map(|item| item.anchors.iter())) + .map(|anchor| { + format!( + "{}:{}:{}:{}", + anchor.file, anchor.start_byte, anchor.end_byte, anchor.start_line + ) + }) + .collect::>(); + anchors.sort(); + anchors.dedup(); + if anchors.is_empty() { + edge.id.clone() + } else { + anchors.join("|") + } +} + +#[cfg(test)] +mod tests { + use super::*; + use compass_model::code_graph::{BuildMetadata, EdgeRecord, NodeKind, NodeRecord}; + use compass_model::provenance::{EvidenceConfidence, EvidenceOrigin, Provenance}; + + fn node(id: &str) -> NodeRecord { + NodeRecord { + id: id.to_owned(), + kind: NodeKind::Function, + roles: Vec::new(), + name: id.to_owned(), + qualified_name: id.to_owned(), + language: None, + framework: None, + source: None, + details: None, + evidence: Vec::new(), + coverage: Vec::new(), + diagnostics: Vec::new(), + community: None, + } + } + + fn edge( + id: &str, + source: &str, + target: &str, + kind: EdgeKind, + confidence: EvidenceConfidence, + ) -> EdgeRecord { + EdgeRecord { + id: id.to_owned(), + key: id.to_owned(), + source: source.to_owned(), + target: target.to_owned(), + kind, + occurrence_rule: None, + relationship_site: None, + details: None, + evidence: vec![Provenance { + origin: EvidenceOrigin::Ast, + extractor: "test".to_owned(), + confidence, + rule: None, + anchors: Vec::new(), + wiring_site: None, + score: None, + candidates: Vec::new(), + }], + weight: Some(1.0), + context: None, + deferred: false, + diagnostics: Vec::new(), + } + } + + fn document() -> GraphDocument { + let mut document = GraphDocument::empty_v1(BuildMetadata { + builder_version: "test".to_owned(), + schema_fingerprint: "test".to_owned(), + source_tree_digest: "test".to_owned(), + configuration_digest: "test".to_owned(), + generation_id: "test".to_owned(), + source_commit: None, + }); + document.nodes = ["a", "b"].into_iter().map(node).collect(); + document + } + + #[test] + fn typed_projection_weights_direction_confidence_and_relation() + -> Result<(), CommunityTopologyError> { + let mut document = document(); + document.links = vec![ + edge("1", "a", "b", EdgeKind::Calls, EvidenceConfidence::Exact), + edge( + "2", + "b", + "a", + EdgeKind::Imports, + EvidenceConfidence::Inferred, + ), + edge( + "3", + "a", + "b", + EdgeKind::Calls, + EvidenceConfidence::Ambiguous, + ), + ]; + let topology = from_typed_document( + &document, + TopologyLimits { + max_nodes: 10, + max_edges: 10, + max_projected_pairs: 10, + max_total_weight: 1_000.0, + }, + )?; + assert_eq!(topology.graph.edge_count(), 1); + assert_eq!(topology.graph.total_weight(), 5.0); + assert_eq!(topology.evidence.forward_occurrence_count, 1); + assert_eq!(topology.evidence.reverse_occurrence_count, 1); + assert_eq!(topology.evidence.omitted_ambiguous_count, 1); + assert_eq!(topology.evidence.retained_relationship_counts["calls"], 1); + assert_eq!(topology.evidence.retained_confidence_counts["exact"], 1); + assert_eq!(topology.evidence.retained_total_weight, 5.0); + Ok(()) + } + + #[test] + fn projection_caps_occurrences_and_is_order_invariant() -> Result<(), CommunityTopologyError> { + let mut document = document(); + document.links = (0..6) + .map(|index| { + edge( + &format!("edge-{index}"), + "a", + "b", + EdgeKind::Calls, + EvidenceConfidence::Exact, + ) + }) + .collect(); + document.links.push(document.links[0].clone()); + let limits = TopologyLimits { + max_nodes: 10, + max_edges: 10, + max_projected_pairs: 10, + max_total_weight: 1_000.0, + }; + let expected = from_typed_document(&document, limits)?; + document.nodes.reverse(); + document.links.reverse(); + let actual = from_typed_document(&document, limits)?; + + assert_eq!(expected.graph.total_weight(), 16.0); + assert_eq!(expected.evidence.omitted_duplicate_occurrence_count, 1); + assert_eq!(expected.evidence.omitted_capped_count, 2); + assert_eq!(actual.evidence, expected.evidence); + assert_eq!(actual.pair_evidence, expected.pair_evidence); + assert_eq!(actual.graph.ids, expected.graph.ids); + assert_eq!( + actual.graph.edges().collect::>(), + expected.graph.edges().collect::>() + ); + Ok(()) + } + + #[test] + fn projection_fails_closed_at_the_total_weight_limit() { + let mut document = document(); + document.links = vec![edge( + "edge", + "a", + "b", + EdgeKind::Calls, + EvidenceConfidence::Exact, + )]; + assert!(matches!( + from_typed_document( + &document, + TopologyLimits { + max_nodes: 10, + max_edges: 10, + max_projected_pairs: 10, + max_total_weight: 1.0, + } + ), + Err(CommunityTopologyError::TotalWeightLimitExceeded { + required: 4.0, + limit: 1.0, + processed: 0, + }) + )); + } +} diff --git a/crates/compass-graph/src/lib.rs b/crates/compass-graph/src/lib.rs index aac7169aa..4b15ea645 100644 --- a/crates/compass-graph/src/lib.rs +++ b/crates/compass-graph/src/lib.rs @@ -2,6 +2,7 @@ mod analyze; mod cluster; +mod community; mod dedup; mod inference; mod quarantine; @@ -20,6 +21,19 @@ pub use cluster::{ cluster_incremental, cohesion_score, community_member_signatures, label_communities_by_hub, remap_communities_to_previous, score_communities, }; +pub use community::{ + COMMUNITY_QUALITY_SCHEMA, COMPATIBILITY_CLUSTER_ALGORITHM, COMPATIBILITY_CLUSTER_LIMITS, + COMPATIBILITY_CLUSTER_QUALITY, COMPATIBILITY_CLUSTER_SEED, COMPATIBILITY_CLUSTER_SEED_TEXT, + COMPATIBILITY_CLUSTER_SELECTOR, COMPATIBILITY_CLUSTER_TOPOLOGY, CandidateAgreement, + CommunityCandidateSummary, CommunityDetectorError, CommunityError, CommunityExecution, + CommunityIdentity, CommunityLimits, CommunityProfile, CommunityQuality, + CommunityQualityArtifact, CommunityQualityArtifactError, CommunityQualityError, + CommunityRequest, CommunityResult, CommunityTopologyError, FallbackReason, PartitionQuality, + PreviousCommunities, QUALITY_CLUSTER_ALGORITHM, QUALITY_CLUSTER_LIMITS, + QUALITY_CLUSTER_QUALITY, QUALITY_CLUSTER_SELECTOR, QUALITY_CLUSTER_TOPOLOGY, ResolutionPolicy, + TopologyEvidence, adjusted_mutual_information, adjusted_rand_index, build_communities, + evaluate_partition_quality, +}; pub use compass_languages::{RawEdgeRecord, RawNodeRecord}; use dedup::deduplicate_owned; pub use dedup::{ diff --git a/crates/compass-history/Cargo.toml b/crates/compass-history/Cargo.toml index 4ca8cc91f..56ca86812 100644 --- a/crates/compass-history/Cargo.toml +++ b/crates/compass-history/Cargo.toml @@ -22,6 +22,7 @@ sha2.workspace = true thiserror.workspace = true tempfile.workspace = true compass-files = { path = "../compass-files", version = "0.3.24" } +compass-graph = { path = "../compass-graph", version = "0.3.24" } compass-analysis = { path = "../compass-analysis", version = "0.3.24" } compass-ir = { path = "../compass-ir", version = "0.3.24" } compass-model = { path = "../compass-model", version = "0.3.24" } diff --git a/crates/compass-history/src/artifacts.rs b/crates/compass-history/src/artifacts.rs index 9d8077c91..d1f5f4653 100644 --- a/crates/compass-history/src/artifacts.rs +++ b/crates/compass-history/src/artifacts.rs @@ -73,6 +73,7 @@ const EDGE_COMPATIBILITY_FIELDS: [&str; 6] = [ const TRUSTED_GRAPH_CONTENT: &str = "history/graph.v1.json"; const PROGRAM_SOURCE_DIGEST_CONTENT: &str = "history/program.source-digest"; const SOURCE_INVENTORY_CONTENT: &str = "source-inventory.json"; +const COMMUNITY_QUALITY_CONTENT: &str = "community-quality.json"; /// All authoritative inputs needed to reconstruct a complete Compass output. #[derive(Clone, Debug, PartialEq)] @@ -225,7 +226,14 @@ impl GraphArtifacts { analysis: Option, manifest: Option, ) -> Result { - let trusted_bytes = canonical_json_bytes(&serde_json::to_value(&trusted)?)?; + let mut trusted_bytes = Vec::new(); + compass_graph::write_canonical_graph_json(&trusted, &mut trusted_bytes).map_err( + |error| { + HistoryError::InvalidArtifacts(format!( + "trusted graph canonical encoding failed: {error}" + )) + }, + )?; let graph = serde_json::to_value(&trusted.graph)? .as_object() .cloned() @@ -282,12 +290,18 @@ impl GraphArtifacts { )); } let value: Value = serde_json::from_slice(bytes)?; - if canonical_json_bytes(&value)? != *bytes { + let document: TrustedGraphDocument = serde_json::from_value(value.clone())?; + let mut streamed = Vec::new(); + compass_graph::write_canonical_graph_json(&document, &mut streamed).map_err(|error| { + HistoryError::InvalidArtifacts(format!( + "trusted graph canonical encoding failed: {error}" + )) + })?; + if canonical_json_bytes(&value)? != *bytes && streamed != *bytes { return Err(HistoryError::InvalidArtifacts( "trusted graph artifact is not canonical JSON".to_owned(), )); } - let document: TrustedGraphDocument = serde_json::from_value(value)?; validate_code_graph(&document).map_err(|error| { HistoryError::InvalidArtifacts(format!("trusted graph validation failed: {error}")) })?; @@ -341,7 +355,7 @@ impl GraphArtifacts { profile_artifact_load("authoritative sidecars", sidecars_started); let graph_path = output_dir.join("graph.json"); let program_path = output_dir.join("program.json"); - let ((document, trusted_graph), program) = { + let ((document, trusted_graph, graph_generation), program) = { let (graph, program) = rayon::join( || { let started = Instant::now(); @@ -360,6 +374,13 @@ impl GraphArtifacts { // to completion concurrently. (graph?, program?) }; + if let Some(bytes) = read_optional_community_quality( + &output_dir.join(COMMUNITY_QUALITY_CONTENT), + &graph_generation, + &trusted_graph, + )? { + authoritative_sidecars.insert(COMMUNITY_QUALITY_CONTENT.to_owned(), bytes); + } authoritative_sidecars.insert(TRUSTED_GRAPH_CONTENT.to_owned(), trusted_graph); if !validate_program && let Some(digest) = program.source_digest { authoritative_sidecars @@ -397,6 +418,7 @@ impl GraphArtifacts { ) -> Result { completion.validate()?; validate_sidecar_paths(&self.authoritative_sidecars)?; + validate_embedded_community_quality(&self.authoritative_sidecars)?; let trusted_graph = self .authoritative_sidecars .contains_key(TRUSTED_GRAPH_CONTENT); @@ -1009,10 +1031,30 @@ impl GraphArtifacts { nodes, links, }; - sidecars.insert( - TRUSTED_GRAPH_CONTENT.to_owned(), - canonical_json_bytes(&serde_json::to_value(trusted)?)?, - ); + let value = serde_json::to_value(&trusted)?; + let sorted_bytes = canonical_json_bytes(&value)?; + let mut streamed_bytes = Vec::new(); + compass_graph::write_canonical_graph_json(&trusted, &mut streamed_bytes).map_err( + |error| { + HistoryError::InvalidArtifacts(format!( + "trusted graph canonical encoding failed: {error}" + )) + }, + )?; + let expected_digest = registry.as_ref().and_then(|registry| { + registry + .iter() + .find(|entry| entry.relative_path == "graph.json") + .and_then(|entry| entry.content_digest) + }); + let trusted_bytes = if expected_digest.is_some_and(|expected| { + <[u8; 32]>::from(Sha256::digest(&streamed_bytes)) == expected + }) { + streamed_bytes + } else { + sorted_bytes + }; + sidecars.insert(TRUSTED_GRAPH_CONTENT.to_owned(), trusted_bytes); } let restored = Self { document: GraphDocument { @@ -1095,9 +1137,27 @@ impl GraphArtifacts { } } -fn load_trusted_graph(path: &Path) -> Result<(GraphDocument, Vec), HistoryError> { - let trusted = TrustedGraphDocument::load_for_recluster(path)?; - let trusted_bytes = canonical_json_bytes(&serde_json::to_value(&trusted)?)?; +fn load_trusted_graph(path: &Path) -> Result<(GraphDocument, Vec, String), HistoryError> { + let (trusted, artifact_digest) = + TrustedGraphDocument::load_for_recluster_with_artifact_digest(path)?; + let generation = trusted.graph.build.generation_id.clone(); + let value = serde_json::to_value(&trusted)?; + let sorted_bytes = canonical_json_bytes(&value)?; + let mut streamed_bytes = Vec::new(); + compass_graph::write_canonical_graph_json(&trusted, &mut streamed_bytes).map_err(|error| { + HistoryError::InvalidArtifacts(format!("trusted graph canonical encoding failed: {error}")) + })?; + let streamed_digest = format!("{:x}", Sha256::digest(&streamed_bytes)); + let sorted_digest = format!("{:x}", Sha256::digest(&sorted_bytes)); + let trusted_bytes = if artifact_digest == streamed_digest { + streamed_bytes + } else if artifact_digest == sorted_digest { + sorted_bytes + } else { + // Normalize non-canonical inputs for old history behavior. A quality + // sidecar bound to such bytes will fail the graph-identity check. + sorted_bytes + }; let graph = serde_json::to_value(&trusted.graph)? .as_object() .cloned() @@ -1122,6 +1182,7 @@ fn load_trusted_graph(path: &Path) -> Result<(GraphDocument, Vec), HistoryEr extras: BTreeMap::new(), }, trusted_bytes, + generation, )) } @@ -1348,7 +1409,15 @@ fn artifact_registry_with_graph_bytes( if is_internal_artifact(path) { continue; } - let mut entry = authoritative_entry(path, "application/octet-stream", bytes); + let media_type = if path == COMMUNITY_QUALITY_CONTENT { + "application/json" + } else { + "application/octet-stream" + }; + let mut entry = authoritative_entry(path, media_type, bytes); + if path == COMMUNITY_QUALITY_CONTENT { + entry.schema_version = Some(1); + } if path != SOURCE_INVENTORY_CONTENT { entry.storage = Some(bytes.clone()); } @@ -1429,7 +1498,12 @@ fn completion_from_partition( fn is_builtin_artifact(path: &str) -> bool { matches!( path, - "graph.json" | "program.json" | "analysis.json" | "labels.json" | "manifest.json" + "graph.json" + | "program.json" + | "analysis.json" + | "labels.json" + | "manifest.json" + | COMMUNITY_QUALITY_CONTENT ) } @@ -2095,6 +2169,78 @@ fn read_optional_json(path: &Path) -> Result, HistoryError> { } } +fn read_optional_authoritative_json( + path: &Path, + expected_schema: &str, +) -> Result>, HistoryError> { + let metadata = match fs::metadata(path) { + Ok(metadata) => metadata, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(source) => return Err(crate::error::io_error(path, source)), + }; + if metadata.len() > crate::MAX_AUTHORITATIVE_BYTES { + return Err(HistoryError::InvalidArtifacts(format!( + "authoritative sidecar {} exceeds byte limit", + path.display() + ))); + } + let bytes = fs::read(path).map_err(|source| crate::error::io_error(path, source))?; + let value: Value = serde_json::from_slice(&bytes)?; + if value.get("schema").and_then(Value::as_str) != Some(expected_schema) { + return Err(HistoryError::InvalidArtifacts(format!( + "authoritative sidecar {} has an unsupported schema", + path.display() + ))); + } + Ok(Some(bytes)) +} + +fn read_optional_community_quality( + path: &Path, + graph_generation: &str, + graph_bytes: &[u8], +) -> Result>, HistoryError> { + let Some(bytes) = + read_optional_authoritative_json(path, compass_graph::COMMUNITY_QUALITY_SCHEMA)? + else { + return Ok(None); + }; + let artifact: compass_graph::CommunityQualityArtifact = serde_json::from_slice(&bytes)?; + let graph_digest = format!("sha256:{:x}", Sha256::digest(graph_bytes)); + artifact + .validate_for_graph(graph_generation, &graph_digest) + .map_err(|error| { + HistoryError::InvalidArtifacts(format!( + "authoritative sidecar {} is invalid: {error}", + path.display() + )) + })?; + Ok(Some(bytes)) +} + +fn validate_embedded_community_quality( + sidecars: &BTreeMap, +) -> Result<(), HistoryError> { + let Some(bytes) = sidecars.get(COMMUNITY_QUALITY_CONTENT) else { + return Ok(()); + }; + let Some(graph_bytes) = sidecars.get(TRUSTED_GRAPH_CONTENT) else { + return Err(HistoryError::InvalidArtifacts( + "community quality sidecar requires a trusted graph artifact".to_owned(), + )); + }; + let graph: TrustedGraphDocument = serde_json::from_slice(graph_bytes)?; + let artifact: compass_graph::CommunityQualityArtifact = serde_json::from_slice(bytes)?; + let graph_digest = format!("sha256:{:x}", Sha256::digest(graph_bytes)); + artifact + .validate_for_graph(&graph.graph.build.generation_id, &graph_digest) + .map_err(|error| { + HistoryError::InvalidArtifacts(format!( + "authoritative sidecar {COMMUNITY_QUALITY_CONTENT} is invalid: {error}" + )) + }) +} + fn read_optional_program( path: &Path, validate_canonical: bool, diff --git a/crates/compass-history/tests/roundtrip.rs b/crates/compass-history/tests/roundtrip.rs index 048ae5866..3d8c71f2a 100644 --- a/crates/compass-history/tests/roundtrip.rs +++ b/crates/compass-history/tests/roundtrip.rs @@ -1,4 +1,5 @@ use std::collections::BTreeMap; +use std::collections::BTreeSet; use compass_analysis::{AnalysisBundle, analyze}; use compass_history::{ @@ -190,6 +191,67 @@ fn trusted_graph_partition_does_not_duplicate_full_graph_as_metadata() Ok(()) } +#[test] +fn community_quality_sidecar_round_trips_and_remains_graph_bound() +-> Result<(), Box> { + let directory = tempfile::tempdir()?; + let graph = empty_trusted_graph(); + let graph_bytes = canonical_json_bytes(&graph)?; + std::fs::write(directory.path().join("graph.json"), &graph_bytes)?; + let typed: compass_model::code_graph::GraphDocument = serde_json::from_value(graph)?; + let changed_sources = BTreeSet::new(); + let limits = compass_graph::CommunityLimits::default(); + let result = compass_graph::build_communities( + &typed, + &compass_graph::CommunityRequest { + profile: compass_graph::CommunityProfile::QualityV1, + resolution: compass_graph::ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits, + }, + )?; + let graph_digest = format!("sha256:{:x}", Sha256::digest(&graph_bytes)); + let artifact = compass_graph::CommunityQualityArtifact::new( + typed.graph.build.generation_id.clone(), + graph_digest.clone(), + result.identity.clone(), + limits, + result.quality.clone(), + )?; + let artifact_bytes = serde_json::to_vec_pretty(&artifact)?; + std::fs::write( + directory.path().join("community-quality.json"), + &artifact_bytes, + )?; + + let loaded = GraphArtifacts::load(directory.path())?; + assert_eq!( + loaded.export_sidecars()["community-quality.json"], + artifact_bytes + ); + let partition = loaded.partition(&completion())?; + let restored = GraphArtifacts::reconstruct(&partition)?; + assert_eq!(restored.export_sidecars(), loaded.export_sidecars()); + + let wrong_graph = compass_graph::CommunityQualityArtifact::new( + "wrong-generation".to_owned(), + graph_digest, + result.identity, + limits, + result.quality, + )?; + std::fs::write( + directory.path().join("community-quality.json"), + serde_json::to_vec_pretty(&wrong_graph)?, + )?; + let error = GraphArtifacts::load(directory.path()).expect_err("binding mismatch must fail"); + assert!(error.to_string().contains("graph identity does not match")); + Ok(()) +} + #[test] fn source_inventory_uses_one_decomposed_canonical_record() -> Result<(), Box> { diff --git a/crates/compass-mcp/tests/code_query_tools.rs b/crates/compass-mcp/tests/code_query_tools.rs index 117a29a27..424dac065 100644 --- a/crates/compass-mcp/tests/code_query_tools.rs +++ b/crates/compass-mcp/tests/code_query_tools.rs @@ -355,6 +355,7 @@ fn cluster_only_output_remains_typed_and_serves_orientation_resources() -> Resul no_viz: true, no_label: true, resolution: 1.0, + resolution_explicit: false, exclude_hubs: None, min_community_size: 1, })?; diff --git a/crates/compass-output/src/lib.rs b/crates/compass-output/src/lib.rs index df89d33ba..99e74fa23 100644 --- a/crates/compass-output/src/lib.rs +++ b/crates/compass-output/src/lib.rs @@ -111,6 +111,10 @@ pub enum OutputError { InvalidOrientationModel { reason: &'static str }, #[error(transparent)] File(#[from] compass_files::FileError), + #[error(transparent)] + Community(#[from] compass_graph::CommunityError), + #[error(transparent)] + Graph(#[from] compass_model::GraphError), #[error("existing graph is non-empty but malformed: {0}")] MalformedGraph(std::path::PathBuf), #[error("refusing to shrink graph from {existing} nodes to {new}; use force to override")] diff --git a/crates/compass-output/src/viewer_model.rs b/crates/compass-output/src/viewer_model.rs index c6f3314f6..65bad3bdc 100644 --- a/crates/compass-output/src/viewer_model.rs +++ b/crates/compass-output/src/viewer_model.rs @@ -330,12 +330,25 @@ pub fn effective_graph_view_model( effective: &compass_agent_graph::EffectiveGraph, title: impl Into, options: &HtmlOptions<'_>, -) -> Result { +) -> Result { let document = effective.graph.to_legacy_document()?; // Agent assertions can change topology. Recompute the communities from the // exact Effective Graph and discard labels/counts derived from the base so // a viewer can never present a mixed-revision topology. - let communities = compass_graph::cluster(&document, compass_graph::ClusterOptions::default()); + let changed_sources = BTreeSet::new(); + let communities = compass_graph::build_communities( + &effective.graph, + &compass_graph::CommunityRequest { + profile: compass_graph::CommunityProfile::QualityV1, + resolution: compass_graph::ResolutionPolicy::Fixed(1.0), + exclude_hubs_percentile: None, + previous: None, + incremental: false, + changed_sources: &changed_sources, + limits: compass_graph::CommunityLimits::default(), + }, + )? + .communities; let effective_options = HtmlOptions { community_labels: None, member_counts: None, diff --git a/docs/README.md b/docs/README.md index a11ce7f50..a739ed6bf 100644 --- a/docs/README.md +++ b/docs/README.md @@ -56,6 +56,7 @@ to yours: | --- | --- | | [How Compass works](concepts/how-it-works.md) | How does a directory become a queryable graph? | | [Graph model](concepts/graph-model.md) | What do the entities and relationships mean? | +| [Community detection and quality](concepts/community-detection.md) | How are deterministic communities built, bounded, and evaluated? | | [Provenance](concepts/provenance.md) | How can I judge where an edge came from? | | [Agent Graph Overlays](concepts/agent-graph-overlays.md) | How can an agent add verified knowledge without rewriting the Base Graph? | | [CompassQL concepts](concepts/compassql.md) | When should I use an exact structural query? | @@ -111,6 +112,8 @@ They are not evidence that an uncompleted design has shipped. | [Evidence resolution framework execution plan](implementation/evidence-resolution-framework-phased-execution-plan.md) | Phased, commit-oriented implementation and verification plan | | [Architecture graph hardening technical design](implementation/architecture-graph-hardening-phased-technical-design.md) | Project-specific architecture projection, quality contract, and phased delivery plan | | [Architecture graph hardening qualification](implementation/architecture-graph-hardening-qualification.md) | Real-repository metrics, screenshots, acceptance gates, and residual diagnostics | +| [Community detection quality technical design](implementation/community-detection-quality-technical-design.md) | Versioned topology, shared quality evidence, native Leiden, bounded selection, and incremental rollout | +| [Community detection quality qualification](implementation/community-detection-quality-qualification.md) | Fixture results, performance decision, and remaining corpus evidence | | [Query recall and accuracy design](implementation/query-recall-accuracy/query-performance-accuracy-recall-phased-technical-design.md) | Phased query-quality architecture, evidence, and rollout boundaries | | [Query implementation plans](plans/README.md) | Ordered, independently executable query-quality work plans | | [Grounded Agent Graph Overlay design](implementation/grounded-agent-graph-overlay-technical-design.md) | Ownership, Grounding, CRUD, composition, security, and history contracts for agent-authored graph enhancement | diff --git a/docs/concepts/community-detection.md b/docs/concepts/community-detection.md new file mode 100644 index 000000000..807589985 --- /dev/null +++ b/docs/concepts/community-detection.md @@ -0,0 +1,93 @@ +# Community detection and quality + +Compass communities are deterministic navigation partitions over a projected +view of the Base Graph. They are useful hypotheses about subsystem boundaries, +not source truth: node and relationship records remain authoritative even when +community membership changes between engine profiles. + +## Production profile + +Clustered typed graphs use this complete, versioned profile: + +```text +algorithm seeded-leiden-modularity/v1 +topology typed-evidence-undirected/v1 +quality community-quality/v1 +selector fixed-resolution/v1 +seed 42 +limits community-limits/v1 +resolution 1 by default, or the positive finite --resolution value +``` + +The topology preserves all Base Graph records unchanged. It creates a separate +undirected clustering projection in which calls and runtime wiring are strong, +imports and type relationships are medium, and containment, references, tests, +and documentation links are weak. Exact evidence contributes its full bounded +weight, inferred evidence contributes half, and ambiguous evidence is recorded +as omitted rather than used to invent affinity. Parallel occurrences are +deduplicated and capped per node-pair and relationship kind; aggregate pair and +total weights are also bounded. + +Native deterministic Leiden local moving and connected refinement produce the +partition. Compass independently checks completeness and connectedness before +publication. It fails with a typed limit or partition error instead of +publishing a partial result. + +## Resolution selection + +`--resolution N` selects one fixed generalized-modularity resolution. Higher +values normally produce smaller communities; lower values normally produce +larger ones. Omitting the option currently uses the fixed value `1`. + +Compass also implements a deterministic three-candidate selector over +`0.75 × base`, `base`, and `4/3 × base`. It compares every candidate at one +common quality resolution, filters invalid or materially worse partitions, and +then applies size, conductance, fragmentation, and digest tie-breaks. Compact +fixtures show that it improves ring-of-cliques and articulation cases, but its +measured clustering time exceeds the production acceptance gate. It therefore +remains qualification-only until optimized; omission of `--resolution` does +not enable it. + +## Incremental updates + +An incremental update admits changed nodes and their complete prior +communities. Adjacent unchanged communities remain visible as frozen anchors, +so a local run does not lose external influence. Compass evaluates the merged +partition on the full topology. Removed nodes, excessive affected regions, +anchor merges, hub-policy changes, or quality regression cause a deterministic +full Leiden fallback. Historical materialization never depends on current-tree +assignments. + +## Quality evidence + +Clustered typed builds publish `community-quality.json` with schema +`compass.community-quality/1`. It is bound to the exact graph generation and +canonical `graph.json` digest and contains: + +- the complete community profile and exact work limits; +- the selected candidate and any rejected candidate summaries; +- modularity, conductance, size, singleton, and connectedness measurements; +- relationship, strength, and confidence mixes; +- bounded node and edge witnesses plus exact omission counts; and +- a digest over the complete result. + +Unknown schemas, unknown fields, profile mismatch, mutation, or graph mismatch +fail validation. Older graphs may legitimately omit this artifact; absence +means unavailable evidence, not a quality score of zero. Immutable history +stores the sidecar verbatim with its realization. + +Community numeric IDs are graph-local. Stable member signatures reduce +unnecessary churn across updates, but detector, topology, resolution, or source +changes can legitimately change membership and IDs. Integrations that need +semantic identity should retain member IDs and the complete profile. + +## Related pages + +- [Graph model](graph-model.md) +- [Output reference](../reference/outputs.md) +- [Command reference](../reference/commands.md) +- [Technical design](../implementation/community-detection-quality-technical-design.md) +- [Qualification](../implementation/community-detection-quality-qualification.md) + +**Next step:** inspect `community-quality.json` beside a clustered graph before +treating a community boundary as an architectural conclusion. diff --git a/docs/implementation/community-detection-quality-qualification.json b/docs/implementation/community-detection-quality-qualification.json new file mode 100644 index 000000000..c1f14ceb4 --- /dev/null +++ b/docs/implementation/community-detection-quality-qualification.json @@ -0,0 +1,704 @@ +{ + "schema": "compass.community-quality-qualification/1", + "fixtureCount": 15, + "acceptance": { + "zeroDisconnectedQualityCommunities": true, + "deterministicRepeatAndPermutation": true, + "exactRecoveryForRequiredFixtures": true, + "noAdjustedRandRegressionOverPointZeroTwo": true, + "resolutionLimitImproved": true, + "articulationImproved": true + }, + "fixtures": [ + { + "name": "dense-groups-one-bridge", + "nodes": 10, + "edges": 21, + "exactRecoveryRequired": true, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.45238095238095233, + "weightedMeanConductance": 0.047619047619047616, + "worstConductance": 0.047619047619047616, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 114, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.45238095238095233, + "weightedMeanConductance": 0.047619047619047616, + "worstConductance": 0.047619047619047616, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 114, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "ring-of-cliques", + "nodes": 36, + "edges": 48, + "exactRecoveryRequired": true, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 7, + "disconnectedCommunities": 0, + "modularity": 0.701388888888889, + "weightedMeanConductance": 0.14583333333333334, + "worstConductance": 0.25, + "largestCommunityFraction": 0.16666666666666666, + "nonIsolateSingletons": 0, + "qualityVisits": 300, + "adjustedRandIndex": 0.5823389021479713, + "adjustedMutualInformation": 0.761961267862392, + "exactRecovery": false, + "falseMerges": 5, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 12, + "disconnectedCommunities": 0, + "modularity": 0.8397435897435895, + "weightedMeanConductance": 0.07692307692307693, + "worstConductance": 0.07692307692307693, + "largestCommunityFraction": 0.08333333333333333, + "nonIsolateSingletons": 0, + "qualityVisits": 300, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "articulation", + "nodes": 9, + "edges": 14, + "exactRecoveryRequired": true, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.4260204081632653, + "weightedMeanConductance": 0.07692307692307694, + "worstConductance": 0.07692307692307693, + "largestCommunityFraction": 0.5555555555555556, + "nonIsolateSingletons": 0, + "qualityVisits": 83, + "adjustedRandIndex": 0.55, + "adjustedMutualInformation": 0.5498769592136334, + "exactRecovery": false, + "falseMerges": 1, + "falseSplits": 1 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.47828408686365254, + "weightedMeanConductance": 0.02040816326530612, + "worstConductance": 0.02040816326530612, + "largestCommunityFraction": 0.5555555555555556, + "nonIsolateSingletons": 0, + "qualityVisits": 83, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "stars-and-multi-hubs", + "nodes": 16, + "edges": 15, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.43333333333333335, + "weightedMeanConductance": 0.06666666666666667, + "worstConductance": 0.06666666666666667, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 108, + "adjustedRandIndex": null, + "adjustedMutualInformation": null, + "exactRecovery": null, + "falseMerges": null, + "falseSplits": null + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.4237425635478638, + "weightedMeanConductance": 0.034482758620689655, + "worstConductance": 0.034482758620689655, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 108, + "adjustedRandIndex": null, + "adjustedMutualInformation": null, + "exactRecovery": null, + "falseMerges": null, + "falseSplits": null + } + }, + { + "name": "directed-fan-in-out", + "nodes": 12, + "edges": 10, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.5, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 76, + "adjustedRandIndex": null, + "adjustedMutualInformation": null, + "exactRecovery": null, + "falseMerges": null, + "falseSplits": null + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.5, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 76, + "adjustedRandIndex": null, + "adjustedMutualInformation": null, + "exactRecovery": null, + "falseMerges": null, + "falseSplits": null + } + }, + { + "name": "reciprocal-versus-one-way", + "nodes": 6, + "edges": 5, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.5, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 34, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.48, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 34, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "parallel-confidence-evidence", + "nodes": 4, + "edges": 8, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.16666666666666663, + "weightedMeanConductance": 0.3333333333333333, + "worstConductance": 0.3333333333333333, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 24, + "adjustedRandIndex": null, + "adjustedMutualInformation": null, + "exactRecovery": null, + "falseMerges": null, + "falseSplits": null + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.0, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.75, + "nonIsolateSingletons": 0, + "qualityVisits": 20, + "adjustedRandIndex": null, + "adjustedMutualInformation": null, + "exactRecovery": null, + "falseMerges": null, + "falseSplits": null + } + }, + { + "name": "containment-dominated-files", + "nodes": 10, + "edges": 25, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.30000000000000004, + "weightedMeanConductance": 0.2, + "worstConductance": 0.2, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 130, + "adjustedRandIndex": 0.8163265306122449, + "adjustedMutualInformation": 0.8162855361574415, + "exactRecovery": false, + "falseMerges": 1, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 2, + "disconnectedCommunities": 0, + "modularity": 0.41799587164571206, + "weightedMeanConductance": 0.0819672131147541, + "worstConductance": 0.08196721311475409, + "largestCommunityFraction": 0.5, + "nonIsolateSingletons": 0, + "qualityVisits": 130, + "adjustedRandIndex": 0.8163265306122449, + "adjustedMutualInformation": 0.8162855361574415, + "exactRecovery": false, + "falseMerges": 1, + "falseSplits": 0 + } + }, + { + "name": "isolates-and-connected-subsystems", + "nodes": 9, + "edges": 15, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.0, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.6666666666666666, + "nonIsolateSingletons": 0, + "qualityVisits": 87, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.0, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.6666666666666666, + "nonIsolateSingletons": 0, + "qualityVisits": 87, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "large-weak-community", + "nodes": 40, + "edges": 39, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 6, + "disconnectedCommunities": 0, + "modularity": 0.6985535831689677, + "weightedMeanConductance": 0.1282051282051282, + "worstConductance": 0.2, + "largestCommunityFraction": 0.225, + "nonIsolateSingletons": 0, + "qualityVisits": 276, + "adjustedRandIndex": 0.0, + "adjustedMutualInformation": 0.0, + "exactRecovery": false, + "falseMerges": 0, + "falseSplits": 1 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 6, + "disconnectedCommunities": 0, + "modularity": 0.6965811965811964, + "weightedMeanConductance": 0.1282051282051282, + "worstConductance": 0.2, + "largestCommunityFraction": 0.225, + "nonIsolateSingletons": 0, + "qualityVisits": 276, + "adjustedRandIndex": 0.0, + "adjustedMutualInformation": 0.0, + "exactRecovery": false, + "falseMerges": 0, + "falseSplits": 1 + } + }, + { + "name": "layered-handler-domain-repository", + "nodes": 12, + "edges": 26, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 3, + "disconnectedCommunities": 0, + "modularity": 0.3550295857988166, + "weightedMeanConductance": 0.3076923076923077, + "worstConductance": 0.4, + "largestCommunityFraction": 0.3333333333333333, + "nonIsolateSingletons": 0, + "qualityVisits": 140, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 3, + "disconnectedCommunities": 0, + "modularity": 0.3550295857988166, + "weightedMeanConductance": 0.3076923076923077, + "worstConductance": 0.4, + "largestCommunityFraction": 0.3333333333333333, + "nonIsolateSingletons": 0, + "qualityVisits": 140, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "tests-and-documentation", + "nodes": 12, + "edges": 21, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.16326530612244897, + "weightedMeanConductance": 0.5714285714285714, + "worstConductance": 0.6, + "largestCommunityFraction": 0.25, + "nonIsolateSingletons": 0, + "qualityVisits": 120, + "adjustedRandIndex": 0.0, + "adjustedMutualInformation": 0.0, + "exactRecovery": false, + "falseMerges": 0, + "falseSplits": 1 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 1, + "disconnectedCommunities": 0, + "modularity": 0.0, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 1.0, + "nonIsolateSingletons": 0, + "qualityVisits": 120, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "lfr-style-mixing-0", + "nodes": 32, + "edges": 77, + "exactRecoveryRequired": true, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.7471749030190589, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.25, + "nonIsolateSingletons": 0, + "qualityVisits": 404, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.7471749030190589, + "weightedMeanConductance": 0.0, + "worstConductance": 0.0, + "largestCommunityFraction": 0.25, + "nonIsolateSingletons": 0, + "qualityVisits": 404, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "lfr-style-mixing-2", + "nodes": 32, + "edges": 85, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.6535640138408305, + "weightedMeanConductance": 0.09411764705882353, + "worstConductance": 0.1111111111111111, + "largestCommunityFraction": 0.25, + "nonIsolateSingletons": 0, + "qualityVisits": 436, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.7219996795385355, + "weightedMeanConductance": 0.02531645569620253, + "worstConductance": 0.030303030303030304, + "largestCommunityFraction": 0.25, + "nonIsolateSingletons": 0, + "qualityVisits": 436, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + }, + { + "name": "lfr-style-mixing-4", + "nodes": 32, + "edges": 93, + "exactRecoveryRequired": false, + "deterministicRepeat": true, + "permutationEqual": true, + "compatibility": { + "algorithm": "seeded-louvain/v1", + "topology": "legacy-undirected/v1", + "selector": "fixed-resolution/v1", + "selectedCandidateResolution": 1.0, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.5760203491733149, + "weightedMeanConductance": 0.17204301075268819, + "worstConductance": 0.2, + "largestCommunityFraction": 0.25, + "nonIsolateSingletons": 0, + "qualityVisits": 468, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + }, + "quality": { + "algorithm": "seeded-leiden-modularity/v1", + "topology": "typed-evidence-undirected/v1", + "selector": "bounded-multiresolution/v1", + "selectedCandidateResolution": 0.75, + "communities": 4, + "disconnectedCommunities": 0, + "modularity": 0.6980643194634965, + "weightedMeanConductance": 0.04938271604938271, + "worstConductance": 0.058823529411764705, + "largestCommunityFraction": 0.25, + "nonIsolateSingletons": 0, + "qualityVisits": 468, + "adjustedRandIndex": 1.0, + "adjustedMutualInformation": 1.0, + "exactRecovery": true, + "falseMerges": 0, + "falseSplits": 0 + } + } + ] +} diff --git a/docs/implementation/community-detection-quality-qualification.md b/docs/implementation/community-detection-quality-qualification.md new file mode 100644 index 000000000..18dc53974 --- /dev/null +++ b/docs/implementation/community-detection-quality-qualification.md @@ -0,0 +1,89 @@ +# Community detection quality qualification + +This report records deterministic fixture qualification for the version-1 +typed topology, native Leiden detector, quality evidence, and bounded selector. +The machine-readable authority is +[`community-detection-quality-qualification.json`](community-detection-quality-qualification.json). + +## Qualified identities + +```text +algorithm seeded-leiden-modularity/v1 +topology typed-evidence-undirected/v1 +quality community-quality/v1 +fixed fixed-resolution/v1 +automatic bounded-multiresolution/v1 +seed 42 +limits community-limits/v1 +``` + +The production cutover uses fixed resolution. The automatic selector remains a +qualification capability, not an omitted-flag default. + +## Deterministic fixture result + +The native runner covers 15 fixture families and repeats each input with node +and edge order reversed. All checked acceptance fields pass: + +| Gate | Result | +| --- | --- | +| Connected selected communities | pass | +| Repeat and permutation equality | pass | +| Required exact planted recovery | pass | +| No ARI regression greater than 0.02 | pass | +| Ring-of-cliques improvement | pass | +| Articulation improvement | pass | + +The ring-of-cliques fixture improves from ARI `0.5823389021` and 5 false +merges under compatibility Louvain to ARI/AMI `1.0` with no false merges or +splits. The articulation fixture improves from ARI `0.55`, one false merge, +and one false split to exact recovery. Dense groups and required deterministic +LFR-style fixtures retain exact recovery. + +Run and byte-compare the report with: + +```bash +./scripts/qualify_code_graph_v1.sh --community-quality \ + --report docs/implementation/community-detection-quality-qualification.json +``` + +## Compact performance decision + +On 2026-09-12, an aarch64 macOS debug build ran all 15 compact fixture families +50 times per sample. Seven-process medians were: + +| Profile | Median | Compatibility ratio | +| --- | ---: | ---: | +| compatibility Louvain | 1.25 s | 1.00× | +| fixed-resolution typed Leiden | 1.82 s | 1.46× | +| three-candidate typed Leiden | 3.01 s | 2.41× | + +These intentionally small debug fixtures magnify topology/evidence setup cost +and are not the pinned real-repository release performance oracle. They do show +that automatic selection cannot be enabled merely from the correctness result. +The production profile therefore stays fixed-resolution as required by the +design's fallback rule. No cold-build or peak-RSS claim is inferred from these +numbers. + +## Corpus status and omissions + +The checked-in gate is completely offline and deterministic. A cold build of +the available pinned Rust corpus (`kache` at +`1a6a4a6067ab98c2867cb2278a033e0a208ff4b9`) was attempted, but both clustered +and `--no-cluster` runs remained in pre-publication extraction work long enough +that they were stopped; no clustering-stage comparison can be inferred from +those incomplete observations. It does not claim +that paths or package names are ground truth. Release qualification must still +record reviewed expectations, exact commits, cold build time, clustering time, +peak RSS, and incremental measurements for the pinned repository corpus. A +missing ecosystem is an explicit corpus omission, not a passing measurement. +This fixture report cannot authorize the automatic selector as the default. + +## Related pages + +- [Community detection concept](../concepts/community-detection.md) +- [Technical design](community-detection-quality-technical-design.md) +- [Performance qualification](../../PERFORMANCE.md) + +**Next step:** run the pinned real-repository performance matrix before a +future change enables bounded automatic selection in production. diff --git a/docs/implementation/community-detection-quality-technical-design.md b/docs/implementation/community-detection-quality-technical-design.md new file mode 100644 index 000000000..0663e0bc0 --- /dev/null +++ b/docs/implementation/community-detection-quality-technical-design.md @@ -0,0 +1,1108 @@ +# Community detection quality technical design + +Status: implemented; automatic selector remains qualification-only + +Scope: `compass-graph` community topology, detection, quality, incremental +updates, and `compass-core` orchestration + +Implementation status (2026-09-12): phases 0 through 7 are implemented. The +production profile uses fixed-resolution typed Leiden. The bounded +three-candidate selector remains qualification-only because its compact +performance measurement did not justify enabling it by default. See the +linked qualification report. + +Extends: +[Architecture graph hardening technical design](architecture-graph-hardening-phased-technical-design.md) + +## Overview + +Compass currently derives graph communities with a deterministic native +multi-level Louvain implementation. It uses weighted edges, a fixed seed, +optional hub exclusion, stable remapping, oversized-community splitting, and +an internal-density score called cohesion. + +The implementation is fast and reproducible, but four limitations prevent +Compass from demonstrating that a changed partition is better: + +1. clustering collapses most typed Base Graph relationships into equivalent + undirected edges; +2. Louvain can return weakly connected or disconnected communities; +3. one fixed resolution is accepted before a small set of hard-coded split + rules runs; and +4. cohesion measures only internal density, not separation, connectedness, + stability, or agreement with reviewed expectations. + +This design introduces one native community module with a small public +interface and five internal responsibilities: + +```text +validated Base Graph + | + v +versioned community topology + | + +---------------------> quality evaluator + | ^ + v | +bounded partition candidates ---------+ + | + v +deterministic selector + | + v +stable IDs + labels + quality evidence +``` + +The implementation path deliberately separates measurement, structural +refactoring, topology semantics, detector changes, and rollout. No behavior +change is accepted only because it increases modularity or produces more +communities. + +## Relationship to architecture projection + +This design changes graph-derived community detection. It does not replace the +existing `compass-output::architecture_projection` module. + +The two quality domains remain distinct: + +| Domain | Owner | Meaning | +| --- | --- | --- | +| Community quality | `compass-graph` | Connectivity and separation of one graph partition | +| Architecture quality | `compass-output::architecture_projection` | Source-scope correctness, grouping, naming, coverage, omissions, and presentation quality | + +The Architecture projection continues to classify Production, Test, +Generated, Vendor, Documentation, and Unknown before architecture grouping. +It may consume community-quality evidence, but it must not redefine the +detector or mutate the Base Graph. + +Communities remain graph-derived hypotheses. They are not official modules or +hand-maintained architecture declarations. + +## Current implementation + +The current path is distributed across several callers: + +```text +compass-core::pipeline + -> convert typed graph to legacy node-link projection + -> cluster_incremental + -> WeightedGraph::from_document + -> seeded Louvain + -> hub reattachment + -> large/low-density split passes + -> stable ID remapping + -> label_communities_by_hub + -> score_communities + -> graph insights and output +``` + +`cluster-only`, historical normalization, architecture projection, and viewer +model construction repeat parts of this sequence. + +### Current topology semantics + +`WeightedGraph::from_document`: + +- sorts node IDs; +- ignores dangling endpoints; +- excludes only table-navigation containment; +- reads `weight`, defaulting to `1.0`; +- collapses duplicate directed endpoint pairs; +- projects the selected edges into symmetric adjacency; and +- does not otherwise use relationship kind, direction, multiplicity, + confidence, or provenance. + +### Current quality semantics + +The public cohesion value is: + +```text +unique internal endpoint pairs / (member_count * (member_count - 1) / 2) +``` + +Singletons receive `1.0`. External cut edges do not affect the score. The +detector uses a separate topology-derived density calculation for its split +rule, while analysis has another cohesion implementation. These calculations +can observe different effective graphs. + +### Current incremental semantics + +An incremental update admits changed nodes, their previous communities, and +the immediate community boundary. It clusters the induced affected subgraph +while freezing all other assignments. Edges from the affected region to +frozen communities are absent from the local objective. The implementation +falls back to a full run when the admitted region exceeds 4,096 nodes or 25% +of the current graph. + +## Goals + +- Make detector quality measurable independently from architecture rendering. +- Give detection and quality exactly the same versioned topology. +- Use relationship kind, evidence confidence, multiplicity, and direction + through an explicit deterministic policy. +- Guarantee that every non-isolate community is internally connected. +- Improve small-module recovery without allowing unbounded resolution search. +- Preserve deterministic output for equivalent graph and profile inputs. +- Keep structural community detection native, local, bounded, and credential + free. +- Preserve Base Graph node, relationship, direction, multiplicity, identity, + and provenance contracts. +- Keep previous community numbering out of content-addressed historical + realization identity. +- Detect incremental quality degradation and fall back to a full run. +- Publish inspectable quality evidence without pretending that one partition + is the only correct architecture. +- Establish real-repository and planted-partition qualification before changing + the default detector. + +## Non-goals + +- Add Graphify or another runtime, test, configuration, or fallback dependency. +- Call a provider, model, embedding service, or vector database. +- Add a dynamic clustering plugin system. +- Infer business ownership from display labels or path-name similarity. +- Rewrite the Base Graph to fit a preferred community partition. +- Hide Test, Generated, Vendor, Documentation, or Unknown records from the Base + Graph. +- Make a prior working-tree partition influence an exact historical + realization. +- Treat higher modularity, lower conductance, or package agreement alone as + ground truth. +- Replace architecture overlays or explicit owner groups with detected + communities. +- Introduce overlapping community membership in this implementation line. +- Adopt Constant Potts Model semantics under the existing `--resolution` + option. CPM may be evaluated later under a separately versioned profile. + +## Required invariants + +### Base Graph fidelity + +- Community topology is a derived read view; Base Graph records are unchanged. +- Every admitted topology edge retains counts by relationship kind and + confidence in its topology evidence. +- Direction and multiplicity may be projected for detection only by an + explicit versioned rule. +- Ambiguous or unsupported meaning never becomes an exact relationship. +- Unknown relationship kinds fail profile validation; they are not silently + assigned a default semantic weight. + +### Determinism + +- Equivalent Base Graphs and community profiles produce equivalent memberships, + quality evidence, diagnostics, and ordering. +- Node and edge input order cannot change the result. +- Node visitation uses a fixed recorded seed and canonical starting order. +- Candidate ties resolve by a canonical partition digest, never hash-map or + filesystem iteration. +- Floating-point comparisons use one named tolerance and deterministic + secondary ordering. +- Parallel evaluation collects candidates into canonical order before + selection. + +### Bounded work + +- Topology nodes, input relationships, projected pairs, total weight, + candidates, levels, local moves, quality visits, and incremental affected + nodes all have explicit limits. +- Candidate generation uses a fixed schedule with at most three partitions in + the first production version. +- Exhausting a limit returns a typed error with required and allowed work. It + never returns an empty or partial partition as success. +- Qualification may run a wider offline matrix, but normal builds remain under + the production limits. + +### Partition integrity + +- Every admitted node belongs to exactly one community. +- Every member ID exists in the input graph. +- Every non-isolate community is internally connected in the selected topology. +- Community member lists and community ordering are canonical. +- Stable remapping changes IDs only; it cannot change membership or quality. +- A failed postcondition prevents publication of the new coherent artifact set. + +### History + +- Algorithm, topology, quality, selector, seed, resolution policy, hub policy, + and all meaning-affecting limits enter the historical build profile. +- Existing historical realizations remain immutable and queryable. +- A rebuild uses the running engine profile and creates a new realization; it + does not rewrite the old one. +- Operational reuse of previous community IDs remains excluded from historical + content identity. + +## Design decisions + +### 1. Qualify before changing the objective + +The first phase adds measurements and fixtures without modifying production +memberships. It records both strengths and known weaknesses of +`seeded-louvain/v1`. + +The qualification suite contains three evidence classes: + +1. compact hand-reviewed topology fixtures; +2. deterministic planted-partition graphs with known memberships; and +3. pinned public repositories with reviewable subsystem expectations. + +Package or directory agreement is supporting evidence, not truth. A codebase +can intentionally place one subsystem across several packages or several +subsystems in one package. + +### 2. Deepen one community module + +The current free-function family becomes one module owned by `compass-graph`. +Its public interface accepts a validated graph plus one complete request and +returns one complete result. + +Target ownership map: + +```text +crates/compass-graph/src/community/ +|-- mod.rs public facade, validation, complete result +|-- topology.rs versioned Base Graph projection +|-- quality.rs partition and per-community evidence +|-- detection.rs Louvain compatibility and native Leiden +|-- selection.rs bounded candidate schedule and ordering +|-- incremental.rs affected region, frozen influence, fallback +`-- labels.rs stable remapping, signatures, base labels +``` + +This is an ownership map, not a requirement to maximize file count. Files stay +combined when their invariants cannot be understood or tested independently. + +### 3. Preserve a behavior-equivalent compatibility profile + +Before adding typed weighting, `topology.rs` reproduces the current +`WeightedGraph::from_document` behavior. The Louvain implementation moves +behind the new facade without semantic edits. + +The compatibility profile remains identified as: + +```text +algorithm: seeded-louvain/v1 +topology: legacy-undirected/v1 +quality: density/v1 +selector: fixed-resolution/v1 +seed: 42 +``` + +Cold, warm, input-permuted, cluster-only, and historical fixture results must +remain byte-equivalent before later phases begin. + +### 4. Add a typed evidence topology + +The new topology consumes typed `compass.graph/1` relationships. A legacy +node-link adapter remains only where direct reclustering compatibility requires +it. Both adapters feed the same validated internal topology. + +The first typed topology remains undirected for partition detection, but its +symmetrization is explicit: + +```text +pair strength = bounded forward evidence + bounded reverse evidence +``` + +Reciprocal relationships therefore contribute more evidence than a one-way +relationship without erasing their original directions. The topology result +retains forward, reverse, relation, confidence, and omission summaries for +inspection. + +Relationship kinds are assigned exhaustively to closed strength classes: + +| Strength class | Candidate relationship families | Intent | +| --- | --- | --- | +| Strong | calls, handles, routes, reads/writes, messaging, scheduling | Runtime or domain flow | +| Medium | imports, depends-on, instantiates, registers, inheritance | Dependency and type coupling | +| Weak | containment, references, exports, aliases, tests, documents | Structural or navigational support | +| Excluded | invalid or profile-inapplicable evidence | Retained in Base Graph, absent from this topology | + +The exact `EdgeKind` table and integer weights are frozen only after Phase 1 +qualification. The table is compile-time exhaustive, reviewed in one file, +and versioned as part of the topology identity. + +Evidence confidence modifies, but never upgrades, relationship strength: + +| Evidence | Initial candidate policy | +| --- | --- | +| Exact/source-derived | Full relation strength | +| Inferred with complete provenance | Reduced relation strength | +| Ambiguous | Excluded from selection topology and counted in evidence | + +Repeated evidence uses a bounded saturating contribution rather than either +discarding all multiplicity or allowing generated repetition to dominate. The +initial cap is four distinct source-anchored occurrences per unordered node +pair and relationship kind. Qualification may lower the cap before the policy +is frozen. + +The graph's finite positive `weight` remains an input factor. Normalization and +the maximum aggregate pair weight are explicit profile constants. Invalid or +overflowing values fail topology construction rather than becoming zero. + +Source scope does not enter the first typed community topology. Production +source isolation remains owned by Architecture projection. A future +scope-specific partition would require a separate profile and contract rather +than importing output-owned path rules into `compass-graph`. + +### 5. Implement native Leiden with modularity + +The first detector improvement keeps the current generalized modularity +objective and resolution meaning. It adds the Leiden refinement phase rather +than changing both algorithm and objective together. + +Each level performs: + +1. deterministic seeded local moving; +2. refinement inside each coarse community from a connected singleton + partition; +3. aggregation using the refined partition; and +4. another level until improvement stops or the level limit is reached. + +Refinement permits only moves that preserve the connectedness conditions of +the candidate community. The selected result is validated independently with +a bounded connected-components pass. A validation failure is a detector error, +not a silent repair. + +The implementation remains native Rust inside `compass-graph`; it adds no +external runtime or clustering library. Small checked-in oracle fixtures may +be derived once from the published algorithm description, but normal tests do +not execute Python, Java, or a network service. + +The detector identity becomes: + +```text +seeded-leiden-modularity/v1 +``` + +The Constant Potts Model is deliberately deferred. Its resolution parameter +has a different density interpretation, so reusing the current option would be +an incompatible semantic shortcut. + +### 6. Select from a bounded resolution schedule + +When the user does not explicitly supply `--resolution`, the quality profile +may generate at most three candidates around the configured base resolution: + +```text +3/4 * base, base, 4/3 * base +``` + +Rational multipliers avoid a hidden arbitrary sweep. Each resulting partition +is evaluated at the common base resolution so values are comparable. An +explicit `--resolution N` requests one fixed candidate at exactly `N` and +therefore preserves direct user control. + +Selection is lexicographic and inspectable, not an opaque weighted sum: + +1. reject incomplete or invalid candidates; +2. reject candidates with disconnected non-isolate communities; +3. retain candidates within the named modularity tolerance of the best + base-resolution modularity; +4. minimize communities that violate the qualified size and separation + thresholds; +5. minimize weighted boundary conductance; +6. minimize non-isolate singleton fragmentation; and +7. break remaining ties by canonical partition digest. + +Candidate schedules, rejected candidates, metrics, and the final selection +reason are part of quality evidence. + +The current oversized and low-density one-shot split rules remain in the +compatibility profile. They are removed from the new profile once Leiden and +candidate selection satisfy the same pathological-size fixtures. A refinement +pass must never silently reset a user-supplied resolution to `1.0`. + +### 7. Evaluate a quality vector, not one magic score + +`CommunityQuality` records evidence rather than claiming universal correctness. + +Per-community evidence includes: + +- member count; +- isolate status; +- connected-component count; +- internal unique edge count and internal weight; +- boundary edge count and boundary weight; +- volume; +- density; +- conductance; +- modularity contribution; +- relation-strength mix; +- evidence-confidence mix; and +- bounded witness node and edge IDs for degraded conditions. + +Partition evidence includes: + +- assigned and omitted node counts; +- community and non-isolate singleton counts; +- disconnected-community count; +- total modularity at the base resolution; +- weighted mean and worst retained conductance; +- largest-community fraction; +- candidate agreement and selection reason; +- algorithm/topology/quality/selector identities; and +- exact limits and omissions. + +Density remains available as the compatibility `cohesion` projection. It is +not used alone to label a partition good. `CommunityMetadata.score` remains +unset until Compass defines and versions a scalar meaning; the quality vector +must not be compressed into that existing optional field prematurely. + +### 8. Preserve frozen influence during incremental updates + +The incremental topology represents each adjacent frozen community as one +locked anchor node. Edge weights from affected nodes to that community are +aggregated onto the anchor. + +```text +frozen community A ----\ + affected topology ---- frozen community B +frozen community C ----/ +``` + +Affected nodes may join or leave an anchored community. Locked anchors cannot +move, and two different frozen anchors cannot merge in a local run. A local +community without an anchor receives a deterministic new ID after selection. + +After local clustering, the complete merged partition is evaluated against the +complete current topology. The implementation falls back to a full run when: + +- any partition invariant fails; +- a non-isolate community is disconnected; +- base-resolution modularity is worse than the deterministic prior-plus-new- + singleton baseline beyond tolerance; +- dominant-community or conductance limits are newly violated; +- two frozen anchors would merge; +- affected work exceeds the existing absolute or fractional limit; or +- a quality or topology limit is exhausted. + +Repeated edit, revert, rename, and delete sequences compare the final +membership with a clean full build. Exact membership equality is required for +the deterministic fixtures where the optimum is unique; quality equivalence +and explicit stable-ID rules apply where multiple partitions tie. + +Temporal agreement with previous working-tree assignments is diagnostic only. +It cannot influence an exact historical realization. + +## Proposed Rust interface + +Names are illustrative but the ownership and information flow are required. + +```rust +pub struct CommunityRequest<'a> { + pub profile: CommunityProfile, + pub resolution: ResolutionPolicy, + pub exclude_hubs_percentile: Option, + pub previous: Option<&'a PreviousCommunities>, + pub changed_sources: &'a BTreeSet, + pub limits: CommunityLimits, +} + +pub enum ResolutionPolicy { + Auto { base: f64 }, + Fixed(f64), +} + +pub enum CommunityProfile { + CompatibilityV1, + QualityV1, +} + +pub struct CommunityResult { + pub communities: Communities, + pub base_labels: BTreeMap, + pub signatures: BTreeMap, + pub quality: PartitionQuality, + pub execution: CommunityExecution, + pub identity: CommunityIdentity, +} + +pub enum CommunityExecution { + Full, + Incremental { affected_nodes: usize }, + FullFallback { affected_nodes: usize, reason: FallbackReason }, +} + +pub fn build_communities( + document: &compass_model::code_graph::GraphDocument, + request: &CommunityRequest<'_>, +) -> Result; +``` + +The existing `cluster`, `cluster_incremental`, `score_communities`, stable +remapping, signature, and hub-label functions remain temporarily as +compatibility facades. New production callers use `build_communities`; the +facades are removed or made crate-private only after downstream migration. + +Two input adapters justify the graph seam during migration: + +- the typed Base Graph adapter for normal builds and history; and +- the validated legacy node-link adapter for supported direct reclustering. + +Both adapters must produce the same internal topology when their source facts +are equivalent. + +## Error and limit model + +Community work returns typed errors: + +```text +invalid_profile +invalid_resolution +invalid_edge_weight +unknown_relationship_kind +dangling_endpoint +topology_limit_exceeded +candidate_limit_exceeded +move_limit_exceeded +quality_limit_exceeded +partition_incomplete +partition_disconnected +partition_duplicate_member +partition_unknown_member +``` + +Each limit error reports: + +```text +stage +required +limit +processed +``` + +The normal pipeline treats these as build failures and retains the prior +coherent artifact set. `cluster-only` writes no partially reclustered graph. +Qualification tools may record a failed candidate, but the production selector +cannot select one. + +The initial production limits retain the current 10-level ceiling and +incremental 4,096-node/25% ceilings. Phase 1 measurements establish explicit +move, projected-pair, weight, and quality-visit ceilings before the new profile +is enabled. + +## Quality qualification + +### Fixture families + +The native fixture suite includes: + +- two dense groups joined by one bridge; +- a ring of cliques that exposes modularity resolution behavior; +- an articulation graph that can create a disconnected Louvain community; +- stars and multi-hub graphs; +- directed fan-in and fan-out graphs; +- reciprocal versus one-way relationships; +- parallel exact, inferred, and ambiguous evidence; +- containment-dominated file graphs; +- isolated nodes mixed with connected subsystems; +- one very large weak community; +- layered handler/domain/repository code topology; +- tests and documentation connected to production code; and +- input-order and edge-order permutations of every compact case. + +Deterministic LFR-style fixtures cover heterogeneous degree and community-size +distributions at several mixing levels. The generator or generated fixture is +native and pinned; qualification never downloads a corpus. + +### Metrics + +Where planted membership exists, qualification records: + +- adjusted Rand index; +- adjusted mutual information; +- exact recovery rate; and +- false merge/split counts. + +For all graphs it records: + +- connectedness; +- modularity on a named common topology and resolution; +- per-community and weighted conductance; +- density; +- largest-community and non-isolate singleton fractions; +- deterministic repeat and permutation equality; +- edit/revert stability; +- clustering wall time; and +- peak resident memory. + +Metrics from different topology identities are never compared as if they used +the same denominator. + +### Real repositories + +Release qualification uses pinned public repositories already present under +`/Volumes/Workspace/Github` where available. Existing checkouts are read-only. +Missing authorized public corpora are cloned only under that mounted volume. + +The corpus should cover: + +- a Rust workspace; +- a Java or Kotlin multi-module project; +- a TypeScript monorepo; +- a Python package with tests and documentation; +- a Go multi-package project; +- a frontend/backend project; and +- Compass itself. + +Each corpus records commit, build profile, topology identity, reviewed +subsystem expectations, and exact omissions. Repository paths or package names +may support review but cannot be the sole accuracy oracle. + +### Acceptance gates + +The new default is eligible only when all of these hold: + +- zero disconnected non-isolate communities across fixture and pinned-corpus + runs; +- exact deterministic repeat and input-permutation equality; +- exact recovery for unambiguous planted fixtures; +- no planted-fixture adjusted metric regression greater than 0.02 and a net + improvement on resolution-limit and articulation families; +- no unexplained real-repository dominant-community or singleton regression; +- no Base Graph node, edge, identity, direction, multiplicity, or provenance + change; +- cold full-build wall time no more than 15% above compatibility on the pinned + median; +- clustering-stage wall time no more than 50% above compatibility on the + pinned median; +- peak RSS no more than 10% above compatibility; +- incremental one-file updates stay within their documented node and memory + bounds; and +- every regression includes bounded witness evidence. + +If the three-candidate auto schedule misses the performance gates, the default +remains fixed-resolution Leiden and multi-resolution selection stays +qualification-only until optimized. + +## Compatibility and versioning + +The default detector change is compatibility-sensitive even if +`compass.graph/1` remains unchanged, because community membership affects +navigation, reports, architecture grouping, history comparisons, and derived +artifacts. + +Meaning-affecting identities move from CLI literals into `compass-graph` and +enter every current and historical build profile: + +```text +cluster_algorithm +cluster_topology +cluster_quality +cluster_selector +cluster_seed +cluster_resolution_policy +cluster_hub_policy +cluster_limits_version +``` + +The first qualified target identities are: + +```text +cluster_algorithm = seeded-leiden-modularity/v1 +cluster_topology = typed-evidence-undirected/v1 +cluster_quality = community-quality/v1 +cluster_selector = bounded-multiresolution/v1 +``` + +If auto selection does not qualify, the selector identity is instead +`fixed-resolution/v1`. + +The configuration digest changes whenever a meaning-affecting identity or +option changes. Existing output is rebuilt coherently rather than partially +reusing old memberships. + +`--resolution N` remains supported and means fixed generalized-modularity +resolution `N`. Omitting it may become bounded automatic selection after that +behavior qualifies. Help and reference documentation must state the +difference. + +`--exclude-hubs N` remains explicit. Hub removal and reattachment operate on +the selected topology and are included in quality evidence. A hub cannot be +reattached solely by an ambiguous excluded edge. + +The rollout requires: + +- native CLI and graph regression coverage; +- command and output reference updates; +- a `MIGRATION.md` note explaining expected community-ID and membership churn; +- a `CHANGELOG.md` entry; +- updated history-profile fixtures; and +- a new qualification report with exact corpus identities and measurements. + +Published historical realizations are never rewritten. No compatibility mode +silently substitutes the old detector for the new profile. + +## Quality evidence publication + +Phases 1 through 5 keep the richer quality result internal and in qualification +reports while its meaning stabilizes. Existing `cohesion` output remains the +compatibility density projection. + +After detector acceptance, normal clustered builds may add the strict artifact: + +```text +community-quality.json +schema: compass.community-quality/1 +``` + +The artifact contains: + +- graph generation and canonical graph digest; +- complete community profile identity; +- selected and rejected candidate summaries; +- per-community quality evidence; +- partition evidence; +- diagnostics with bounded witnesses; +- limits and exact omissions; and +- a canonical result digest. + +It joins the guarded coherent artifact set and immutable history realization. +Unknown majors fail explicitly. A missing artifact on an older graph means +quality evidence is unavailable, not zero or good. + +The human report may summarize this artifact. Renderers cannot recalculate or +reinterpret its status. Architecture projection may reference its community +metrics only after validating graph generation, digest, and profile identity. + +## Implementation phases + +### Phase 0: approve design and freeze current behavior + +Actions: + +1. Add current Louvain, topology, split, remapping, and density behavior tables. +2. Add byte-equivalence fixtures for normal, cluster-only, and historical paths. +3. Record current clustering stage time, RSS, memberships, modularity, density, + connectedness, conductance, and size distribution. +4. Record the existing algorithm identity in one `compass-graph` constant. + +Done when: + +- production behavior is unchanged; +- every current split threshold has a named characterization test; and +- the baseline report is reproducible. + +Suggested commits: + +1. `test(graph): characterize community topology and quality` +2. `docs(graph): record community detection baseline` +3. `refactor(graph): centralize clustering identity constants` + +### Phase 1: build the quality qualification seam + +Actions: + +1. Implement shared internal topology statistics. +2. Implement per-community and partition quality evidence. +3. Make current cohesion call the shared density calculation. +4. Add planted and pathological fixtures. +5. Add a focused native qualification runner and deterministic JSON report. +6. Capture the pinned real-repository baseline. + +Done when: + +- all quality consumers observe the same compatibility topology; +- formulas have direct reference tests; +- every metric reports its denominator and omissions; and +- no production membership changes. + +Suggested commits: + +1. `feat(graph): add shared community quality evidence` +2. `test(graph): add planted community qualification fixtures` +3. `test(graph): add deterministic community quality runner` +4. `docs(graph): publish compatibility quality baseline` + +### Phase 2: deepen the community module without semantic change + +Actions: + +1. Move compatibility topology and Louvain behind the new facade. +2. Move incremental clustering, remapping, signatures, and base labels. +3. Route `pipeline`, `cluster_existing`, history, viewer model, and + architecture projection through the facade or shared quality evaluator. +4. Remove duplicated orchestration and cohesion implementations. +5. Add typed errors and explicit work accounting. + +Done when: + +- compatibility results remain byte-equivalent; +- callers no longer sequence detector internals; +- the complete result is the test surface; and +- graph/output ownership remains unchanged. + +Suggested commits: + +1. `refactor(graph): introduce community result facade` +2. `refactor(graph): move compatibility topology and detector` +3. `refactor(core): consume complete community results` +4. `refactor(output): reuse community quality evidence` + +### Phase 3: add typed topology as a candidate + +Actions: + +1. Add the typed Base Graph adapter. +2. Add the exhaustive relationship strength table. +3. Add confidence and bounded multiplicity contributions. +4. Add reciprocal-direction aggregation and evidence summaries. +5. Run a bounded policy matrix against Phase 1 qualification. +6. Freeze the best justified weights as `typed-evidence-undirected/v1`. + +Done when: + +- typed relationships are never silently defaulted; +- every topology edge is explainable by bounded evidence counts; +- Base Graph artifacts are byte-identical; and +- the candidate meets planted, real-repository, time, and memory gates. + +Suggested commits: + +1. `feat(graph): project typed community topology` +2. `feat(graph): weight relationship and confidence evidence` +3. `test(graph): qualify bounded topology policies` +4. `docs(graph): freeze typed topology v1` + +### Phase 4: implement and qualify native Leiden + +Actions: + +1. Extract the common deterministic local-moving kernel. +2. Implement connected refinement and aggregate propagation. +3. Add independent connectedness validation. +4. Add oracle, articulation, resolution, tie, and limit tests. +5. Compare Leiden and compatibility Louvain on identical topologies. + +Done when: + +- every selected Leiden community is connected; +- deterministic and limit gates pass; +- quality improves on the targeted fixtures; and +- performance stays within the acceptance envelope. + +Suggested commits: + +1. `refactor(graph): isolate deterministic local moving` +2. `feat(graph): add native Leiden refinement` +3. `test(graph): verify Leiden connectivity and determinism` +4. `docs(graph): record Leiden qualification` + +### Phase 5: add bounded candidate selection + +Actions: + +1. Distinguish omitted resolution from explicit `--resolution` in CLI parsing. +2. Generate the three rational auto candidates. +3. Implement common-resolution evaluation and lexicographic selection. +4. Record rejection and tie-break evidence. +5. Remove new-profile dependence on the legacy split passes. +6. Run full quality and performance qualification. + +Done when: + +- explicit resolution produces exactly one candidate; +- automatic work never exceeds three candidates; +- selector output is deterministic and inspectable; and +- the performance gates decide whether auto selection may become default. + +Suggested commits: + +1. `feat(graph): evaluate bounded resolution candidates` +2. `feat(graph): select partitions from quality evidence` +3. `feat(cli): preserve explicit fixed resolution intent` +4. `test(graph): qualify automatic partition selection` + +### Phase 6: preserve incremental quality + +Actions: + +1. Replace the induced-only local topology with locked frozen-community anchors. +2. Add full-topology merged-partition evaluation. +3. Add explicit quality fallback reasons. +4. Add repeated edit/revert/rename/delete sequences. +5. Measure one-file update time and RSS on pinned corpora. + +Done when: + +- frozen external influence participates in local selection; +- a degraded local result always falls back or fails explicitly; +- unaffected assignments remain frozen when the local result qualifies; and +- incremental performance retains its documented advantage. + +Suggested commits: + +1. `feat(graph): retain frozen influence in local clustering` +2. `feat(graph): guard incremental partition quality` +3. `test(graph): cover incremental drift and reversibility` +4. `docs(graph): publish incremental quality measurements` + +### Phase 7: coordinated default cutover + +Actions: + +1. Freeze algorithm, topology, quality, selector, and limit identities. +2. Thread those identities through build state, configuration digests, and + immutable history profiles. +3. Switch normal, cluster-only, viewer, and historical materialization callers + together. +4. Publish `compass.community-quality/1` if its contract has been accepted. +5. Update command, output, concept, compatibility, migration, changelog, and + performance documentation. +6. Run the complete repository baseline and matching product gates. + +Done when: + +- no caller silently uses default Louvain; +- old output is coherently invalidated and rebuilt; +- historical realizations remain immutable; +- every public consumer validates the same profile identity; and +- the final qualification report satisfies all acceptance gates. + +Suggested commits: + +1. `feat(history): version complete community profiles` +2. `feat(graph): adopt qualified community profile` +3. `feat(output): publish community quality evidence` +4. `docs: document community detection cutover` + +### Phase 8: cleanup after one release line + +Actions: + +1. Remove obsolete orchestration helpers and duplicated density code. +2. Make compatibility facades crate-private unless a supported library caller + still requires them. +3. Retain only code required to read immutable prior artifacts; do not emulate + old builds implicitly. +4. Re-run size, time, RSS, and determinism qualification. + +Done when: + +- the community module has one production interface; +- algorithm policy is not duplicated in CLI, core, output, or viewer code; and +- compatibility documentation matches retained behavior. + +## Verification matrix + +Every Cargo command must use this worktree's dedicated target directory under +`/Volumes/Workspace/crabbuild-target`. + +### Narrow checks + +```bash +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo test -p compass-graph --locked + +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo test -p compass-core --test code_graph_v1_determinism --locked + +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo test -p compass-cli --test history_cli --locked + +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo test -p compass-cli --test viewer_export_cli --locked +``` + +### Product and graph gates + +```bash +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo test -p compass-cli --test compass_product --locked + +sh scripts/check_product_boundary.sh +./scripts/qualify_code_graph_v1.sh --fixtures-only +``` + +The code-graph qualification runner must gain a clustered community-quality +mode; its existing extraction-focused no-cluster mode remains unchanged. + +### Baseline before rollout + +```bash +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo fmt --all -- --check + +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo clippy --workspace --lib --bins --locked -- -D warnings + +CARGO_TARGET_DIR=/Volumes/Workspace/crabbuild-target/compass-c112 \ + cargo test --workspace --lib --bins --locked +``` + +Viewer tests are required when a new quality artifact or visualization is +published: + +```bash +npm ci +npm run typecheck:js +npm run test:js +node scripts/check_viewer_assets.mjs +``` + +## Rollback + +Phases 0 through 6 are additive or internal and can be reverted independently +while the compatibility profile remains the default. + +After the Phase 7 cutover: + +- do not rewrite published current or historical artifacts; +- restore a previous default only with a new engine/profile identity; +- invalidate current output through the normal configuration digest; +- document the membership change; and +- publish a new qualification report. + +Rollback never means interpreting one profile's memberships as another +profile's result. + +## Risks and mitigations + +| Risk | Mitigation | +| --- | --- | +| Typed weights encode arbitrary preferences | Freeze only after planted and real-corpus policy qualification; publish the table and evidence mix | +| Multi-resolution multiplies cost | Cap at three candidates; explicit resolution uses one; withhold auto default if performance fails | +| Leiden implementation is complex | Keep modularity unchanged, add oracle fixtures, validate connectedness independently | +| Conductance favors trivial partitions | Use it only after integrity and modularity plateau gates, never as a sole objective | +| Package agreement becomes circular truth | Treat it as supporting review evidence, not the detector's input or only oracle | +| Incremental state affects history | Exclude prior assignments from historical selection and content identity | +| New profile causes community-ID churn | Version the complete profile, rebuild coherently, document migration, preserve immutable history | +| Quality artifact is mistaken for truth | Publish a metric vector, witnesses, limits, and explicit hypothesis language | +| Structural refactor hides semantic changes | Require compatibility byte-equivalence before typed topology or Leiden commits | + +## Research basis + +- Traag, Waltman, and van Eck, + [From Louvain to Leiden: guaranteeing well-connected communities](https://www.nature.com/articles/s41598-019-41695-z) + motivates the connected refinement phase and independent connectivity gate. +- Fortunato and Barthélemy, + [Resolution limit in community detection](https://pmc.ncbi.nlm.nih.gov/articles/PMC1765466/) + motivates bounded multi-resolution qualification instead of treating one + modularity optimum as universal. +- Lancichinetti, Fortunato, and Radicchi, + [Benchmark graphs for testing community detection algorithms](https://arxiv.org/abs/0805.4770) + motivates heterogeneous planted-partition fixtures. +- Leicht and Newman, + [Community structure in directed networks](https://arxiv.org/abs/0709.4500) + motivates making direction projection explicit rather than silently + discarding it. + +These papers inform the design. Compass's contract remains defined by native +implementation, fixtures, pinned qualification, and published versioned +profiles. + +## Related pages + +- [Architecture graph hardening technical design](architecture-graph-hardening-phased-technical-design.md) +- [Architecture graph hardening qualification](architecture-graph-hardening-qualification.md) +- [Extraction pipeline](extraction-pipeline.md) +- [Workspace tour](workspace-tour.md) +- [Graph model](../concepts/graph-model.md) +- [How Compass works](../concepts/how-it-works.md) +- [Design principles](../design/principles.md) +- [Compatibility ledger](../../COMPATIBILITY.md) +- [Performance qualification](../../PERFORMANCE.md) +- [Community detection qualification](community-detection-quality-qualification.md) + +**Next step:** retain fixed-resolution Leiden in production and complete the +pinned real-repository performance matrix before proposing automatic selection. diff --git a/docs/reference/commands.md b/docs/reference/commands.md index 38ca41d4b..56f407d35 100644 --- a/docs/reference/commands.md +++ b/docs/reference/commands.md @@ -105,6 +105,13 @@ contain different uncommitted changes and must not write one shared mutable output directory. Repository-wide immutable history and its verified-content cache remain shared through the Git common directory. +Clustered builds use deterministic fixed-resolution Leiden over the typed +evidence topology. Omitting `--resolution` uses `1`; `--resolution N` uses the +single positive finite value `N`. Higher values generally create smaller +communities. The three-candidate automatic selector is qualification-only and +is not enabled by omitting this option. `--no-cluster` skips community +membership, analysis, labels, and `community-quality.json`. + ### `extract` Expose the full build surface: @@ -243,6 +250,12 @@ compass cluster-only [PATH] [--min-community-size=N] ``` +For a typed `compass.graph/1` input this uses the same fixed-resolution Leiden +profile as a normal build and atomically republishes graph-bound +`community-quality.json`. A schema-less legacy graph retains compatibility +Louvain behavior and does not publish quality evidence. The command never +interprets a missing older quality artifact as successful evidence. + ### `label` Generate/update semantic community labels: @@ -267,6 +280,9 @@ and in the bounded architecture report. It does not remove nodes, edges, or community assignments from the graph; omitted communities remain queryable and are included in the report's coverage disclosure. The default is `3`. +When labeling first reclusters a typed graph, its resolution behavior and +quality artifact are the same as `cluster-only`. + ## Read and query ### `query` diff --git a/docs/reference/outputs.md b/docs/reference/outputs.md index 40320e77d..3a2e3e597 100644 --- a/docs/reference/outputs.md +++ b/docs/reference/outputs.md @@ -17,6 +17,7 @@ compass-out/ ├── manifest.json ├── program.json # only with --program or --program-artifact ├── graph-overview.json # clustered builds +├── community-quality.json # clustered typed builds ├── cache/ # Compass-owned disposable cache layout ├── current-snapshot ├── snapshots// @@ -74,6 +75,7 @@ paths. | `program.json` (optional) | provenance-aware Program IR | program inspection, semantic analysis | | `GRAPH_REPORT.md` | derived human orientation | architecture survey | | `orientation.json` | versioned Agent Orientation bound to the same graph generation | coding assistants and MCP | +| `community-quality.json` | strict graph-bound community evidence | detector inspection, qualification, immutable history | | `graph.html` | derived optional visualization | interactive exploration | | `manifest.json` | incremental build state | next compatible update | | binary query caches | disposable acceleration | internal query loading | @@ -152,6 +154,27 @@ share an endpoint pair (ordered for directed graphs, unordered for undirected graphs), including repeated self-loops. Consumers do not need to request this promotion. +## `community-quality.json` + +Clustered typed builds publish schema `compass.community-quality/1`. The +artifact records the exact `graphGeneration` and SHA-256 `graphDigest`, the +algorithm/topology/quality/selector/seed/limits identity, the numeric limits, +partition metrics, per-community evidence, candidate summaries, bounded +witnesses, exact omissions, and `resultDigest`. + +Consumers must reject unknown schemas or fields and call the equivalent of +`validate_for_graph` against the selected canonical `graph.json`. A digest, +generation, or profile mismatch means the files are not one coherent artifact +set. `resultDigest` detects mutation of the quality payload itself. A missing +artifact is valid for an older graph, a schema-less legacy recluster, or a +`--no-cluster` build and means quality evidence is unavailable. + +Metrics form a vector rather than a pass/fail truth label. Modularity is +reported at the named evaluation resolution; conductance, connectedness, +largest-community fraction, singleton count, topology evidence mixes, and +witness omissions must be interpreted alongside it. Numeric community IDs are +local to this graph realization. + ### Inference levels Graph-building commands accept `--inference-level low|medium|high|max`. The diff --git a/scripts/qualify_code_graph_v1.sh b/scripts/qualify_code_graph_v1.sh index d7692d56f..801a07b4a 100755 --- a/scripts/qualify_code_graph_v1.sh +++ b/scripts/qualify_code_graph_v1.sh @@ -15,6 +15,7 @@ usage() { cat >&2 <] $0 --repositories [--local-repository ] EOF exit 2 @@ -23,11 +24,21 @@ EOF MODE= REPOSITORIES_MANIFEST= LOCAL_REPOSITORY= +COMMUNITY_REPORT= case "${1:-}" in --fixtures-only) MODE=fixtures shift ;; + --community-quality) + MODE=community-quality + shift + if [[ "${1:-}" == "--report" ]]; then + COMMUNITY_REPORT="${2:-}" + [[ -n "$COMMUNITY_REPORT" ]] || usage + shift 2 + fi + ;; --repositories) MODE=repositories REPOSITORIES_MANIFEST="${2:-}" @@ -45,6 +56,35 @@ case "${1:-}" in esac [[ "$#" -eq 0 ]] || usage +if [[ "$MODE" == community-quality ]]; then + cd "$QUALIFY_ROOT" + REPORT_A="$QUALIFY_TMP/community-quality-a.json" + REPORT_B="$QUALIFY_TMP/community-quality-b.json" + cargo run --quiet --locked -p compass-graph \ + --example community_quality_qualification >"$REPORT_A" + cargo run --quiet --locked -p compass-graph \ + --example community_quality_qualification >"$REPORT_B" + cmp "$REPORT_A" "$REPORT_B" + python3 - "$REPORT_A" <<'PY' +import json +import pathlib +import sys + +report = json.loads(pathlib.Path(sys.argv[1]).read_text()) +if report.get("schema") != "compass.community-quality-qualification/1": + raise SystemExit("unexpected community quality qualification schema") +failed = [name for name, passed in report.get("acceptance", {}).items() if passed is not True] +if failed: + raise SystemExit(f"community quality acceptance failed: {', '.join(failed)}") +print(json.dumps(report, sort_keys=True, separators=(",", ":"))) +PY + if [[ -n "$COMMUNITY_REPORT" ]]; then + mkdir -p "$(dirname "$COMMUNITY_REPORT")" + cp "$REPORT_A" "$COMMUNITY_REPORT" + fi + exit 0 +fi + [[ -f "$PARSER_ROOT/sources/language_definitions.json" && -d "$PARSER_ROOT/parsers" ]] || { echo "[code-graph-v1] offline qualification requires a pre-provisioned parser source bundle at $PARSER_ROOT (set TSLP_PARSER_SOURCE_DIR)" >&2 exit 1 From c376465124d7356da7ebdffb03aa0c36d6b1d3f5 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 12 Sep 2026 09:16:14 -0700 Subject: [PATCH 2/3] test(graph): rebaseline topology for Leiden --- tests/qualification/code-graph-v1-topology.json | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/qualification/code-graph-v1-topology.json b/tests/qualification/code-graph-v1-topology.json index 8af0850b4..81174513e 100644 --- a/tests/qualification/code-graph-v1-topology.json +++ b/tests/qualification/code-graph-v1-topology.json @@ -2,8 +2,9 @@ "schema": "compass.code-graph-topology-policy/1", "topology": { "minimums": { + "communities": 225, "edges": 1284, - "exactCrossCommunityEdges": 21, + "exactCrossCommunityEdges": 5, "exactCrossFileEdges": 81, "exactCrossFileEdgesPerThousandNodes": 63, "exactEdgeBearingNodePermille": 711, @@ -16,14 +17,14 @@ "uniqueTypedEndpointPairs": 1261 }, "maximums": { - "communities": 242, + "communities": 226, "connectedComponents": 222, "exactConnectedComponents": 496, "exactIsolatedNodes": 368, "exactSelfLoops": 0, "isolatedNodes": 96, "selfLoops": 0, - "singletonCommunities": 108 + "singletonCommunities": 96 }, "relationshipMinimums": { "calls": { From 8fb6c989511b87b43934bbb1f2a280335333f951 Mon Sep 17 00:00:00 2001 From: forhappy Date: Sat, 12 Sep 2026 09:18:59 -0700 Subject: [PATCH 3/3] ci: install cargo-audit from its lockfile --- .github/workflows/compass-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/compass-ci.yml b/.github/workflows/compass-ci.yml index d69941ac7..413bbd1e0 100644 --- a/.github/workflows/compass-ci.yml +++ b/.github/workflows/compass-ci.yml @@ -221,6 +221,8 @@ jobs: checks: write steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - name: Install locked cargo-audit + run: cargo install cargo-audit --version 0.22.2 --locked - name: Audit Rust dependencies uses: rustsec/audit-check@858dc40f52ca2b8570b7a997c1c4e35c6fc9a432 # Node 24 update with: