diff --git a/application/single_app/config.py b/application/single_app/config.py index cfaf7f49..97635269 100644 --- a/application/single_app/config.py +++ b/application/single_app/config.py @@ -96,7 +96,7 @@ EXECUTOR_TYPE = 'thread' EXECUTOR_MAX_WORKERS = 30 SESSION_TYPE = 'filesystem' -VERSION = "0.260.024" +VERSION = "0.260.025" IS_DEVELOPMENT = is_development_env_enabled() SESSION_COOKIE_SAMESITE = os.getenv('SESSION_COOKIE_SAMESITE', 'Lax') diff --git a/application/single_app/functions_mixed_source_orchestration.py b/application/single_app/functions_mixed_source_orchestration.py index 7d5dc68f..f2d4095a 100644 --- a/application/single_app/functions_mixed_source_orchestration.py +++ b/application/single_app/functions_mixed_source_orchestration.py @@ -412,7 +412,15 @@ def normalize_document_context_request( def should_run_tabular_evidence(user_question, has_narrative_sources=False): - """Return whether a mixed-source question needs tabular data or schema evidence.""" + """Return whether an in-scope tabular source should be computed for this question. + + Evidence gathering is additive. When an authorized tabular source is in scope the + tabular engine runs unless the question unambiguously targets a narrative artifact, + because deciding which evidence is relevant belongs to the synthesis step rather + than to this gate. Indexed tabular chunks carry only a truncated schema preview, so + skipping computation leaves the model with a handful of preview rows that can never + support a numeric conclusion. + """ normalized_question = " ".join(str(user_question or "").strip().lower().split()) if not normalized_question: return True @@ -430,10 +438,12 @@ def should_run_tabular_evidence(user_question, has_narrative_sources=False): "across the files", "across the documents", "across the sources", "mixed sources", ) - narrative_markers = ( + # Only artifact markers suppress computation. Topic words such as "report" or + # "policy" describe subject matter, not which engine can answer, and previously + # suppressed computation over spreadsheets that held the requested values. + narrative_artifact_markers = ( "pdf", "docx", "word document", "presentation", "powerpoint", - "paragraph", "section", "policy", "procedure", "contract", - "agreement", "memo", "letter", "narrative", "prose", "report", + "paragraph", "section", ) if any(marker in normalized_question for marker in tabular_markers): @@ -441,13 +451,9 @@ def should_run_tabular_evidence(user_question, has_narrative_sources=False): if any(marker in normalized_question for marker in collective_markers): return True if has_narrative_sources and any( - marker in normalized_question for marker in narrative_markers + marker in normalized_question for marker in narrative_artifact_markers ): return False - if normalized_question in {"summarize", "summary", "summarize the selected sources"}: - return True - if has_narrative_sources: - return False return True @@ -1413,7 +1419,14 @@ def execute_tabular_evidence_sources( source_kind=SOURCE_KIND_TABULAR, engine=EVIDENCE_ENGINE_TABULAR_TOOLS, status=EVIDENCE_STATUS_SKIPPED, - summary="Tabular processing was not needed for this narrative-only request.", + summary=( + "Tabular computation was not run for this source, so its full table was " + "never read. Any indexed excerpt from this source contains only a " + "truncated schema preview of the first few rows. Do not derive counts, " + "totals, averages, minimums, maximums, trends, or any other numeric " + "conclusion from those preview rows. Call the tabular analysis action if " + "values from this source are required." + ), coverage={ "selection_mode": normalized_selection_mode, "terminal": True, @@ -1921,6 +1934,11 @@ def build_mixed_source_evidence_handoff( "and tabular tool citations; do not convert computed table facts into unsupported narrative claims. " "When selection_mode is selected, current selected-source evidence supersedes prior document " "grounding; do not use prior source claims to fill missing current coverage. " + "This handoff is your starting evidence, not your only means of gathering evidence: if you have " + "actions available and this handoff does not contain what the question needs, call the appropriate " + "action to obtain it and reason over the handoff and the action results together before answering. " + "Never derive numeric conclusions from an indexed preview of a tabular source whose evidence status " + "is not completed; obtain those values from a computed tabular result instead. " f"{partial_coverage_instruction}\n\n{serialized_payload}" ), "mixed_source_coverage": coverage, diff --git a/application/single_app/route_backend_chats.py b/application/single_app/route_backend_chats.py index 13411322..b3e6fa77 100644 --- a/application/single_app/route_backend_chats.py +++ b/application/single_app/route_backend_chats.py @@ -4523,7 +4523,9 @@ def build_search_augmentation_system_prompt(retrieved_content): Retrieved Excerpts: {retrieved_content} - Base your answer only on information supported by the retrieved excerpts and any computed tool-backed results included elsewhere in this conversation context. If the answer is not supported by that information, say so. + These excerpts are your starting evidence, not your only means of gathering evidence. If you have actions or tools available and the excerpts do not contain what the question needs, call the appropriate action to obtain it, then reason over the retrieved excerpts and the action results together. Gather the evidence you are capable of gathering before declining to answer. + Ground every claim in a retrieved excerpt, in computed tool-backed results included elsewhere in this conversation context, or in a result you obtained by calling an action. Never estimate, infer, or fabricate values that none of those sources support; if the evidence is still missing after you have used the actions available to you, say so. + Excerpts drawn from a spreadsheet or other tabular source contain only a truncated schema preview of that file, not its data. Never derive counts, totals, averages, minimums, maximums, trends, or any other numeric conclusion from those preview rows; obtain such values from a computed tabular result instead. If computed tabular results are provided in another system message, treat them as authoritative for row-level values, calculations, and numeric conclusions. Do not say that you lack direct access to the data when those computed results are present. Example diff --git a/docs/explanation/fixes/AGENT_ACTIONS_WITH_WORKSPACE_EVIDENCE_FIX.md b/docs/explanation/fixes/AGENT_ACTIONS_WITH_WORKSPACE_EVIDENCE_FIX.md new file mode 100644 index 00000000..f93472be --- /dev/null +++ b/docs/explanation/fixes/AGENT_ACTIONS_WITH_WORKSPACE_EVIDENCE_FIX.md @@ -0,0 +1,175 @@ +# Agent Actions Ignored When Workspace Evidence Is Present + +**Fixed in version: 0.260.025** + +**Issue:** [#1332](https://github.com/microsoft/simplechat/issues/1332) + +**Related:** [#1021](https://github.com/microsoft/simplechat/issues/1021) — turn-level orchestration across chat capabilities, the strategic solution to this class of problem. This fix addresses the concrete symptom and does not close that initiative. + +## Issue + +Selecting an agent that has actions, enabling a workspace, and asking a specific +quantitative question produced an answer that: + +1. never invoked any of the agent's actions, and +2. reported numbers that were not actually present in the spreadsheet it cited. + +The reported case was a telemetry question against a workspace containing an +Excel file. Workspace search retrieved a narrative document plus the spreadsheet, +and the assistant answered from retrieved text alone, fabricating values. + +The turn behaved as "retrieval **or** actions" instead of "retrieval **and** +actions". Evidence gathering should be additive: gather everything the turn is +capable of gathering, then reason over the union and decide what is relevant. + +## Root Cause + +Three independent defects combined to produce the symptom. + +### 1. Tabular computation was suppressed by the presence of any narrative source + +`should_run_tabular_evidence()` in `functions_mixed_source_orchestration.py` was +a keyword heuristic that ended with a blanket rule: + +```python +if has_narrative_sources: + return False +``` + +A single PDF landing in the relevance results suppressed computation over an +authorized spreadsheet. The heuristic also treated topic words — `report`, +`policy`, `procedure`, `contract`, `agreement`, `memo`, `letter`, `narrative`, +`prose` — as evidence-type signals. Those words describe subject matter, not +which engine can answer a question, so they misfired frequently. + +When the gate returned `False`, `execute_tabular_evidence_sources(..., +execute=False)` emitted a `skipped` evidence envelope and the tabular engine +never ran. + +Note that when `enable_mixed_source_chat_search` is disabled, the legacy path in +`route_backend_chats.py` computes workspace tabular sources unconditionally. The +mixed-source path had regressed that behavior; this fix restores parity. + +### 2. Only a truncated preview of a spreadsheet is indexed + +`_build_tabular_schema_summary()` in `functions_documents.py` indexes a single +schema chunk holding at most `TABULAR_SCHEMA_SUMMARY_MAX_PREVIEW_ROWS` (3) rows +per sheet. This is intentional — the full file lives in blob storage and the +tabular engine reads it directly. + +However, when defect 1 skipped computation, that preview was still handed to the +model as ordinary retrieved text. The model then derived counts and averages from +three rows, which is the direct source of the incorrect values. + +The indexed chunk even ends with "This file is available for detailed analysis +via the Tabular Processing plugin" — the model read the advertisement for the +tool while being instructed not to use it. + +### 3. The retrieval augmentation prompt forbade using actions + +`build_search_augmentation_system_prompt()` in `route_backend_chats.py` +instructed: + +> Base your answer only on information supported by the retrieved excerpts and +> any computed tool-backed results included elsewhere in this conversation +> context. + +The mixed-source evidence handoff built by +`build_mixed_source_evidence_handoff()` was likewise a closed "synthesize one +answer" instruction. + +Agent actions were in fact available. Agents are constructed with +`FunctionChoiceBehavior.Auto()` in `semantic_kernel_loader.py`, and the agent is +invoked with the augmented history through `selected_agent.invoke_stream(...)`. +No code disables tools when documents are in scope. The model simply obeyed the +instruction not to look anywhere else, and the retrieved excerpts appeared +sufficient, so it never called an action. + +## Files Modified + +| File | Change | +|---|---| +| `application/single_app/functions_mixed_source_orchestration.py` | Inverted the tabular gate; narrowed narrative markers; rewrote the skipped-envelope summary; added action permission and a preview-row guard to the evidence handoff | +| `application/single_app/route_backend_chats.py` | Rewrote `build_search_augmentation_system_prompt()` | +| `application/single_app/config.py` | Version `0.260.024` -> `0.260.025` | +| `functional_tests/test_agent_actions_with_workspace_evidence.py` | New regression test | +| `functional_tests/test_mixed_source_chat_search_consistency.py` | Updated the gating contract a generic question now computes rather than skips | + +## Code Changes + +### Additive tabular gating + +`should_run_tabular_evidence()` now defaults to running. Computation is skipped +only when narrative sources are present **and** the question unambiguously names +a narrative artifact: + +```python +narrative_artifact_markers = ( + "pdf", "docx", "word document", "presentation", "powerpoint", + "paragraph", "section", +) +``` + +Topic words no longer suppress computation. Explicit tabular intent and +collective phrasing still short-circuit to `True`. + +### Skipped sources are now self-correcting + +A skipped tabular envelope previously read "Tabular processing was not needed for +this narrative-only request", which told the model the source was irrelevant. It +now states that the full table was never read, that any indexed excerpt is a +truncated preview, that numeric conclusions must not be drawn from it, and that +the tabular analysis action should be called if values are required. + +### Prompt contract permits and expects action use + +The retrieval augmentation prompt now frames excerpts as starting evidence rather +than the only permitted evidence, directs the model to call an available action +when the excerpts lack what the question needs, keeps a hard no-fabrication rule, +and forbids deriving any numeric conclusion from tabular preview rows. + +The mixed-source handoff carries the same permission plus a guard against +numeric conclusions drawn from an indexed preview of a source whose evidence +status is not `completed`. + +No new setting was introduced. These are correctness fixes and apply +unconditionally. + +## Validation + +```powershell +python .\functional_tests\test_agent_actions_with_workspace_evidence.py +python -m pytest .\functional_tests\test_mixed_source_manifest_contracts.py .\functional_tests\test_tabular_computed_results_prompt_priority.py -q +``` + +The new test asserts: + +- a quantitative question with narrative sources present now computes the + tabular source (the reported regression); +- an unambiguous narrative-artifact question still skips computation; +- topic words such as "report" no longer suppress computation; +- the skipped envelope warns against numeric conclusions from preview rows; +- the search prompt permits action invocation and no longer says "only"; +- the handoff instruction permits action invocation. + +Existing contracts in `test_tabular_computed_results_prompt_priority.py` and +`test_mixed_source_manifest_contracts.py` continue to pass unchanged. + +### Known unrelated failures + +Three tests in `test_mixed_source_chat_search_consistency.py` fail both before +and after this change, at identical assertions. Their harness builds a synthetic +namespace for `_execute_mixed_source_tabular_evidence` that is missing +`maybe_queue_search_tabular_generated_output`, so the stubbed tabular runner +raises and every source reports `failed`. A third failure originates in +`foundry_agent_runtime.py`. Both are pre-existing harness drift and are out of +scope for this fix. + +## Before / After + +| | Before | After | +|---|---|---| +| Spreadsheet + PDF in scope, quantitative question | Tabular engine skipped | Tabular engine runs | +| Model's view of a skipped spreadsheet | "not needed for this narrative-only request" | Explicit warning that the table was not read and the action should be called | +| Retrieved excerpts insufficient | Model answers from excerpts or declines | Model calls an available action, then reasons over both | +| Numbers from a 3-row preview | Permitted implicitly | Explicitly forbidden | diff --git a/docs/explanation/fixes/index.md b/docs/explanation/fixes/index.md index c8335252..a7c05cdc 100644 --- a/docs/explanation/fixes/index.md +++ b/docs/explanation/fixes/index.md @@ -39,3 +39,4 @@ category: Version History - [Generated Artifact Paging, Truncation, and Guidance Carry-Forward Fix](GENERATED_ARTIFACT_PAGING_AND_GUIDANCE_FIX.md) - [Admin Settings Pane Variable Scope Fix](ADMIN_SETTINGS_PANE_VARIABLE_SCOPE_FIX.md) - [Inline Media Cited-Only Gating Fix](INLINE_MEDIA_CITED_ONLY_GATING_FIX.md) +- [Agent Actions With Workspace Evidence Fix](AGENT_ACTIONS_WITH_WORKSPACE_EVIDENCE_FIX.md) diff --git a/docs/explanation/release-notes/index.md b/docs/explanation/release-notes/index.md index fe7ebbf1..c164f91e 100644 --- a/docs/explanation/release-notes/index.md +++ b/docs/explanation/release-notes/index.md @@ -20,6 +20,7 @@ This page includes the latest release notes inline. Older release sections are s | Version | Page | | --- | --- | +| v0.260.025 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.024 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.023 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.021 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | @@ -31,7 +32,7 @@ This page includes the latest release notes inline. Older release sections are s | v0.260.015 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.014 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | | v0.260.013 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | -| v0.260.012 | [Release notes index]({{ '/explanation/release_notes/' | relative_url }}) | +| v0.260.012 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | | v0.260.011 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | | v0.260.010 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | | v0.260.009 | [Release notes 0.260 series]({{ '/explanation/release-notes/v0.260/' | relative_url }}) | @@ -67,6 +68,25 @@ This page includes the latest release notes inline. Older release sections are s ## Latest release notes +### **(v0.260.025)** + +#### Bug Fixes + +* **Agent Actions Are No Longer Skipped When A Workspace Is In Scope** + * Selecting an agent that has actions and enabling a workspace produced answers that never invoked any of the agent's actions. The assistant answered from retrieved document text alone, even when the retrieved excerpts did not contain what the question asked for. + * The retrieval prompt instructed the model to base its answer *only* on the retrieved excerpts, so although the agent's actions were attached and available, the model was told not to reach for them. Retrieved excerpts are now framed as starting evidence, and the model is directed to call an available action when the excerpts lack what the question needs, then reason over the excerpts and the action results together. The rule against fabricating unsupported values is unchanged. + * (Ref: `build_search_augmentation_system_prompt`, `build_mixed_source_evidence_handoff`, agent actions, workspace search, [#1332](https://github.com/microsoft/simplechat/issues/1332)) + +* **Spreadsheets In A Workspace Are Now Actually Computed** + * A quantitative question about a spreadsheet could return values that were not in the file. Tabular computation was suppressed whenever workspace search also returned any narrative document, and the heuristic treated topic words such as "report", "policy", and "memo" as reasons to skip computation entirely. + * Because only a truncated three-row preview of a spreadsheet is indexed for search, skipping computation left the model deriving totals and averages from those preview rows. Tabular sources in scope are now computed unless the question unambiguously names a narrative artifact such as a PDF or presentation, restoring parity with the behavior already used when mixed-source search is disabled. + * (Ref: `should_run_tabular_evidence`, `functions_mixed_source_orchestration.py`, tabular processing, mixed-source evidence, [#1332](https://github.com/microsoft/simplechat/issues/1332)) + +* **A Skipped Spreadsheet Now Tells The Model What It Is Missing** + * When tabular computation is skipped, the evidence record previously said processing "was not needed", which implied the source was irrelevant and left the model free to compute from indexed preview rows. + * It now states that the full table was never read, that any indexed excerpt is a truncated preview, that numeric conclusions must not be drawn from it, and that the tabular analysis action should be called if values from that source are required. + * (Ref: `execute_tabular_evidence_sources`, evidence envelopes, tabular citations, [#1332](https://github.com/microsoft/simplechat/issues/1332)) + ### **(v0.260.024)** #### Bug Fixes @@ -311,23 +331,3 @@ This page includes the latest release notes inline. Older release sections are s * **Stale Tab Names In Latest Features** * Several Latest Features entries pointed readers at tabs by their old names after the settings moved. * (Ref: `latest-features` pane) - -### **(v0.260.012)** - -#### User Interface Enhancements - -* **New Data Lifecycle Group For Retention, Classification And Archiving** - * Retention policy, document classification and conversation archiving all decide how long content lives and how it is labelled, but they were split across Workspaces and Safety. They now sit together in a **Data Lifecycle** group with a tab each: **Retention**, **Classification** and **Archiving**. - * Conversation archiving in particular was buried under Safety, which described what it protects against rather than what it does. - * (Ref: navigation map, `retention-policy-section`, `document-classification-section`, `conversation-archiving-section`) - -* **Chat Group Gathers The Settings That Shape A Conversation** - * Settings that change what a conversation looks and behaves like were spread across AI Models, Workspaces and Safety. The **Chat** group now holds them in two tabs. - * **Chat Experience** collects model thought display, chat file uploads (with the conversation contents drawer) and workspace scope lock. - * **Feedback & Alerts** collects user feedback and desktop notifications, which are both about how the app talks back to the user rather than about safety enforcement. - * (Ref: `chat-experience`, `feedback-alerts`, `processing-thoughts-section`, `chat-file-uploads-section`, `workspace-scope-lock-section`, `user-feedback-section`, `desktop-notifications-section`) - -* **Settings Keep Their Values Through The Move** - * Cards were relocated between tabs without renaming a single field, so every saved value is preserved and the form submits exactly the payload it did before. - * Sidebar search still finds a setting by group, tab or card name, so you can reach anything without knowing where it now lives. - * (Ref: admin settings field contract, `admin_settings_nav.py`) diff --git a/docs/explanation/release-notes/v0.260.md b/docs/explanation/release-notes/v0.260.md index d9ac58f6..b364c885 100644 --- a/docs/explanation/release-notes/v0.260.md +++ b/docs/explanation/release-notes/v0.260.md @@ -1,6 +1,6 @@ --- title: "Release notes 0.260 series" -description: "SimpleChat release notes for 0.260.011 – 0.260.001." +description: "SimpleChat release notes for 0.260.012 – 0.260.001." section: "Reference" layout: page --- @@ -11,6 +11,26 @@ layout: page [Back to release notes index]({{ '/explanation/release_notes/' | relative_url }}) +### **(v0.260.012)** + +#### User Interface Enhancements + +* **New Data Lifecycle Group For Retention, Classification And Archiving** + * Retention policy, document classification and conversation archiving all decide how long content lives and how it is labelled, but they were split across Workspaces and Safety. They now sit together in a **Data Lifecycle** group with a tab each: **Retention**, **Classification** and **Archiving**. + * Conversation archiving in particular was buried under Safety, which described what it protects against rather than what it does. + * (Ref: navigation map, `retention-policy-section`, `document-classification-section`, `conversation-archiving-section`) + +* **Chat Group Gathers The Settings That Shape A Conversation** + * Settings that change what a conversation looks and behaves like were spread across AI Models, Workspaces and Safety. The **Chat** group now holds them in two tabs. + * **Chat Experience** collects model thought display, chat file uploads (with the conversation contents drawer) and workspace scope lock. + * **Feedback & Alerts** collects user feedback and desktop notifications, which are both about how the app talks back to the user rather than about safety enforcement. + * (Ref: `chat-experience`, `feedback-alerts`, `processing-thoughts-section`, `chat-file-uploads-section`, `workspace-scope-lock-section`, `user-feedback-section`, `desktop-notifications-section`) + +* **Settings Keep Their Values Through The Move** + * Cards were relocated between tabs without renaming a single field, so every saved value is preserved and the form submits exactly the payload it did before. + * Sidebar search still finds a setting by group, tab or card name, so you can reach anything without knowing where it now lives. + * (Ref: admin settings field contract, `admin_settings_nav.py`) + ### **(v0.260.011)** #### User Interface Enhancements diff --git a/docs/explanation/release_notes.md b/docs/explanation/release_notes.md index 5e349e02..5c27acd5 100644 --- a/docs/explanation/release_notes.md +++ b/docs/explanation/release_notes.md @@ -2,6 +2,25 @@ For feature-focused and fix-focused drill-downs by version, see [Features by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/features) and [Fixes by Version](https://github.com/microsoft/simplechat/tree/main/docs/explanation/fixes). +### **(v0.260.025)** + +#### Bug Fixes + +* **Agent Actions Are No Longer Skipped When A Workspace Is In Scope** + * Selecting an agent that has actions and enabling a workspace produced answers that never invoked any of the agent's actions. The assistant answered from retrieved document text alone, even when the retrieved excerpts did not contain what the question asked for. + * The retrieval prompt instructed the model to base its answer *only* on the retrieved excerpts, so although the agent's actions were attached and available, the model was told not to reach for them. Retrieved excerpts are now framed as starting evidence, and the model is directed to call an available action when the excerpts lack what the question needs, then reason over the excerpts and the action results together. The rule against fabricating unsupported values is unchanged. + * (Ref: `build_search_augmentation_system_prompt`, `build_mixed_source_evidence_handoff`, agent actions, workspace search, [#1332](https://github.com/microsoft/simplechat/issues/1332)) + +* **Spreadsheets In A Workspace Are Now Actually Computed** + * A quantitative question about a spreadsheet could return values that were not in the file. Tabular computation was suppressed whenever workspace search also returned any narrative document, and the heuristic treated topic words such as "report", "policy", and "memo" as reasons to skip computation entirely. + * Because only a truncated three-row preview of a spreadsheet is indexed for search, skipping computation left the model deriving totals and averages from those preview rows. Tabular sources in scope are now computed unless the question unambiguously names a narrative artifact such as a PDF or presentation, restoring parity with the behavior already used when mixed-source search is disabled. + * (Ref: `should_run_tabular_evidence`, `functions_mixed_source_orchestration.py`, tabular processing, mixed-source evidence, [#1332](https://github.com/microsoft/simplechat/issues/1332)) + +* **A Skipped Spreadsheet Now Tells The Model What It Is Missing** + * When tabular computation is skipped, the evidence record previously said processing "was not needed", which implied the source was irrelevant and left the model free to compute from indexed preview rows. + * It now states that the full table was never read, that any indexed excerpt is a truncated preview, that numeric conclusions must not be drawn from it, and that the tabular analysis action should be called if values from that source are required. + * (Ref: `execute_tabular_evidence_sources`, evidence envelopes, tabular citations, [#1332](https://github.com/microsoft/simplechat/issues/1332)) + ### **(v0.260.024)** #### Bug Fixes diff --git a/functional_tests/test_agent_actions_with_workspace_evidence.py b/functional_tests/test_agent_actions_with_workspace_evidence.py new file mode 100644 index 00000000..ba051060 --- /dev/null +++ b/functional_tests/test_agent_actions_with_workspace_evidence.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +# test_agent_actions_with_workspace_evidence.py +""" +Functional test for additive evidence gathering when a workspace is in scope. +Version: 0.260.025 +Implemented in: 0.260.025 + +This test ensures that selecting an agent with actions and enabling a workspace +no longer degrades into a retrieval-only turn. It validates that: + +1. An authorized tabular source is computed even when narrative sources are also + in scope, instead of being skipped by a keyword heuristic. +2. A skipped tabular source tells the model the full table was never read and + that preview rows cannot support numeric conclusions. +3. The retrieval augmentation prompt permits, rather than forbids, calling the + agent's actions when the retrieved excerpts are insufficient. +4. The mixed-source evidence handoff carries the same permission and the same + preview-row guard. + +Regression context: a question about battery telemetry with a spreadsheet in the +workspace returned fabricated numbers because the tabular engine was skipped and +the model was told to answer only from the 3-row indexed schema preview. +""" + +import ast +import os +import sys + +sys.path.append(os.path.dirname(os.path.abspath(__file__))) + +ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +APP_ROOT = os.path.join(ROOT_DIR, 'application', 'single_app') +sys.path.insert(0, APP_ROOT) + +from test_support.versioning import assert_app_version_at_least + +import functions_mixed_source_orchestration as orchestration + +ROUTE_FILE = os.path.join(APP_ROOT, 'route_backend_chats.py') +TARGET_FUNCTIONS = {'build_search_augmentation_system_prompt'} + + +def load_search_prompt_helper(): + """Load the retrieval augmentation prompt helper from the chat route source.""" + with open(ROUTE_FILE, 'r', encoding='utf-8') as file_handle: + route_content = file_handle.read() + + parsed = ast.parse(route_content, filename=ROUTE_FILE) + selected_nodes = [ + node for node in parsed.body + if isinstance(node, ast.FunctionDef) and node.name in TARGET_FUNCTIONS + ] + + module = ast.Module(body=selected_nodes, type_ignores=[]) + namespace = {} + exec(compile(module, ROUTE_FILE, 'exec'), namespace) + return namespace['build_search_augmentation_system_prompt'] + + +def test_tabular_evidence_runs_alongside_narrative_sources(): + """Narrative sources in scope must not suppress computing a tabular source.""" + print("🔍 Testing additive tabular evidence gating...") + + try: + assert_app_version_at_least("0.260.025") + + # The reported regression: a specific quantitative question that uses none + # of the hardcoded tabular keywords, asked while narrative sources are also + # in relevance scope. + assert orchestration.should_run_tabular_evidence( + "What was the battery telemetry during the descent?", + has_narrative_sources=True, + ) is True, "Quantitative question was skipped because narrative sources existed" + + # Topic words describe subject matter, not which engine can answer. + for topic_question in ( + "Give me a report on battery performance.", + "What does the policy say about battery thresholds?", + "Summarize the memo and the battery readings.", + ): + assert orchestration.should_run_tabular_evidence( + topic_question, + has_narrative_sources=True, + ) is True, f"Topic word suppressed tabular computation: {topic_question}" + + # An unambiguous narrative-artifact request still skips computation. + assert orchestration.should_run_tabular_evidence( + "What policy does the PDF state?", + has_narrative_sources=True, + ) is False, "Narrative-artifact request should still skip tabular computation" + + # Explicit tabular intent always computes, with or without narrative sources. + assert orchestration.should_run_tabular_evidence( + "Calculate the total and average from the spreadsheet.", + has_narrative_sources=True, + ) is True + assert orchestration.should_run_tabular_evidence( + "Summarize all selected documents.", + has_narrative_sources=True, + ) is True + + # With no narrative sources at all, computation is always appropriate. + assert orchestration.should_run_tabular_evidence( + "What does the PDF say?", + has_narrative_sources=False, + ) is True + + print("✅ Additive tabular evidence gating passed") + return True + + except Exception as exc: + print(f"❌ Test failed: {exc}") + import traceback + traceback.print_exc() + return False + + +def test_skipped_tabular_envelope_blocks_preview_row_math(): + """A skipped tabular source must warn against computing from preview rows.""" + print("🔍 Testing skipped tabular evidence envelope guidance...") + + try: + executor_calls = [] + envelopes = orchestration.execute_tabular_evidence_sources( + [{"document_id": "personal-xlsx"}], + lambda source: executor_calls.append(source), + "selected", + execute=False, + ) + + assert executor_calls == [], "Executor should not run when execution is skipped" + assert len(envelopes) == 1, envelopes + envelope = envelopes[0] + + assert envelope["status"] == orchestration.EVIDENCE_STATUS_SKIPPED + assert envelope["coverage"]["terminal"] is True + + summary = envelope["summary"] + assert "truncated schema preview" in summary, summary + assert "Do not derive counts" in summary, summary + assert "tabular analysis action" in summary, summary + assert "was not needed" not in summary, ( + "Skipped summary must not imply the source was irrelevant" + ) + + print("✅ Skipped tabular envelope guidance passed") + return True + + except Exception as exc: + print(f"❌ Test failed: {exc}") + import traceback + traceback.print_exc() + return False + + +def test_search_prompt_permits_action_invocation(): + """The retrieval prompt must not forbid the agent from calling its actions.""" + print("🔍 Testing retrieval augmentation prompt action permission...") + + try: + build_search_prompt = load_search_prompt_helper() + prompt = build_search_prompt('Excerpt A') + + # The closed-book instruction that suppressed action invocation. + assert 'Base your answer only on information supported by' not in prompt, prompt + + # Actions are now explicitly permitted and expected. + assert 'starting evidence, not your only means of gathering evidence' in prompt, prompt + assert 'call the appropriate action' in prompt, prompt + assert 'before declining to answer' in prompt, prompt + + # Anti-fabrication guarantee is preserved. + assert 'Never estimate, infer, or fabricate values' in prompt, prompt + + # Preview rows can never support numeric conclusions. + assert 'truncated schema preview' in prompt, prompt + assert 'Never derive counts' in prompt, prompt + + # Existing contracts asserted by + # test_tabular_computed_results_prompt_priority.py must still hold. + assert 'computed tool-backed results included elsewhere in this conversation context' in prompt, prompt + assert 'Do not say that you lack direct access to the data' in prompt, prompt + assert "If the answer isn't in the excerpts, say so." not in prompt, prompt + + print("✅ Retrieval augmentation prompt action permission passed") + return True + + except Exception as exc: + print(f"❌ Test failed: {exc}") + import traceback + traceback.print_exc() + return False + + +def test_mixed_source_handoff_permits_action_invocation(): + """The mixed-source handoff must permit actions and guard preview-row math.""" + print("🔍 Testing mixed-source handoff action permission...") + + try: + manifest = [{ + "document_id": "personal-xlsx", + "display_name": "battery_telemetry.xlsx", + "source_kind": orchestration.SOURCE_KIND_TABULAR, + "scope": orchestration.SOURCE_SCOPE_PERSONAL, + "authorization_status": orchestration.AUTHORIZATION_STATUS_AUTHORIZED, + }] + evidence_envelopes = orchestration.execute_tabular_evidence_sources( + [{"document_id": "personal-xlsx"}], + lambda source: None, + "selected", + execute=False, + ) + + handoff = orchestration.build_mixed_source_evidence_handoff( + manifest, + evidence_envelopes, + "selected", + ) + content = handoff["content"] + + assert 'starting evidence, not your only means of gathering evidence' in content, content + assert 'call the appropriate action' in content, content + assert 'Never derive numeric conclusions from an indexed preview' in content, content + + # Synthesis and citation contracts are preserved. + assert 'Synthesize one answer' in content, content + assert 'Preserve narrative source citations' in content, content + + print("✅ Mixed-source handoff action permission passed") + return True + + except Exception as exc: + print(f"❌ Test failed: {exc}") + import traceback + traceback.print_exc() + return False + + +if __name__ == "__main__": + original_log_event = orchestration.log_event + orchestration.log_event = lambda *args, **kwargs: None + + tests = [ + test_tabular_evidence_runs_alongside_narrative_sources, + test_skipped_tabular_envelope_blocks_preview_row_math, + test_search_prompt_permits_action_invocation, + test_mixed_source_handoff_permits_action_invocation, + ] + results = [] + + try: + for test in tests: + print(f"\n🧪 Running {test.__name__}...") + results.append(test()) + finally: + orchestration.log_event = original_log_event + + print(f"\n📊 Results: {sum(results)}/{len(results)} tests passed") + sys.exit(0 if all(results) else 1) diff --git a/functional_tests/test_mixed_source_chat_search_consistency.py b/functional_tests/test_mixed_source_chat_search_consistency.py index eba3c5aa..535831a1 100644 --- a/functional_tests/test_mixed_source_chat_search_consistency.py +++ b/functional_tests/test_mixed_source_chat_search_consistency.py @@ -2,8 +2,8 @@ # test_mixed_source_chat_search_consistency.py """ Functional test for mixed-source Chat and Search consistency. -Version: 0.250.064 -Implemented in: 0.250.064 +Version: 0.260.025 +Implemented in: 0.250.064; additive tabular evidence gating updated in 0.260.025 This test ensures Phase 2 of #1057 consumes the Phase 1 #1056 contracts for standard and streaming Chat plus workflow Search without implementing later @@ -253,8 +253,10 @@ def test_narrative_only_prompt_skips_rows_and_explicit_failure_is_partial(): gpt_model="test-model", settings={"tabular": True}, ) - assert runner_calls == [] - assert generic_narrative["evidence_envelopes"][0]["status"] == "skipped" + # Evidence gathering is additive: a generic question no longer suppresses + # computation of an in-scope tabular source. Only an unambiguous + # narrative-artifact request (the PDF case above) skips it. + assert generic_narrative["evidence_envelopes"][0]["status"] != "skipped" failing_executor, failure_calls = _load_shared_tabular_executor( failing_file_name="broken.csv"