diff --git a/.github/workflows/llm-benchmark-periodic.yml b/.github/workflows/llm-benchmark-periodic.yml index 7c1860282bb..e5f5b1e7c5f 100644 --- a/.github/workflows/llm-benchmark-periodic.yml +++ b/.github/workflows/llm-benchmark-periodic.yml @@ -39,6 +39,16 @@ on: description: 'Run benchmarks without uploading results' required: false default: 'false' + skip_task_catalog_upload: + description: 'Skip uploading the benchmark task catalog' + required: false + type: boolean + default: false + post_discord: + description: 'Post the analysis summary to Discord' + required: false + type: boolean + default: false permissions: contents: read @@ -166,7 +176,8 @@ jobs: LLM_BENCH_CONCURRENCY: "8" LLM_BENCH_RUST_CONCURRENCY: "4" LLM_BENCH_CSHARP_CONCURRENCY: "2" - LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }} + LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || matrix.lang == 'csharp' && '1' || '2' }} + LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports INPUT_LANGUAGE: ${{ matrix.lang }} INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }} INPUT_MODELS: ${{ inputs.models || '' }} @@ -174,6 +185,7 @@ jobs: INPUT_CATEGORIES: ${{ inputs.categories || '' }} INPUT_TASKS: ${{ inputs.tasks || '' }} INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }} + INPUT_SKIP_TASK_CATALOG_UPLOAD: ${{ inputs.skip_task_catalog_upload || 'false' }} run: | LANG="$INPUT_LANGUAGE" MODEL_SET="$INPUT_MODEL_SET" @@ -182,6 +194,7 @@ jobs: CATEGORIES="$INPUT_CATEGORIES" TASKS="$INPUT_TASKS" DRY_RUN="$INPUT_DRY_RUN" + SKIP_TASK_CATALOG_UPLOAD="$INPUT_SKIP_TASK_CATALOG_UPLOAD" case "$MODEL_SET" in website_active) @@ -219,6 +232,9 @@ jobs: if [ "$DRY_RUN" = "true" ]; then EXTRA_ARGS+=(--dry-run) fi + if [ "$SKIP_TASK_CATALOG_UPLOAD" = "true" ]; then + EXTRA_ARGS+=(--skip-task-catalog-upload) + fi if [ "$MODEL_SET" = "website_active" ]; then llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}" @@ -227,3 +243,81 @@ jobs: else llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}" fi + + - name: Upload analysis reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: llm-benchmark-analysis-${{ matrix.lang }} + path: ${{ runner.temp }}/llm-benchmark-reports + if-no-files-found: ignore + retention-days: 14 + + summarize: + name: Summarize benchmark analysis + if: always() + needs: run-benchmarks + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Test Discord formatter + run: python3 tools/xtask-llm-benchmark/scripts/test_discord_summary.py + + - name: Download analysis reports + continue-on-error: true + uses: actions/download-artifact@v4 + with: + pattern: llm-benchmark-analysis-* + path: reports + merge-multiple: true + + - name: Publish workflow summary + run: | + echo "# LLM benchmark analysis" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + + if [ ! -d reports ] || ! find reports -type f -name '*.md' -print -quit | grep -q .; then + echo "No analysis reports were produced." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + + echo "Full per-model reports are available in the workflow artifacts." >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + + find reports -type f -name '*.md' -print | sort | while IFS= read -r report; do + awk '/^## Failure patterns/{exit} NR <= 80 {print}' "$report" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + echo "---" >> "$GITHUB_STEP_SUMMARY" + echo >> "$GITHUB_STEP_SUMMARY" + done + + - name: Prepare Discord summary + if: github.event_name == 'schedule' || inputs.post_discord + env: + RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} + RUN_LABEL: ${{ github.event_name == 'schedule' && 'Weekly scheduled run' || 'Manual run' }} + run: > + python3 tools/xtask-llm-benchmark/scripts/discord_summary.py + --reports-dir reports + --run-url "$RUN_URL" + --run-label "$RUN_LABEL" + --output discord-payload.json + + - name: Post Discord summary + if: github.event_name == 'schedule' || inputs.post_discord + continue-on-error: true + env: + DISCORD_WEBHOOK_URL: ${{ secrets.LLM_BENCHMARK_DISCORD_WEBHOOK_URL }} + run: | + if [ -z "$DISCORD_WEBHOOK_URL" ]; then + echo "::warning::LLM_BENCHMARK_DISCORD_WEBHOOK_URL is not configured" + exit 0 + fi + + curl --fail --silent --show-error \ + --header 'Content-Type: application/json' \ + --data-binary @discord-payload.json \ + "$DISCORD_WEBHOOK_URL" diff --git a/tools/xtask-llm-benchmark/scripts/discord_summary.py b/tools/xtask-llm-benchmark/scripts/discord_summary.py new file mode 100644 index 00000000000..c76f418f73a --- /dev/null +++ b/tools/xtask-llm-benchmark/scripts/discord_summary.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 + +import argparse +import json +import re +from collections import defaultdict +from dataclasses import dataclass +from pathlib import Path + +GREEN = 0x57F287 +YELLOW = 0xFEE75C +RED = 0xED4245 +MAX_ITEMS = 5 +MAX_FIELD_LENGTH = 1000 + + +@dataclass +class Report: + language: str + mode: str + model: str + passed_tasks: int + total_tasks: int + actions: list[str] + other_findings: list[str] + + +def field(text: str, name: str) -> str: + match = re.search(rf"^- {re.escape(name)}: (.+)$", text, re.MULTILINE) + return match.group(1) if match else "unknown" + + +def language_name(language: str) -> str: + return { + "csharp": "C#", + "rust": "Rust", + "typescript": "TypeScript", + }.get(language, language) + + +def parse_report(text: str) -> Report: + language = field(text, "Language") + mode = field(text, "Mode") + model = field(text, "Model") + task_match = re.match(r"(\d+)/(\d+)", field(text, "Tasks")) + passed_tasks, total_tasks = map(int, task_match.groups()) if task_match else (0, 0) + + actions = [] + recommended = text.partition("## Recommended actions")[2].partition("## Failure patterns")[0] + for line in recommended.splitlines(): + if line.startswith("- **["): + actions.append(f"- {language_name(language)} / {mode}: {line[2:]}") + + other_findings = [] + failures = text.partition("## Failure patterns")[2] + for section in re.split(r"^### ", failures, flags=re.MULTILINE)[1:]: + heading = section.splitlines()[0] + title_match = re.match(r"(.+) \(\d+ tasks?\)$", heading) + title = title_match.group(1) if title_match else heading + classification = re.search(r"^- \*\*Classification:\*\* (.+)$", section, re.MULTILINE) + if classification and classification.group(1) in { + "Model limitation", + "Infrastructure/provider problem", + "No action", + }: + other_findings.append( + f"- {language_name(language)} / {mode} / {model}: {title} - {classification.group(1)}" + ) + + return Report(language, mode, model, passed_tasks, total_tasks, actions, other_findings) + + +def load_reports(reports_dir: Path) -> list[Report]: + return [ + parse_report(path.read_text(encoding="utf-8")) + for path in sorted(reports_dir.rglob("*.md")) + ] + + +def rate(passed: int, total: int) -> str: + percent = passed * 100 / total if total else 0 + return f"{passed}/{total} ({percent:.1f}%)" + + +def field_value(lines: list[str], empty: str, overflow_label: str) -> str: + unique = list(dict.fromkeys(lines)) + visible = unique[:MAX_ITEMS] or [empty] + if len(unique) > MAX_ITEMS: + visible.append(f"- ...and {len(unique) - MAX_ITEMS} more {overflow_label}(s)") + value = "\n".join(visible) + if len(value) > MAX_FIELD_LENGTH: + return value[: MAX_FIELD_LENGTH - 30].rstrip() + "\n... View the full analysis." + return value + + +def build_payload(reports: list[Report], run_url: str, run_label: str) -> dict: + totals = [0, 0] + by_language: dict[str, list[int]] = defaultdict(lambda: [0, 0]) + actions = [] + other_findings = [] + + for report in reports: + totals[0] += report.passed_tasks + totals[1] += report.total_tasks + by_language[report.language][0] += report.passed_tasks + by_language[report.language][1] += report.total_tasks + actions.extend(report.actions) + other_findings.extend(report.other_findings) + + passed, total = totals + pass_percent = passed * 100 / total if total else 0 + has_infrastructure_failure = any("Infrastructure/provider problem" in item for item in other_findings) + if not total or pass_percent < 90 or has_infrastructure_failure: + color = RED + elif actions or pass_percent < 95: + color = YELLOW + else: + color = GREEN + + language_rates = [ + f"- **{language_name(language)}:** {rate(*counts)}" + for language, counts in sorted(by_language.items()) + ] + return { + "username": "SpacetimeDB LLM Benchmarks", + "allowed_mentions": {"parse": []}, + "embeds": [ + { + "title": "LLM Benchmark Analysis", + "url": run_url, + "description": f"**{rate(*totals)}** task runs passed", + "color": color, + "fields": [ + { + "name": "By language", + "value": field_value( + language_rates, + "No analysis reports were produced.", + "language", + ), + "inline": False, + }, + { + "name": "Action items", + "value": field_value(actions, "None", "action"), + "inline": False, + }, + { + "name": "Other failures", + "value": field_value(other_findings, "None", "finding"), + "inline": False, + }, + ], + "footer": {"text": run_label}, + } + ], + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Build a Discord embed from LLM benchmark analysis reports.") + parser.add_argument("--reports-dir", type=Path, required=True) + parser.add_argument("--run-url", required=True) + parser.add_argument("--run-label", required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + + payload = build_payload(load_reports(args.reports_dir), args.run_url, args.run_label) + args.output.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/tools/xtask-llm-benchmark/scripts/test_discord_summary.py b/tools/xtask-llm-benchmark/scripts/test_discord_summary.py new file mode 100644 index 00000000000..bf1137aa80a --- /dev/null +++ b/tools/xtask-llm-benchmark/scripts/test_discord_summary.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 + +import tempfile +import unittest +from pathlib import Path + +from discord_summary import GREEN, RED, YELLOW, build_payload, field_value, load_reports + + +def report( + *, + language: str = "csharp", + mode: str = "guidelines", + model: str = "test-model", + tasks: str = "36/37 (97.3%)", + action: str | None = None, + pattern: str | None = None, + classification: str = "Model limitation", +) -> str: + recommended = action or "No repository changes recommended." + failures = ( + f"### {pattern} (1 task)\n\n- **Classification:** {classification}\n" + if pattern + else "No failures detected.\n" + ) + return f"""# LLM Benchmark Analysis + +- Language: {language} +- Mode: {mode} +- Model: {model} +- Tasks: {tasks} +- Scorers: 100/100 (100.0%) + +## Recommended actions + +{recommended} + +## Failure patterns + +{failures}""" + + +class DiscordSummaryTests(unittest.TestCase): + def load(self, *contents: str): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + for index, content in enumerate(contents): + (root / f"report-{index}.md").write_text(content, encoding="utf-8") + return load_reports(root) + + def test_aggregates_rates_and_formats_findings(self): + reports = self.load( + report(tasks="36/37 (97.3%)"), + report( + language="rust", + tasks="34/37 (91.9%)", + pattern="Incorrect sum-type syntax", + ), + ) + + embed = build_payload(reports, "https://example.com/run", "Weekly run")["embeds"][0] + + self.assertEqual(embed["description"], "**70/74 (94.6%)** task runs passed") + self.assertEqual(embed["color"], YELLOW) + self.assertIn("**C#:** 36/37 (97.3%)", embed["fields"][0]["value"]) + self.assertIn("Rust / guidelines / test-model", embed["fields"][2]["value"]) + + def test_action_items_force_review_status(self): + reports = self.load( + report( + action="- **[Skill problem | High] Clarify transactions** — Update the skill. Evidence: t_075.", + ) + ) + + embed = build_payload(reports, "https://example.com/run", "Weekly run")["embeds"][0] + + self.assertEqual(embed["color"], YELLOW) + self.assertIn("C# / guidelines", embed["fields"][1]["value"]) + + def test_healthy_and_infrastructure_colors(self): + healthy = build_payload( + self.load(report(tasks="37/37 (100.0%)")), + "https://example.com/run", + "Weekly run", + )["embeds"][0] + infrastructure = build_payload( + self.load( + report( + tasks="37/37 (100.0%)", + pattern="Provider timeout", + classification="Infrastructure/provider problem", + ) + ), + "https://example.com/run", + "Weekly run", + )["embeds"][0] + + self.assertEqual(healthy["color"], GREEN) + self.assertEqual(infrastructure["color"], RED) + + def test_empty_and_long_fields_stay_valid(self): + payload = build_payload([], "https://example.com/run", "Manual run") + empty = payload["embeds"][0] + long_value = field_value([f"- {index} {'x' * 400}" for index in range(6)], "None", "finding") + + self.assertEqual(payload["allowed_mentions"], {"parse": []}) + self.assertEqual(empty["color"], RED) + self.assertIn("No analysis reports", empty["fields"][0]["value"]) + self.assertLessEqual(len(long_value), 1000) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/xtask-llm-benchmark/src/bench/analysis.rs b/tools/xtask-llm-benchmark/src/bench/analysis.rs index cb23fbb6cf5..2b2f014b772 100644 --- a/tools/xtask-llm-benchmark/src/bench/analysis.rs +++ b/tools/xtask-llm-benchmark/src/bench/analysis.rs @@ -7,11 +7,14 @@ use anyhow::Result; use spacetimedb_data_structures::map::HashMap; use std::path::Path; +const MAX_CONTEXT_CHARS: usize = 20_000; + pub async fn run_analysis( outcomes: &[RunOutcome], lang: &str, mode: &str, model_name: &str, + context: &str, bench_root: &Path, llm: &dyn LlmProvider, ) -> Result> { @@ -24,7 +27,7 @@ pub async fn run_analysis( return Ok(None); } - let prompt = build_prompt(lang, mode, model_name, bench_root, &failures); + let prompt = build_prompt(lang, mode, model_name, context, bench_root, &failures); let route = ModelRoute::new( "gpt-5.4-mini", @@ -49,9 +52,10 @@ pub fn system_prompt() -> String { } pub const SYSTEM_PROMPT: &str = "\ -You summarize LLM benchmark failures for SpacetimeDB into structured markdown. \ +You turn LLM benchmark failures for SpacetimeDB into evidence-based, structured markdown. \ Each failure includes the model's generated code, the scorer error, and the golden (correct) answer when available. \ -Write in third person for a public benchmark page. Do not address the reader."; +Write in third person for a public benchmark page. Do not address the reader. \ +Recommend repository changes only when the supplied evidence supports them."; fn context_description(mode: &str) -> &'static str { match mode { @@ -103,7 +107,14 @@ fn read_golden(bench_root: &Path, task_id: &str, lang: &str) -> Option { None } -pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, failures: &[&RunOutcome]) -> String { +pub fn build_prompt( + lang: &str, + mode: &str, + model_name: &str, + context: &str, + bench_root: &Path, + failures: &[&RunOutcome], +) -> String { let lang_display = match lang { "rust" => "Rust", "csharp" => "C#", @@ -118,6 +129,16 @@ pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, count = failures.len(), ); + let context_is_complete = context.chars().count() <= MAX_CONTEXT_CHARS; + if has_context(mode) { + prompt.push_str("### Context supplied to the model\n\n"); + prompt.push_str(&format!("```\n{}\n```\n", truncate(context, MAX_CONTEXT_CHARS))); + if !context_is_complete { + prompt.push_str("The context excerpt was truncated. Treat context-gap conclusions as low confidence.\n"); + } + prompt.push('\n'); + } + for f in failures.iter().take(15) { prompt.push_str(&format!("### {} ({}/{})\n", f.task, f.passed_tests, f.total_tests)); @@ -141,40 +162,177 @@ pub fn build_prompt(lang: &str, mode: &str, model_name: &str, bench_root: &Path, prompt.push_str(&format!("({} more failures not shown)\n\n", failures.len() - 15)); } - prompt.push_str(&analysis_instructions(mode)); + prompt.push_str(&analysis_instructions_with_context(mode, context_is_complete)); prompt } pub fn analysis_instructions(mode: &str) -> String { - let fix_line = if has_context(mode) { + analysis_instructions_with_context(mode, false) +} + +fn analysis_instructions_with_context(mode: &str, context_is_complete: bool) -> String { + let context_gap_line = if has_context(mode) { let name = context_name(mode); - format!("5. **{name} gap:** What's missing or unclear in the {name} that led to this mistake\n") + if context_is_complete { + format!("- **{name} gap:** What is missing or unclear in the {name}, or `None`\n") + } else { + format!("- **{name} gap:** `Needs manual review`; the complete {name} was not supplied\n") + } } else { String::new() }; + let context_rule = if has_context(mode) && !context_is_complete { + "- Because the complete context was not supplied, do not classify a group as `Skill problem` or `Documentation problem`.\n" + } else { + "" + }; format!( "\ --- -Group failures by root cause pattern. Use this exact structure for each group: +Begin with this section: + +## Recommended actions + +List at most five distinct, repository-owned actions supported by the evidence. Use: + +- **[Classification | Confidence] Short title** — Plain-language action. Evidence: task IDs. + +If the failures are isolated model mistakes, provider failures, or otherwise do not justify a repository change, write: +`No repository changes recommended.` + +Then group failures by root cause using this exact structure: + +## Failure patterns ### [Pattern Name] (N tasks) -1. **What the model wrote:** Show the relevant incorrect lines from the generated code -2. **What was expected:** Show the relevant lines from the golden answer -3. **What the error says:** Quote the scorer error that identifies the problem -4. **Why this happened:** Why the model likely made this mistake (e.g. confused with another framework, hallucinated API, singular vs plural naming) -5. **Affected tasks:** list of task IDs -{fix_line} +- **Classification:** One of `Eval problem`, `Skill problem`, `Documentation problem`, `API/ergonomics problem`, `Model limitation`, `Infrastructure/provider problem`, or `No action` +- **Confidence:** `High`, `Medium`, or `Low` +- **What the model wrote:** Relevant incorrect lines from the generated code +- **What was expected:** Relevant lines from the golden answer +- **What the error says:** The scorer error that identifies the problem +- **Why this happened:** The likely root cause +- **Suggested action:** A plain-language repository change, or `None` +- **Suggested area:** The likely repository area, or `None` +- **Affected tasks:** Task IDs +{context_gap_line} Rules: - Group tasks that fail for the same reason. Do not repeat the same analysis per task. - Show only the relevant lines, not entire files. - Skip provider errors (timeouts, 429s) with a brief note. +- Do not recommend changing an eval, skill, documentation, or API merely because one model made an isolated mistake. +- Do not invent evidence, repository paths, or implementation details that were not supplied. +- Prefer `Model limitation`, `Infrastructure/provider problem`, or `No action` when the evidence does not support a repository change. +- Classify a context gap as `Skill problem` or `Documentation problem` only when the supplied context directly supports that conclusion. +- `Eval problem`, `Skill problem`, `Documentation problem`, and `API/ergonomics problem` require a non-`None` suggested action and a matching entry under `Recommended actions`. +- When `Suggested action` is `None`, use `Model limitation`, `Infrastructure/provider problem`, or `No action`. +- Write `No repository changes recommended.` only when no failure group recommends a repository change. +{context_rule}\ " ) } +pub fn build_report( + outcomes: &[RunOutcome], + lang: &str, + mode: &str, + model_name: &str, + analysis: Option<&str>, +) -> String { + let passed_tasks = outcomes + .iter() + .filter(|outcome| outcome.total_tests > 0 && outcome.passed_tests == outcome.total_tests) + .count(); + let total_tasks = outcomes.len(); + let passed_scorers: u32 = outcomes.iter().map(|outcome| outcome.passed_tests).sum(); + let total_scorers: u32 = outcomes.iter().map(|outcome| outcome.total_tests).sum(); + let failures_without_output: Vec<&str> = outcomes + .iter() + .filter(|outcome| outcome.passed_tests < outcome.total_tests && outcome.llm_output.is_none()) + .map(|outcome| outcome.task.as_str()) + .collect(); + let unavailable_output_section = if failures_without_output.is_empty() { + None + } else { + let task_count = failures_without_output.len(); + let task_label = if task_count == 1 { "task" } else { "tasks" }; + let tasks = failures_without_output + .iter() + .map(|task| format!("`{task}`")) + .collect::>() + .join(", "); + Some(format!( + "\ +### Model output unavailable ({task_count} {task_label}) + +- **Classification:** Infrastructure/provider problem +- **Tasks:** {tasks} +- **What happened:** The model request failed before producing output, so source-level failure analysis was not possible. +- **Suggested action:** Retry the affected tasks and inspect the benchmark logs if the failure persists." + )) + }; + + let mut report = format!( + "\ +# LLM Benchmark Analysis + +- Language: {lang} +- Mode: {mode} +- Model: {model_name} +- Tasks: {passed_tasks}/{total_tasks} ({task_percent:.1}%) +- Scorers: {passed_scorers}/{total_scorers} ({scorer_percent:.1}%) + +", + task_percent = percent(passed_tasks as u32, total_tasks as u32), + scorer_percent = percent(passed_scorers, total_scorers), + ); + + match analysis { + Some(analysis) => { + report.push_str(analysis.trim()); + if let Some(section) = unavailable_output_section { + report.push_str("\n\n"); + report.push_str(§ion); + } + } + None if unavailable_output_section.is_some() => { + report.push_str(&format!( + "\ +## Recommended actions + +No repository changes recommended. + +## Failure patterns + +{}", + unavailable_output_section.unwrap() + )); + } + None => report.push_str( + "\ +## Recommended actions + +No repository changes recommended. + +## Failure patterns + +No failures detected.", + ), + } + report.push('\n'); + report +} + +fn percent(passed: u32, total: u32) -> f64 { + if total == 0 { + 0.0 + } else { + f64::from(passed) * 100.0 / f64::from(total) + } +} + fn extract_reasons(details: &HashMap) -> Vec { details .iter() @@ -192,3 +350,137 @@ fn truncate(s: &str, max: usize) -> &str { None => s, } } + +#[cfg(test)] +mod tests { + use super::*; + + fn outcome(task: &str, passed_tests: u32, total_tests: u32) -> RunOutcome { + RunOutcome { + hash: String::new(), + task: task.to_string(), + lang: "typescript".to_string(), + golden_published: true, + model_name: "test-model".to_string(), + total_tests, + passed_tests, + llm_output: None, + category: None, + route_api_model: None, + golden_db: None, + llm_db: None, + work_dir_golden: None, + work_dir_llm: None, + scorer_details: None, + vendor: String::new(), + input_tokens: None, + output_tokens: None, + generation_duration_ms: None, + started_at: None, + finished_at: None, + } + } + + #[test] + fn instructions_request_actionable_evidence_based_output() { + let instructions = analysis_instructions("guidelines"); + + assert!(instructions.contains("## Recommended actions")); + assert!(instructions.contains("Eval problem")); + assert!(instructions.contains("Skill problem")); + assert!(instructions.contains("Model limitation")); + assert!(instructions.contains("Do not invent evidence")); + assert!(instructions.contains("AI guidelines gap")); + assert!(instructions.contains("Needs manual review")); + assert!(instructions.contains("do not classify a group as `Skill problem` or `Documentation problem`")); + } + + #[test] + fn no_context_does_not_request_a_context_gap() { + let instructions = analysis_instructions("no_context"); + + assert!(!instructions.contains("context gap:")); + assert!(!instructions.contains("guidelines gap:")); + } + + #[test] + fn report_includes_task_and_scorer_pass_rates() { + let outcomes = vec![outcome("t_001", 3, 3), outcome("t_002", 1, 2), outcome("t_003", 0, 0)]; + let report = build_report( + &outcomes, + "typescript", + "guidelines", + "test-model", + Some("## Recommended actions\n\n- Fix the skill."), + ); + + assert!(report.contains("- Tasks: 1/3 (33.3%)")); + assert!(report.contains("- Scorers: 4/5 (80.0%)")); + assert!(report.contains("- Fix the skill.")); + } + + #[test] + fn live_prompt_supplies_context_for_gap_analysis() { + let failed = outcome("t_001", 0, 1); + let prompt = build_prompt( + "typescript", + "guidelines", + "test-model", + "Use transactions for procedure writes.", + Path::new("missing-benchmark-root"), + &[&failed], + ); + + assert!(prompt.contains("### Context supplied to the model")); + assert!(prompt.contains("Use transactions for procedure writes.")); + assert!(prompt.contains("What is missing or unclear in the AI guidelines")); + assert!(!prompt.contains("complete AI guidelines was not supplied")); + } + + #[test] + fn passing_report_recommends_no_changes() { + let report = build_report( + &[outcome("t_001", 3, 3)], + "typescript", + "guidelines", + "test-model", + None, + ); + + assert!(report.contains("No repository changes recommended.")); + assert!(report.contains("No failures detected.")); + } + + #[test] + fn failed_request_without_output_is_reported_as_infrastructure() { + let report = build_report( + &[outcome("t_001", 0, 1)], + "typescript", + "guidelines", + "test-model", + None, + ); + + assert!(report.contains("Model output unavailable (1 task)")); + assert!(report.contains("Infrastructure/provider problem")); + assert!(report.contains("`t_001`")); + assert!(!report.contains("No failures detected.")); + } + + #[test] + fn failed_request_without_output_is_added_to_model_analysis() { + let mut generated_failure = outcome("t_001", 0, 1); + generated_failure.llm_output = Some("generated source".to_string()); + let report = build_report( + &[generated_failure, outcome("t_002", 0, 1)], + "typescript", + "guidelines", + "test-model", + Some("## Recommended actions\n\nNo repository changes recommended.\n\n## Failure patterns\n\n### Invalid API"), + ); + + assert!(report.contains("### Invalid API")); + assert!(report.contains("Model output unavailable (1 task)")); + assert!(report.contains("`t_002`")); + } +} diff --git a/tools/xtask-llm-benchmark/src/bench/runner.rs b/tools/xtask-llm-benchmark/src/bench/runner.rs index fc6da3884af..f993f80c883 100644 --- a/tools/xtask-llm-benchmark/src/bench/runner.rs +++ b/tools/xtask-llm-benchmark/src/bench/runner.rs @@ -522,12 +522,28 @@ fn dry_run_analysis_path(run_id: &str, lang_name: &str, mode: &str, route: &Mode .join(format!("run-{run_id}-{lang_name}-{mode}-{route_tag}-analysis.md")) } +fn configured_analysis_path(lang_name: &str, mode: &str, route: &ModelRoute) -> Option { + let report_dir = std::env::var_os("LLM_BENCHMARK_REPORT_DIR").filter(|value| !value.is_empty())?; + let route_tag = sanitize_db_name(&route.display_name); + Some(PathBuf::from(report_dir).join(format!("{lang_name}-{mode}-{route_tag}.md"))) +} + +fn write_analysis_report( + path: &Path, + cfg: &BenchRunContext<'_>, + outcomes: &[RunOutcome], + analysis: Option<&str>, +) -> Result<()> { + fs::create_dir_all(path.parent().unwrap_or_else(|| Path::new(".")))?; + let report = + crate::bench::analysis::build_report(outcomes, cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, analysis); + fs::write(path, report)?; + Ok(()) +} + async fn maybe_generate_analysis(cfg: &BenchRunContext<'_>, outcomes: &[RunOutcome]) -> Result> { - let should_run = if cfg.dry_run { - cfg.local_analysis - } else { - cfg.api_client.is_some() - }; + let configured_path = configured_analysis_path(cfg.lang.as_str(), cfg.mode, cfg.route); + let should_run = cfg.local_analysis || configured_path.is_some() || (!cfg.dry_run && cfg.api_client.is_some()); if !should_run { return Ok(None); @@ -538,27 +554,30 @@ async fn maybe_generate_analysis(cfg: &BenchRunContext<'_>, outcomes: &[RunOutco cfg.lang.as_str(), cfg.mode, &cfg.route.display_name, + cfg.context, cfg.bench_root, cfg.llm, ) .await?; if cfg.dry_run - && let (Some(text), Some(run_id)) = (analysis.as_deref(), cfg.dry_run_id.as_deref()) + && let Some(run_id) = cfg.dry_run_id.as_deref() { let path = dry_run_analysis_path(run_id, cfg.lang.as_str(), cfg.mode, cfg.route); - let _ = fs::create_dir_all(path.parent().unwrap_or_else(|| Path::new("."))); - let contents = format!( - "# Local Benchmark Analysis\n\n- Lang: {}\n- Mode: {}\n- Model: {}\n\n{}", - cfg.lang.as_str(), - cfg.mode, - cfg.route.display_name, - text - ); - match fs::write(&path, contents) { + match write_analysis_report(&path, cfg, outcomes, analysis.as_deref()) { Ok(()) => println!("Local analysis: {}", path.display()), Err(e) => eprintln!("[warn] failed to write local analysis: {e}"), } + } + + if let Some(path) = configured_path { + match write_analysis_report(&path, cfg, outcomes, analysis.as_deref()) { + Ok(()) => println!("Analysis report: {}", path.display()), + Err(e) => eprintln!("[warn] failed to write analysis report: {e}"), + } + } + + if cfg.dry_run { return Ok(None); } diff --git a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs index be76423286d..fae506b4813 100644 --- a/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs +++ b/tools/xtask-llm-benchmark/src/bin/llm_benchmark.rs @@ -21,7 +21,9 @@ use xtask_llm_benchmark::context::constants::ALL_MODES; use xtask_llm_benchmark::context::{build_context, compute_processed_context_hash}; use xtask_llm_benchmark::eval::Lang; use xtask_llm_benchmark::llm::types::Vendor; -use xtask_llm_benchmark::llm::{default_model_routes, make_provider_from_env, LlmProvider, ModelRoute}; +use xtask_llm_benchmark::llm::{ + default_model_routes, make_provider_from_env, LlmProvider, ModelRoute, ReasoningEffort, +}; #[derive(Clone, Debug)] struct ModelGroup { @@ -68,6 +70,10 @@ impl std::str::FromStr for ModelGroup { after_help = "Notes:\n • Anthropic ids: claude-sonnet-4-5, claude-sonnet-4, claude-3-7-sonnet-latest, claude-3-5-sonnet-latest\n • Base URLs must not include /v1; models must be valid for the chosen provider.\n" )] struct Cli { + /// Reasoning effort used for every model request + #[arg(long, value_enum, default_value_t = ReasoningEffort::Medium, global = true)] + reasoning: ReasoningEffort, + #[command(subcommand)] command: Commands, } @@ -139,6 +145,10 @@ struct RunArgs { #[arg(long)] dry_run: bool, + /// Skip uploading the benchmark task catalog + #[arg(long)] + skip_task_catalog_upload: bool, + /// When used with --dry-run, also generate local markdown analysis files #[arg(long, requires = "dry_run")] local_analysis: bool, @@ -215,20 +225,20 @@ fn main() -> Result<()> { let cli = Cli::parse(); match cli.command { - Commands::Run(args) => cmd_run(args), - Commands::Analyze(args) => cmd_analyze(args), + Commands::Run(args) => cmd_run(args, cli.reasoning), + Commands::Analyze(args) => cmd_analyze(args, cli.reasoning), } } /* ------------------------------ run ------------------------------ */ -fn cmd_run(args: RunArgs) -> Result<()> { - run_benchmarks(args)?; +fn cmd_run(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { + run_benchmarks(args, reasoning)?; Ok(()) } /// Core benchmark runner used by both `run` and `ci-quickfix` -fn run_benchmarks(args: RunArgs) -> Result<()> { +fn run_benchmarks(args: RunArgs, reasoning: ReasoningEffort) -> Result<()> { let dry_run = args.dry_run; let local_analysis = args.local_analysis; let dry_run_id = dry_run.then(|| { @@ -281,7 +291,8 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { let bench_root = find_bench_root(); // Upload task catalog before running benchmarks - if let Some(ref api) = upload_client + if !args.skip_task_catalog_upload + && let Some(ref api) = upload_client && let Err(e) = api.upload_task_catalog(&bench_root) { eprintln!("[warn] failed to upload task catalog: {e}"); @@ -314,7 +325,7 @@ fn run_benchmarks(args: RunArgs) -> Result<()> { } let llm_provider = if !config.goldens_only && !config.hash_only { - let provider = make_provider_from_env()?; + let provider = make_provider_from_env(reasoning)?; let rt = runtime.as_ref().expect("failed to initialize runtime for preflight"); let routes = filter_routes(&config); preflight_llm_routes(rt, provider.as_ref(), &routes, &modes)?; @@ -378,7 +389,7 @@ fn report_server_status(guard: Option<&mut SpacetimeDbGuard>) { /* ------------------------------ analyze ------------------------------ */ -fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { +fn cmd_analyze(args: AnalyzeArgs, reasoning: ReasoningEffort) -> Result<()> { let api = ApiClient::from_env() .context("failed to initialize API client")? .context("LLM_BENCHMARK_UPLOAD_URL required for analyze")?; @@ -434,7 +445,7 @@ fn cmd_analyze(args: AnalyzeArgs) -> Result<()> { // Initialize LLM provider for analysis let runtime = tokio::runtime::Builder::new_current_thread().enable_all().build()?; - let provider = make_provider_from_env()?; + let provider = make_provider_from_env(reasoning)?; let analysis_route = ModelRoute::new( "gpt-5.4-mini", @@ -964,6 +975,7 @@ mod tests { models: None, model_source: ModelSource::Static, dry_run: false, + skip_task_catalog_upload: false, local_analysis: false, route_overrides: None, } @@ -989,6 +1001,24 @@ mod tests { } } + #[test] + fn reasoning_defaults_to_medium_and_accepts_an_override() { + let default = Cli::try_parse_from(["llm", "run", "--hash-only"]).unwrap(); + assert_eq!(default.reasoning, ReasoningEffort::Medium); + + let overridden = Cli::try_parse_from(["llm", "run", "--hash-only", "--reasoning", "high"]).unwrap(); + assert_eq!(overridden.reasoning, ReasoningEffort::High); + } + + #[test] + fn task_catalog_upload_can_be_skipped() { + let cli = Cli::try_parse_from(["llm", "run", "--hash-only", "--skip-task-catalog-upload"]).unwrap(); + let Commands::Run(args) = cli.command else { + panic!("expected run command"); + }; + assert!(args.skip_task_catalog_upload); + } + #[test] fn explicit_models_bypass_remote_model_source() { let mut args = base_run_args(); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs index 8bb0d1ac734..b8bd6cd55e6 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/anthropic.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ anthropic_ctx_limit_tokens, build_anthropic_messages, desired_output_tokens, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; use anyhow::{anyhow, bail, Context, Result}; use reqwest::header::{HeaderMap, HeaderName, HeaderValue, CONTENT_TYPE}; use reqwest::{Client, StatusCode}; @@ -30,7 +30,7 @@ impl AnthropicClient { format!("{}/v1/messages", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); let mut static_prefix = prompt.static_prefix.clone().unwrap_or_default(); @@ -73,13 +73,25 @@ impl AnthropicClient { struct Req { model: String, max_tokens: u32, + thinking: ThinkingConfig, + output_config: OutputConfig, #[serde(skip_serializing_if = "Option::is_none")] system: Option, messages: Vec, } + #[derive(Serialize)] + struct ThinkingConfig { + r#type: &'static str, + } + #[derive(Serialize)] + struct OutputConfig { + effort: ReasoningEffort, + } let req = Req { model: model_norm.to_string(), max_tokens, + thinking: ThinkingConfig { r#type: "adaptive" }, + output_config: OutputConfig { effort: reasoning }, system: system_json, messages, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs index 5b7bc7b8636..ccf3164d856 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/deepseek.rs @@ -7,7 +7,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deepseek_ctx_limit_tokens, deterministic_trim_prefix, estimate_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct DeepSeekClient { @@ -21,7 +21,7 @@ impl DeepSeekClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); let system = prompt.system.clone(); @@ -47,6 +47,12 @@ impl DeepSeekClient { model: &'a str, messages: Vec>, temperature: f32, + thinking: ThinkingConfig, + reasoning_effort: ReasoningEffort, + } + #[derive(Serialize)] + struct ThinkingConfig { + r#type: &'static str, } #[derive(Serialize)] struct Msg<'a> { @@ -78,6 +84,8 @@ impl DeepSeekClient { model, messages, temperature: 0.0, + thinking: ThinkingConfig { r#type: "enabled" }, + reasoning_effort: reasoning, }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/clients/google.rs b/tools/xtask-llm-benchmark/src/llm/clients/google.rs index ad309a88477..f822be51e5f 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/google.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/google.rs @@ -8,7 +8,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, gemini_ctx_limit_tokens, non_context_reserve_tokens_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; /// Google uses API key in the query string rather than Authorization header. #[derive(Clone)] @@ -23,7 +23,7 @@ impl GoogleGeminiClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { // ---- Never trim system or dynamic segments ---- let system = prompt.system.clone(); let segs: Vec> = prompt.segments.clone(); @@ -49,10 +49,24 @@ impl GoogleGeminiClient { #[serde(skip_serializing_if = "Option::is_none")] system_instruction: Option>, contents: Vec>, + #[serde(rename = "generationConfig")] + generation_config: GenerationConfig, #[serde(skip_serializing_if = "Option::is_none")] safety_settings: Option>, } + #[derive(Serialize)] + struct GenerationConfig { + #[serde(rename = "thinkingConfig")] + thinking_config: ThinkingConfig, + } + + #[derive(Serialize)] + struct ThinkingConfig { + #[serde(rename = "thinkingLevel")] + thinking_level: ReasoningEffort, + } + #[derive(Serialize)] struct SystemInstruction<'a> { parts: [Part<'a>; 1], @@ -100,6 +114,11 @@ impl GoogleGeminiClient { let req = Req { system_instruction, contents, + generation_config: GenerationConfig { + thinking_config: ThinkingConfig { + thinking_level: reasoning, + }, + }, safety_settings: None, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs index 05ea663019e..9bf4d87df9d 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/meta.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/meta.rs @@ -7,7 +7,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, meta_ctx_limit_tokens, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct MetaLlamaClient { @@ -22,7 +22,7 @@ impl MetaLlamaClient { Self { base, api_key, http } } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, _reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); // Build input like other clients diff --git a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs index 254fe5b8f63..604afed9c60 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/mod.rs @@ -20,7 +20,7 @@ pub use openrouter::OpenRouterClient; pub use xai::XaiGrokClient; use crate::llm::prompt::BuiltPrompt; -use crate::llm::types::LlmOutput; +use crate::llm::types::{LlmOutput, ReasoningEffort}; #[derive(Debug, Clone)] pub struct ClientPreflight { @@ -51,7 +51,7 @@ pub trait LlmClient: Send + Sync { ))) } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result; + async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result; } macro_rules! impl_direct_llm_client { @@ -62,8 +62,13 @@ macro_rules! impl_direct_llm_client { $provider_name } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { - <$ty>::generate(self, model, prompt).await + async fn generate( + &self, + model: &str, + prompt: &BuiltPrompt, + reasoning: ReasoningEffort, + ) -> Result { + <$ty>::generate(self, model, prompt, reasoning).await } } }; @@ -87,7 +92,7 @@ impl LlmClient for OpenRouterClient { Ok(ClientPreflight::new(status.summary())) } - async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { - OpenRouterClient::generate(self, model, prompt).await + async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { + OpenRouterClient::generate(self, model, prompt, reasoning).await } } diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs index 9fed933d3fb..537b4a6b84a 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openai.rs @@ -4,7 +4,7 @@ use crate::llm::segmentation::{ build_openai_responses_input, deterministic_trim_prefix, estimate_tokens, headroom_tokens_env, non_context_reserve_tokens_env, openai_ctx_limit_tokens, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; use anyhow::{bail, Context, Result}; use reqwest::{Client, StatusCode}; use serde::{Deserialize, Serialize}; @@ -29,7 +29,7 @@ impl OpenAiClient { format!("{}/v1/responses", self.base.trim_end_matches('/')) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let system = prompt.system.clone(); let segs = prompt.segments.clone(); @@ -85,14 +85,21 @@ impl OpenAiClient { struct Req<'a> { model: &'a str, input: Vec, + reasoning: ReasoningConfig, #[serde(skip_serializing_if = "Option::is_none")] max_output_tokens: Option, } + #[derive(Serialize)] + struct ReasoningConfig { + effort: ReasoningEffort, + } + let url = self.responses_url(); let payload = Req { model, input, + reasoning: ReasoningConfig { effort: reasoning }, max_output_tokens: None, }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs index df4c062c1bb..c4e806666ee 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/openrouter.rs @@ -8,7 +8,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, non_context_reserve_tokens_env, output_token_limit_env, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; const OPENROUTER_BASE: &str = "https://openrouter.ai/api/v1"; @@ -162,7 +162,7 @@ impl OpenRouterClient { Ok(()) } - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/chat/completions", self.base.trim_end_matches('/')); let system = prompt.system.clone(); @@ -183,6 +183,7 @@ impl OpenRouterClient { model: &'a str, messages: Vec>, temperature: f32, + reasoning: ReasoningConfig, #[serde(skip_serializing_if = "Option::is_none")] top_p: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -195,6 +196,11 @@ impl OpenRouterClient { content: &'a str, } + #[derive(Serialize)] + struct ReasoningConfig { + effort: ReasoningEffort, + } + let mut messages: Vec = Vec::new(); if let Some(sys) = system.as_deref() @@ -222,6 +228,7 @@ impl OpenRouterClient { model, messages, temperature: 0.0, + reasoning: ReasoningConfig { effort: reasoning }, top_p: None, max_tokens: output_token_limit_env().map(|limit| limit.max(1) as u32), }; diff --git a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs index be345c82b32..46062c6d9bc 100644 --- a/tools/xtask-llm-benchmark/src/llm/clients/xai.rs +++ b/tools/xtask-llm-benchmark/src/llm/clients/xai.rs @@ -6,7 +6,7 @@ use crate::llm::prompt::BuiltPrompt; use crate::llm::segmentation::{ deterministic_trim_prefix, non_context_reserve_tokens_env, xai_ctx_limit_tokens, Segment, }; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[derive(Clone)] pub struct XaiGrokClient { @@ -21,7 +21,7 @@ impl XaiGrokClient { } /// Uses BuiltPrompt (system, static_prefix, segments) and maps to xAI /chat/completions. - pub async fn generate(&self, model: &str, prompt: &BuiltPrompt) -> Result { + pub async fn generate(&self, model: &str, prompt: &BuiltPrompt, reasoning: ReasoningEffort) -> Result { let url = format!("{}/v1/chat/completions", self.base.trim_end_matches('/')); // Never trim system or dynamic segments @@ -41,6 +41,7 @@ impl XaiGrokClient { model: &'a str, messages: Vec>, temperature: f32, + reasoning_effort: ReasoningEffort, } #[derive(Serialize)] @@ -75,6 +76,7 @@ impl XaiGrokClient { model, messages, temperature: 0.0, + reasoning_effort: reasoning, }; let auth = HttpClient::bearer(&self.api_key); diff --git a/tools/xtask-llm-benchmark/src/llm/config.rs b/tools/xtask-llm-benchmark/src/llm/config.rs index 5eacdf40248..2e76dcbfdef 100644 --- a/tools/xtask-llm-benchmark/src/llm/config.rs +++ b/tools/xtask-llm-benchmark/src/llm/config.rs @@ -6,7 +6,7 @@ use crate::llm::clients::{ AnthropicClient, DeepSeekClient, GoogleGeminiClient, MetaLlamaClient, OpenAiClient, OpenRouterClient, XaiGrokClient, }; use crate::llm::provider::{LlmProvider, RouterProvider}; -use crate::llm::types::Vendor; +use crate::llm::types::{ReasoningEffort, Vendor}; fn force_vendor_from_env() -> Option { match env::var("LLM_VENDOR").ok().as_deref() { @@ -34,7 +34,7 @@ fn force_vendor_from_env() -> Option { /// When OPENROUTER_API_KEY is set, it acts as a fallback for any vendor that doesn't /// have its own direct API key configured. This means you can set just OPENROUTER_API_KEY /// to run all models through OpenRouter, or mix direct keys with OpenRouter fallback. -pub fn make_provider_from_env() -> Result> { +pub fn make_provider_from_env(reasoning: ReasoningEffort) -> Result> { let http = HttpClient::new()?; // Filter out empty strings so an empty env var falls through to OpenRouter. @@ -84,6 +84,8 @@ pub fn make_provider_from_env() -> Result> { let openrouter = openrouter_key.map(|k| OpenRouterClient::new(http.clone(), k)); let force = force_vendor_from_env(); - let router = RouterProvider::new(openai, anthropic, google, xai, deepseek, meta, openrouter, force); + let router = RouterProvider::new( + openai, anthropic, google, xai, deepseek, meta, openrouter, force, reasoning, + ); Ok(Arc::new(router)) } diff --git a/tools/xtask-llm-benchmark/src/llm/mod.rs b/tools/xtask-llm-benchmark/src/llm/mod.rs index 4b72185c760..9e6548ec55a 100644 --- a/tools/xtask-llm-benchmark/src/llm/mod.rs +++ b/tools/xtask-llm-benchmark/src/llm/mod.rs @@ -10,4 +10,4 @@ pub use config::make_provider_from_env; pub use model_routes::{default_model_routes, ModelRoute}; pub use prompt::PromptBuilder; pub use provider::{LlmProvider, RouterProvider}; -pub use types::LlmOutput; +pub use types::{LlmOutput, ReasoningEffort}; diff --git a/tools/xtask-llm-benchmark/src/llm/provider.rs b/tools/xtask-llm-benchmark/src/llm/provider.rs index 355f2e19a3e..1dba17764b9 100644 --- a/tools/xtask-llm-benchmark/src/llm/provider.rs +++ b/tools/xtask-llm-benchmark/src/llm/provider.rs @@ -8,7 +8,7 @@ use crate::llm::clients::{ }; use crate::llm::model_routes::ModelRoute; use crate::llm::prompt::BuiltPrompt; -use crate::llm::types::{LlmOutput, Vendor}; +use crate::llm::types::{LlmOutput, ReasoningEffort, Vendor}; #[async_trait] pub trait LlmProvider: Send + Sync { @@ -19,6 +19,7 @@ pub trait LlmProvider: Send + Sync { pub struct RouterProvider { clients: HashMap>, pub force: Option, + reasoning: ReasoningEffort, } impl RouterProvider { @@ -32,6 +33,7 @@ impl RouterProvider { meta: Option, openrouter: Option, force: Option, + reasoning: ReasoningEffort, ) -> Self { let mut clients: HashMap> = HashMap::new(); @@ -57,7 +59,11 @@ impl RouterProvider { clients.insert(Vendor::OpenRouter, Box::new(client)); } - Self { clients, force } + Self { + clients, + force, + reasoning, + } } } @@ -105,7 +111,7 @@ impl LlmProvider for RouterProvider { ); } - resolved.client.generate(&resolved.model, prompt).await + resolved.client.generate(&resolved.model, prompt, self.reasoning).await } } diff --git a/tools/xtask-llm-benchmark/src/llm/types.rs b/tools/xtask-llm-benchmark/src/llm/types.rs index 5ccfd14f339..f83a45b3d81 100644 --- a/tools/xtask-llm-benchmark/src/llm/types.rs +++ b/tools/xtask-llm-benchmark/src/llm/types.rs @@ -1,6 +1,16 @@ +use clap::ValueEnum; use serde::{Deserialize, Serialize}; use std::fmt; +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, ValueEnum)] +#[serde(rename_all = "lowercase")] +pub enum ReasoningEffort { + Low, + #[default] + Medium, + High, +} + /// Output from an LLM generation call, including token usage. #[derive(Debug, Clone, Default)] pub struct LlmOutput { @@ -68,3 +78,13 @@ impl fmt::Display for Vendor { f.write_str(self.slug()) } } + +#[cfg(test)] +mod tests { + use super::ReasoningEffort; + + #[test] + fn reasoning_effort_uses_provider_wire_values() { + assert_eq!(serde_json::to_string(&ReasoningEffort::Medium).unwrap(), r#""medium""#); + } +}