Add LLM benchmark analysis and Discord reporting

.github/workflows/llm-benchmark-periodic.yml

@@ -39,6 +39,16 @@ on:
         description: 'Run benchmarks without uploading results'
         required: false
         default: 'false'
+      skip_task_catalog_upload:
+        description: 'Skip uploading the benchmark task catalog'
+        required: false
+        type: boolean
+        default: false
+      post_discord:
+        description: 'Post the analysis summary to Discord'
+        required: false
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -166,14 +176,16 @@ jobs:
           LLM_BENCH_CONCURRENCY: "8"
           LLM_BENCH_RUST_CONCURRENCY: "4"
           LLM_BENCH_CSHARP_CONCURRENCY: "2"
-          LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || '2' }}
+          LLM_BENCH_ROUTE_CONCURRENCY: ${{ matrix.lang == 'typescript' && '4' || matrix.lang == 'csharp' && '1' || '2' }}
+          LLM_BENCHMARK_REPORT_DIR: ${{ runner.temp }}/llm-benchmark-reports
           INPUT_LANGUAGE: ${{ matrix.lang }}
           INPUT_MODEL_SET: ${{ inputs.model_set || 'website_active' }}
           INPUT_MODELS: ${{ inputs.models || '' }}
           INPUT_MODES: ${{ inputs.modes || 'guidelines,no_context' }}
           INPUT_CATEGORIES: ${{ inputs.categories || '' }}
           INPUT_TASKS: ${{ inputs.tasks || '' }}
           INPUT_DRY_RUN: ${{ inputs.dry_run || 'false' }}
+          INPUT_SKIP_TASK_CATALOG_UPLOAD: ${{ inputs.skip_task_catalog_upload || 'false' }}
         run: |
           LANG="$INPUT_LANGUAGE"
           MODEL_SET="$INPUT_MODEL_SET"
@@ -182,6 +194,7 @@ jobs:
           CATEGORIES="$INPUT_CATEGORIES"
           TASKS="$INPUT_TASKS"
           DRY_RUN="$INPUT_DRY_RUN"
+          SKIP_TASK_CATALOG_UPLOAD="$INPUT_SKIP_TASK_CATALOG_UPLOAD"
 
           case "$MODEL_SET" in
             website_active)
@@ -219,6 +232,9 @@ jobs:
           if [ "$DRY_RUN" = "true" ]; then
             EXTRA_ARGS+=(--dry-run)
           fi
+          if [ "$SKIP_TASK_CATALOG_UPLOAD" = "true" ]; then
+            EXTRA_ARGS+=(--skip-task-catalog-upload)
+          fi
 
           if [ "$MODEL_SET" = "website_active" ]; then
             llm_benchmark run --lang "$LANG" --modes "$MODES" --model-source remote "${EXTRA_ARGS[@]}"
@@ -227,3 +243,81 @@ jobs:
           else
             llm_benchmark run --lang "$LANG" --modes "$MODES" --models "${MODEL_ARGS[@]}" "${EXTRA_ARGS[@]}"
           fi
+
+      - name: Upload analysis reports
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: llm-benchmark-analysis-${{ matrix.lang }}
+          path: ${{ runner.temp }}/llm-benchmark-reports
+          if-no-files-found: ignore
+          retention-days: 14
+
+  summarize:
+    name: Summarize benchmark analysis
+    if: always()
+    needs: run-benchmarks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Test Discord formatter
+        run: python3 tools/xtask-llm-benchmark/scripts/test_discord_summary.py
+
+      - name: Download analysis reports
+        continue-on-error: true
+        uses: actions/download-artifact@v4
+        with:
+          pattern: llm-benchmark-analysis-*
+          path: reports
+          merge-multiple: true
+
+      - name: Publish workflow summary
+        run: |
+          echo "# LLM benchmark analysis" >> "$GITHUB_STEP_SUMMARY"
+          echo >> "$GITHUB_STEP_SUMMARY"
+
+          if [ ! -d reports ] || ! find reports -type f -name '*.md' -print -quit | grep -q .; then
+            echo "No analysis reports were produced." >> "$GITHUB_STEP_SUMMARY"
+            exit 0
+          fi
+
+          echo "Full per-model reports are available in the workflow artifacts." >> "$GITHUB_STEP_SUMMARY"
+          echo >> "$GITHUB_STEP_SUMMARY"
+
+          find reports -type f -name '*.md' -print | sort | while IFS= read -r report; do
+            awk '/^## Failure patterns/{exit} NR <= 80 {print}' "$report" >> "$GITHUB_STEP_SUMMARY"
+            echo >> "$GITHUB_STEP_SUMMARY"
+            echo "---" >> "$GITHUB_STEP_SUMMARY"
+            echo >> "$GITHUB_STEP_SUMMARY"
+          done
+
+      - name: Prepare Discord summary
+        if: github.event_name == 'schedule' || inputs.post_discord
+        env:
+          RUN_URL: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}
+          RUN_LABEL: ${{ github.event_name == 'schedule' && 'Weekly scheduled run' || 'Manual run' }}
+        run: >
+          python3 tools/xtask-llm-benchmark/scripts/discord_summary.py
+          --reports-dir reports
+          --run-url "$RUN_URL"
+          --run-label "$RUN_LABEL"
+          --output discord-payload.json
+
+      - name: Post Discord summary
+        if: github.event_name == 'schedule' || inputs.post_discord
+        continue-on-error: true
+        env:
+          DISCORD_WEBHOOK_URL: ${{ secrets.LLM_BENCHMARK_DISCORD_WEBHOOK_URL }}
+        run: |
+          if [ -z "$DISCORD_WEBHOOK_URL" ]; then
+            echo "::warning::LLM_BENCHMARK_DISCORD_WEBHOOK_URL is not configured"
+            exit 0
+          fi
+
+          curl --fail --silent --show-error \
+            --header 'Content-Type: application/json' \
+            --data-binary @discord-payload.json \
+            "$DISCORD_WEBHOOK_URL"

tools/xtask-llm-benchmark/scripts/discord_summary.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+
+import argparse
+import json
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+from pathlib import Path
+
+GREEN = 0x57F287
+YELLOW = 0xFEE75C
+RED = 0xED4245
+MAX_ITEMS = 5
+MAX_FIELD_LENGTH = 1000
+
+
+@dataclass
+class Report:
+    language: str
+    mode: str
+    model: str
+    passed_tasks: int
+    total_tasks: int
+    actions: list[str]
+    other_findings: list[str]
+
+
+def field(text: str, name: str) -> str:
+    match = re.search(rf"^- {re.escape(name)}: (.+)$", text, re.MULTILINE)
+    return match.group(1) if match else "unknown"
+
+
+def language_name(language: str) -> str:
+    return {
+        "csharp": "C#",
+        "rust": "Rust",
+        "typescript": "TypeScript",
+    }.get(language, language)
+
+
+def parse_report(text: str) -> Report:
+    language = field(text, "Language")
+    mode = field(text, "Mode")
+    model = field(text, "Model")
+    task_match = re.match(r"(\d+)/(\d+)", field(text, "Tasks"))
+    passed_tasks, total_tasks = map(int, task_match.groups()) if task_match else (0, 0)
+
+    actions = []
+    recommended = text.partition("## Recommended actions")[2].partition("## Failure patterns")[0]
+    for line in recommended.splitlines():
+        if line.startswith("- **["):
+            actions.append(f"- {language_name(language)} / {mode}: {line[2:]}")
+
+    other_findings = []
+    failures = text.partition("## Failure patterns")[2]
+    for section in re.split(r"^### ", failures, flags=re.MULTILINE)[1:]:
+        heading = section.splitlines()[0]
+        title_match = re.match(r"(.+) \(\d+ tasks?\)$", heading)
+        title = title_match.group(1) if title_match else heading
+        classification = re.search(r"^- \*\*Classification:\*\* (.+)$", section, re.MULTILINE)
+        if classification and classification.group(1) in {
+            "Model limitation",
+            "Infrastructure/provider problem",
+            "No action",
+        }:
+            other_findings.append(
+                f"- {language_name(language)} / {mode} / {model}: {title} - {classification.group(1)}"
+            )
+
+    return Report(language, mode, model, passed_tasks, total_tasks, actions, other_findings)
+
+
+def load_reports(reports_dir: Path) -> list[Report]:
+    return [
+        parse_report(path.read_text(encoding="utf-8"))
+        for path in sorted(reports_dir.rglob("*.md"))
+    ]
+
+
+def rate(passed: int, total: int) -> str:
+    percent = passed * 100 / total if total else 0
+    return f"{passed}/{total} ({percent:.1f}%)"
+
+
+def field_value(lines: list[str], empty: str, overflow_label: str) -> str:
+    unique = list(dict.fromkeys(lines))
+    visible = unique[:MAX_ITEMS] or [empty]
+    if len(unique) > MAX_ITEMS:
+        visible.append(f"- ...and {len(unique) - MAX_ITEMS} more {overflow_label}(s)")
+    value = "\n".join(visible)
+    if len(value) > MAX_FIELD_LENGTH:
+        return value[: MAX_FIELD_LENGTH - 30].rstrip() + "\n... View the full analysis."
+    return value
+
+
+def build_payload(reports: list[Report], run_url: str, run_label: str) -> dict:
+    totals = [0, 0]
+    by_language: dict[str, list[int]] = defaultdict(lambda: [0, 0])
+    actions = []
+    other_findings = []
+
+    for report in reports:
+        totals[0] += report.passed_tasks
+        totals[1] += report.total_tasks
+        by_language[report.language][0] += report.passed_tasks
+        by_language[report.language][1] += report.total_tasks
+        actions.extend(report.actions)
+        other_findings.extend(report.other_findings)
+
+    passed, total = totals
+    pass_percent = passed * 100 / total if total else 0
+    has_infrastructure_failure = any("Infrastructure/provider problem" in item for item in other_findings)
+    if not total or pass_percent < 90 or has_infrastructure_failure:
+        color = RED
+    elif actions or pass_percent < 95:
+        color = YELLOW
+    else:
+        color = GREEN
+
+    language_rates = [
+        f"- **{language_name(language)}:** {rate(*counts)}"
+        for language, counts in sorted(by_language.items())
+    ]
+    return {
+        "username": "SpacetimeDB LLM Benchmarks",
+        "allowed_mentions": {"parse": []},
+        "embeds": [
+            {
+                "title": "LLM Benchmark Analysis",
+                "url": run_url,
+                "description": f"**{rate(*totals)}** task runs passed",
+                "color": color,
+                "fields": [
+                    {
+                        "name": "By language",
+                        "value": field_value(
+                            language_rates,
+                            "No analysis reports were produced.",
+                            "language",
+                        ),
+                        "inline": False,
+                    },
+                    {
+                        "name": "Action items",
+                        "value": field_value(actions, "None", "action"),
+                        "inline": False,
+                    },
+                    {
+                        "name": "Other failures",
+                        "value": field_value(other_findings, "None", "finding"),
+                        "inline": False,
+                    },
+                ],
+                "footer": {"text": run_label},
+            }
+        ],
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Build a Discord embed from LLM benchmark analysis reports.")
+    parser.add_argument("--reports-dir", type=Path, required=True)
+    parser.add_argument("--run-url", required=True)
+    parser.add_argument("--run-label", required=True)
+    parser.add_argument("--output", type=Path, required=True)
+    args = parser.parse_args()
+
+    payload = build_payload(load_reports(args.reports_dir), args.run_url, args.run_label)
+    args.output.write_text(json.dumps(payload, ensure_ascii=False), encoding="utf-8")
+
+
+if __name__ == "__main__":
+    main()