Group CI logs and surface failures on the default (non-bucketed) test run

.claude/rules/testing/built-version-ci.md

@@ -24,3 +24,6 @@ Invariants when editing:
 - `test-ff-matrix.yml` owns the ff-matrix bucket glob.
 - Scheduler jobs in `test-smokes-built.yml` set per-leg OS scope through their `runners:` inputs.
 - Keep the per-call suffix in `test-ff-matrix.yml`'s concurrency group so sibling calls cannot cancel one another.
+- Every `test-smokes.yml` and `test-ff-matrix.yml` call site passes a
+  `label-tag` that distinguishes same-OS jobs in the combined run summary.
+  Per-OS jobs may share a tag because labels also include the OS.

.claude/rules/testing/github-actions-reporting.md

@@ -0,0 +1,41 @@
+---
+paths:
+  - .github/workflows/test-smokes.yml
+  - .github/workflows/test-smokes-built.yml
+  - .github/workflows/test-smokes-parallel.yml
+  - .github/workflows/test-ff-matrix.yml
+  - .github/workflows/update-test-timing.yml
+  - src/tools/github.ts
+  - tests/test.ts
+  - tests/gha-grouping.ts
+  - tests/integration/playwright-tests.test.ts
+  - tests/tools/check-gha-log.ts
+  - tests/unit/check-gha-log.test.ts
+  - tests/unit/gha-grouping.test.ts
+  - tests/unit/github-actions-reporting.test.ts
+  - tests/unit/harness-failure-reporting.test.ts
+  - tests/unit/harness-reporting-fixture.ts
+---
+
+# GitHub Actions test reporting
+
+`llm-docs/github-actions-test-reporting.md` is the deep dive. Read only the
+sections relevant to the edit:
+
+- Workflow ownership or environment wiring: "Ownership" and "Workflow contract".
+- Group opening, closing, or log validation: "Log grouping" and "Verification".
+- Annotations, labels, excerpts, or step summaries: "Failure reporting" and
+  "Annotation budget and labels".
+
+For a trivial edit, the invariants below are sufficient.
+
+Keep these invariants:
+
+- A step has one group and annotation owner. Bucket loops set
+  `QUARTO_TESTS_GHA_ORCHESTRATED=1`; otherwise the harness owns them.
+- Step-summary entries are emitted in both paths.
+- Failure lines and Deno's final failure sections remain outside groups.
+- The annotation count is step-wide; Deno module state is only per test file.
+- Every reusable-workflow call passes a `label-tag` that distinguishes same-OS
+  jobs in the run summary.
+- Do not enable parallel test-file execution without redesigning grouping.

.github/workflows/test-ff-matrix.yml

@@ -38,6 +38,11 @@ on:
         required: false
         type: string
         default: '["ubuntu-latest", "windows-latest"]'
+      label-tag:
+        description: "Short alphanumeric tag for distinguishing same-OS jobs in failure summaries"
+        required: false
+        type: string
+        default: ""
   workflow_dispatch:
     inputs:
       extra-r-packages:
@@ -100,3 +105,4 @@ jobs:
       quarto-artifact-run-id: ${{ inputs.quarto-artifact-run-id }}
       ref: ${{ inputs.ref }}
       runners: ${{ inputs.runners || '["ubuntu-latest", "windows-latest"]' }}
+      label-tag: ${{ inputs.label-tag || 'ffdev' }}

.github/workflows/test-smokes-built.yml

@@ -53,11 +53,10 @@ jobs:
         shell: bash
         run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
 
-      - name: Compute version marker
+      - name: Compute build version
         id: version
         shell: bash
-        # Build metadata keeps quarto-required range checks valid.
-        run: echo "version=$(cat version.txt)+test.$(date +%Y%m%d)" >> "$GITHUB_OUTPUT"
+        run: echo "version=$(cat version.txt)" >> "$GITHUB_OUTPUT"
 
       - name: Build dist tarball
         uses: ./.github/actions/build-dist-tarball
@@ -80,6 +79,7 @@ jobs:
       quarto-artifact-name: built-quarto-linux-amd64
       runners: '["ubuntu-latest"]'
       buckets: ${{ github.event.inputs.buckets }} # empty = full run
+      label-tag: bldsmk
 
   run-playwright-artifact:
     name: Playwright tests against built artifact
@@ -94,6 +94,7 @@ jobs:
       quarto-artifact-name: built-quarto-linux-amd64
       runners: '["ubuntu-latest"]'
       buckets: '["integration/playwright-tests.test.ts"]'
+      label-tag: bldpw
 
   run-ff-matrix-artifact:
     name: Feature-format matrix against built artifact
@@ -107,6 +108,7 @@ jobs:
       quarto-install: artifact
       quarto-artifact-name: built-quarto-linux-amd64
       runners: '["ubuntu-latest"]'
+      label-tag: bldff
 
   # Test an existing published release.
   resolve-release:
@@ -169,6 +171,7 @@ jobs:
       quarto-version: ${{ needs.resolve-release.outputs.version }}
       runners: '["ubuntu-latest", "windows-latest"]'
       buckets: ${{ github.event.inputs.buckets }} # empty = full run
+      label-tag: relsmk
 
   run-playwright-release:
     name: Playwright tests against published release
@@ -181,6 +184,7 @@ jobs:
       quarto-version: ${{ needs.resolve-release.outputs.version }}
       runners: '["ubuntu-latest"]'
       buckets: '["integration/playwright-tests.test.ts"]'
+      label-tag: relpw
 
   run-ff-matrix-release:
     name: Feature-format matrix against published release
@@ -192,6 +196,7 @@ jobs:
       quarto-install: release
       quarto-version: ${{ needs.resolve-release.outputs.version }}
       runners: '["ubuntu-latest", "windows-latest"]'
+      label-tag: relff
 
   # Reuse artifacts from a completed create-release run.
   resolve-nightly:
@@ -262,6 +267,7 @@ jobs:
       quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }}
       runners: '["ubuntu-latest"]'
       buckets: ${{ github.event.inputs.buckets }} # empty = full run
+      label-tag: ngtsmk
 
   run-smokes-nightly-windows:
     name: Smoke tests against nightly build (windows)
@@ -278,6 +284,7 @@ jobs:
       quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }}
       runners: '["windows-latest"]'
       buckets: ${{ github.event.inputs.buckets }} # empty = full run
+      label-tag: ngtsmk
 
   run-smokes-nightly-mac:
     name: Smoke tests against nightly build (macOS)
@@ -294,6 +301,7 @@ jobs:
       quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }}
       runners: '["macos-latest"]'
       buckets: ${{ github.event.inputs.buckets }} # empty = full run
+      label-tag: ngtsmk
 
   # Each caller job accepts one OS-specific artifact.
   run-playwright-nightly-linux:
@@ -311,6 +319,7 @@ jobs:
       quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }}
       runners: '["ubuntu-latest"]'
       buckets: '["integration/playwright-tests.test.ts"]'
+      label-tag: ngtpw
 
   run-playwright-nightly-mac:
     name: Playwright tests against nightly build (macOS)
@@ -327,6 +336,7 @@ jobs:
       quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }}
       runners: '["macos-latest"]'
       buckets: '["integration/playwright-tests.test.ts"]'
+      label-tag: ngtpw
 
   run-ff-matrix-nightly-linux:
     name: Feature-format matrix against nightly build (linux)
@@ -342,6 +352,7 @@ jobs:
       quarto-artifact-name: Deb Zip
       quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }}
       runners: '["ubuntu-latest"]'
+      label-tag: ngtff
 
   run-ff-matrix-nightly-windows:
     name: Feature-format matrix against nightly build (windows)
@@ -357,3 +368,4 @@ jobs:
       quarto-artifact-name: Windows Zip
       quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }}
       runners: '["windows-latest"]'
+      label-tag: ngtff

.github/workflows/test-smokes-parallel.yml

@@ -104,3 +104,5 @@ jobs:
     with:
       buckets: ${{ matrix.buckets.files }}
       extra-r-packages: ${{ inputs.extra-r-packages }}
+      # The bucket index distinguishes same-OS matrix jobs.
+      label-tag: b${{ matrix.buckets.num }}

.github/workflows/test-smokes.yml

@@ -49,6 +49,11 @@ on:
         required: false
         type: string
         default: '["ubuntu-latest", "windows-latest"]'
+      label-tag:
+        description: "Short alphanumeric tag for distinguishing same-OS jobs in failure summaries"
+        required: false
+        type: string
+        default: ""
   workflow_dispatch:
     inputs:
       buckets:
@@ -92,6 +97,8 @@ jobs:
     runs-on: ${{ matrix.os }}
     env:
       PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/ms-playwright-browsers
+      # Distinguishes this call from other jobs on the same OS.
+      QUARTO_TESTS_GHA_LABEL_TAG: ${{ inputs.label-tag }}
     steps:
       - name: Checkout Repo
         uses: actions/checkout@v6
@@ -491,6 +498,8 @@ jobs:
           # Useful as TinyTeX latest release is checked in run-test.sh
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BUCKETS_JSON: ${{ inputs.buckets }}
+          # Prevent the harness from duplicating this loop's workflow commands.
+          QUARTO_TESTS_GHA_ORCHESTRATED: "1"
         run: |
           haserror=0
           failed_tests=()
@@ -540,6 +549,8 @@ jobs:
           # Useful as TinyTeX latest release is checked in run-test.sh
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BUCKETS_JSON: ${{ inputs.buckets }}
+          # Prevent the harness from duplicating this loop's workflow commands.
+          QUARTO_TESTS_GHA_ORCHESTRATED: "1"
         run: |
           $haserror=$false
           $failed_tests=@()

.github/workflows/update-test-timing.yml

@@ -13,6 +13,7 @@ jobs:
     with:
       buckets: ""
       time-test: true
+      label-tag: timing
 
   create-pr:
     name: Create PR with updated timing file

llm-docs/built-version-testing-architecture.md

@@ -188,7 +188,8 @@ Key points:
 
 The feature-format bucket glob (`../dev-docs/feature-format-matrix/qmd-files/**/*.qmd`) is defined only in `test-ff-matrix.yml`.
 Built-mode callers use its `workflow_call` trigger, while its existing dev triggers remain.
-The workflow forwards install, artifact, ref, runner, and R-package inputs to `test-smokes.yml`, with dev defaults for non-call triggers.
+The workflow forwards install, artifact, ref, runner, R-package, and `label-tag` inputs to `test-smokes.yml`, with dev defaults for non-call triggers.
+The tag defaults to `ffdev` for standalone dev triggers and distinguishes same-OS jobs in the shared failure summary.
 
 Reusable-workflow concurrency is evaluated in the caller's context. The group therefore includes a suffix based on `inputs.runners` and `github.run_id`, preventing sibling feature-format legs from canceling one another.
 Dev triggers use a constant `-dev` suffix.
@@ -216,11 +217,17 @@ The trigger fires after every completed create-release run, including manual and
 
 **Revisit when:** maintainers want a single nightly build-and-test status and are willing to couple the workflows.
 
-### D2. Version marker: semver *build metadata* (`X.Y.Z+test.YYYYMMDD`)
+### D2. Version stamp: the plain three-component version (`X.Y.Z`)
 
-Built test distributions use `$(cat version.txt)+test.$(date +%Y%m%d)`.
-Do not use a prerelease suffix, which fails plain `>=X.Y` `quarto-required` ranges, or a fourth numeric component, which is invalid semver.
-Build metadata preserves range comparisons while distinguishing the build from the `99.9.9` dev version.
+Built test distributions use `$(cat version.txt)`, a plain `X.Y.Z`. Pandoc's
+`Version` type, used by the `version` shortcode, cannot parse semver build
+metadata such as `+test.YYYYMMDD`; a plain version also matches a release
+artifact more closely. The `99.9.9` sentinel still distinguishes dev mode.
+
+Do not use a prerelease suffix: prerelease versions fail ordinary `>=X.Y`
+`quarto-required` ranges. Do not add a fourth numeric component: it is not
+valid semver. Identify trial builds by workflow, ref, and SHA instead of the
+version string.
 
 ### D3. Dist outside the checkout + `99.9.9` sentinel refusal