diff --git a/.claude/rules/testing/built-version-ci.md b/.claude/rules/testing/built-version-ci.md index d12c832e269..c24c3aa01c7 100644 --- a/.claude/rules/testing/built-version-ci.md +++ b/.claude/rules/testing/built-version-ci.md @@ -24,3 +24,6 @@ Invariants when editing: - `test-ff-matrix.yml` owns the ff-matrix bucket glob. - Scheduler jobs in `test-smokes-built.yml` set per-leg OS scope through their `runners:` inputs. - Keep the per-call suffix in `test-ff-matrix.yml`'s concurrency group so sibling calls cannot cancel one another. +- Every `test-smokes.yml` and `test-ff-matrix.yml` call site passes a + `label-tag` that distinguishes same-OS jobs in the combined run summary. + Per-OS jobs may share a tag because labels also include the OS. diff --git a/.claude/rules/testing/github-actions-reporting.md b/.claude/rules/testing/github-actions-reporting.md new file mode 100644 index 00000000000..4d978fdda36 --- /dev/null +++ b/.claude/rules/testing/github-actions-reporting.md @@ -0,0 +1,41 @@ +--- +paths: + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-built.yml + - .github/workflows/test-smokes-parallel.yml + - .github/workflows/test-ff-matrix.yml + - .github/workflows/update-test-timing.yml + - src/tools/github.ts + - tests/test.ts + - tests/gha-grouping.ts + - tests/integration/playwright-tests.test.ts + - tests/tools/check-gha-log.ts + - tests/unit/check-gha-log.test.ts + - tests/unit/gha-grouping.test.ts + - tests/unit/github-actions-reporting.test.ts + - tests/unit/harness-failure-reporting.test.ts + - tests/unit/harness-reporting-fixture.ts +--- + +# GitHub Actions test reporting + +`llm-docs/github-actions-test-reporting.md` is the deep dive. Read only the +sections relevant to the edit: + +- Workflow ownership or environment wiring: "Ownership" and "Workflow contract". +- Group opening, closing, or log validation: "Log grouping" and "Verification". +- Annotations, labels, excerpts, or step summaries: "Failure reporting" and + "Annotation budget and labels". + +For a trivial edit, the invariants below are sufficient. + +Keep these invariants: + +- A step has one group and annotation owner. Bucket loops set + `QUARTO_TESTS_GHA_ORCHESTRATED=1`; otherwise the harness owns them. +- Step-summary entries are emitted in both paths. +- Failure lines and Deno's final failure sections remain outside groups. +- The annotation count is step-wide; Deno module state is only per test file. +- Every reusable-workflow call passes a `label-tag` that distinguishes same-OS + jobs in the run summary. +- Do not enable parallel test-file execution without redesigning grouping. diff --git a/.github/workflows/test-ff-matrix.yml b/.github/workflows/test-ff-matrix.yml index cfcdf66f422..2fa09a3cacc 100644 --- a/.github/workflows/test-ff-matrix.yml +++ b/.github/workflows/test-ff-matrix.yml @@ -38,6 +38,11 @@ on: required: false type: string default: '["ubuntu-latest", "windows-latest"]' + label-tag: + description: "Short alphanumeric tag for distinguishing same-OS jobs in failure summaries" + required: false + type: string + default: "" workflow_dispatch: inputs: extra-r-packages: @@ -100,3 +105,4 @@ jobs: quarto-artifact-run-id: ${{ inputs.quarto-artifact-run-id }} ref: ${{ inputs.ref }} runners: ${{ inputs.runners || '["ubuntu-latest", "windows-latest"]' }} + label-tag: ${{ inputs.label-tag || 'ffdev' }} diff --git a/.github/workflows/test-smokes-built.yml b/.github/workflows/test-smokes-built.yml index 856e4b8a8ba..f16410d0b0d 100644 --- a/.github/workflows/test-smokes-built.yml +++ b/.github/workflows/test-smokes-built.yml @@ -53,11 +53,10 @@ jobs: shell: bash run: echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Compute version marker + - name: Compute build version id: version shell: bash - # Build metadata keeps quarto-required range checks valid. - run: echo "version=$(cat version.txt)+test.$(date +%Y%m%d)" >> "$GITHUB_OUTPUT" + run: echo "version=$(cat version.txt)" >> "$GITHUB_OUTPUT" - name: Build dist tarball uses: ./.github/actions/build-dist-tarball @@ -80,6 +79,7 @@ jobs: quarto-artifact-name: built-quarto-linux-amd64 runners: '["ubuntu-latest"]' buckets: ${{ github.event.inputs.buckets }} # empty = full run + label-tag: bldsmk run-playwright-artifact: name: Playwright tests against built artifact @@ -94,6 +94,7 @@ jobs: quarto-artifact-name: built-quarto-linux-amd64 runners: '["ubuntu-latest"]' buckets: '["integration/playwright-tests.test.ts"]' + label-tag: bldpw run-ff-matrix-artifact: name: Feature-format matrix against built artifact @@ -107,6 +108,7 @@ jobs: quarto-install: artifact quarto-artifact-name: built-quarto-linux-amd64 runners: '["ubuntu-latest"]' + label-tag: bldff # Test an existing published release. resolve-release: @@ -169,6 +171,7 @@ jobs: quarto-version: ${{ needs.resolve-release.outputs.version }} runners: '["ubuntu-latest", "windows-latest"]' buckets: ${{ github.event.inputs.buckets }} # empty = full run + label-tag: relsmk run-playwright-release: name: Playwright tests against published release @@ -181,6 +184,7 @@ jobs: quarto-version: ${{ needs.resolve-release.outputs.version }} runners: '["ubuntu-latest"]' buckets: '["integration/playwright-tests.test.ts"]' + label-tag: relpw run-ff-matrix-release: name: Feature-format matrix against published release @@ -192,6 +196,7 @@ jobs: quarto-install: release quarto-version: ${{ needs.resolve-release.outputs.version }} runners: '["ubuntu-latest", "windows-latest"]' + label-tag: relff # Reuse artifacts from a completed create-release run. resolve-nightly: @@ -262,6 +267,7 @@ jobs: quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["ubuntu-latest"]' buckets: ${{ github.event.inputs.buckets }} # empty = full run + label-tag: ngtsmk run-smokes-nightly-windows: name: Smoke tests against nightly build (windows) @@ -278,6 +284,7 @@ jobs: quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["windows-latest"]' buckets: ${{ github.event.inputs.buckets }} # empty = full run + label-tag: ngtsmk run-smokes-nightly-mac: name: Smoke tests against nightly build (macOS) @@ -294,6 +301,7 @@ jobs: quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["macos-latest"]' buckets: ${{ github.event.inputs.buckets }} # empty = full run + label-tag: ngtsmk # Each caller job accepts one OS-specific artifact. run-playwright-nightly-linux: @@ -311,6 +319,7 @@ jobs: quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["ubuntu-latest"]' buckets: '["integration/playwright-tests.test.ts"]' + label-tag: ngtpw run-playwright-nightly-mac: name: Playwright tests against nightly build (macOS) @@ -327,6 +336,7 @@ jobs: quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["macos-latest"]' buckets: '["integration/playwright-tests.test.ts"]' + label-tag: ngtpw run-ff-matrix-nightly-linux: name: Feature-format matrix against nightly build (linux) @@ -342,6 +352,7 @@ jobs: quarto-artifact-name: Deb Zip quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["ubuntu-latest"]' + label-tag: ngtff run-ff-matrix-nightly-windows: name: Feature-format matrix against nightly build (windows) @@ -357,3 +368,4 @@ jobs: quarto-artifact-name: Windows Zip quarto-artifact-run-id: ${{ needs.resolve-nightly.outputs.run-id }} runners: '["windows-latest"]' + label-tag: ngtff diff --git a/.github/workflows/test-smokes-parallel.yml b/.github/workflows/test-smokes-parallel.yml index 39b46b2295a..168433b3eb4 100644 --- a/.github/workflows/test-smokes-parallel.yml +++ b/.github/workflows/test-smokes-parallel.yml @@ -104,3 +104,5 @@ jobs: with: buckets: ${{ matrix.buckets.files }} extra-r-packages: ${{ inputs.extra-r-packages }} + # The bucket index distinguishes same-OS matrix jobs. + label-tag: b${{ matrix.buckets.num }} diff --git a/.github/workflows/test-smokes.yml b/.github/workflows/test-smokes.yml index d694798da2f..744c13438af 100644 --- a/.github/workflows/test-smokes.yml +++ b/.github/workflows/test-smokes.yml @@ -49,6 +49,11 @@ on: required: false type: string default: '["ubuntu-latest", "windows-latest"]' + label-tag: + description: "Short alphanumeric tag for distinguishing same-OS jobs in failure summaries" + required: false + type: string + default: "" workflow_dispatch: inputs: buckets: @@ -92,6 +97,8 @@ jobs: runs-on: ${{ matrix.os }} env: PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/ms-playwright-browsers + # Distinguishes this call from other jobs on the same OS. + QUARTO_TESTS_GHA_LABEL_TAG: ${{ inputs.label-tag }} steps: - name: Checkout Repo uses: actions/checkout@v6 @@ -491,6 +498,8 @@ jobs: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUCKETS_JSON: ${{ inputs.buckets }} + # Prevent the harness from duplicating this loop's workflow commands. + QUARTO_TESTS_GHA_ORCHESTRATED: "1" run: | haserror=0 failed_tests=() @@ -540,6 +549,8 @@ jobs: # Useful as TinyTeX latest release is checked in run-test.sh GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} BUCKETS_JSON: ${{ inputs.buckets }} + # Prevent the harness from duplicating this loop's workflow commands. + QUARTO_TESTS_GHA_ORCHESTRATED: "1" run: | $haserror=$false $failed_tests=@() diff --git a/.github/workflows/update-test-timing.yml b/.github/workflows/update-test-timing.yml index e415e7994bd..058b9d3722f 100644 --- a/.github/workflows/update-test-timing.yml +++ b/.github/workflows/update-test-timing.yml @@ -13,6 +13,7 @@ jobs: with: buckets: "" time-test: true + label-tag: timing create-pr: name: Create PR with updated timing file diff --git a/llm-docs/built-version-testing-architecture.md b/llm-docs/built-version-testing-architecture.md index dea19b74a2b..3a963a266ec 100644 --- a/llm-docs/built-version-testing-architecture.md +++ b/llm-docs/built-version-testing-architecture.md @@ -188,7 +188,8 @@ Key points: The feature-format bucket glob (`../dev-docs/feature-format-matrix/qmd-files/**/*.qmd`) is defined only in `test-ff-matrix.yml`. Built-mode callers use its `workflow_call` trigger, while its existing dev triggers remain. -The workflow forwards install, artifact, ref, runner, and R-package inputs to `test-smokes.yml`, with dev defaults for non-call triggers. +The workflow forwards install, artifact, ref, runner, R-package, and `label-tag` inputs to `test-smokes.yml`, with dev defaults for non-call triggers. +The tag defaults to `ffdev` for standalone dev triggers and distinguishes same-OS jobs in the shared failure summary. Reusable-workflow concurrency is evaluated in the caller's context. The group therefore includes a suffix based on `inputs.runners` and `github.run_id`, preventing sibling feature-format legs from canceling one another. Dev triggers use a constant `-dev` suffix. @@ -216,11 +217,17 @@ The trigger fires after every completed create-release run, including manual and **Revisit when:** maintainers want a single nightly build-and-test status and are willing to couple the workflows. -### D2. Version marker: semver *build metadata* (`X.Y.Z+test.YYYYMMDD`) +### D2. Version stamp: the plain three-component version (`X.Y.Z`) -Built test distributions use `$(cat version.txt)+test.$(date +%Y%m%d)`. -Do not use a prerelease suffix, which fails plain `>=X.Y` `quarto-required` ranges, or a fourth numeric component, which is invalid semver. -Build metadata preserves range comparisons while distinguishing the build from the `99.9.9` dev version. +Built test distributions use `$(cat version.txt)`, a plain `X.Y.Z`. Pandoc's +`Version` type, used by the `version` shortcode, cannot parse semver build +metadata such as `+test.YYYYMMDD`; a plain version also matches a release +artifact more closely. The `99.9.9` sentinel still distinguishes dev mode. + +Do not use a prerelease suffix: prerelease versions fail ordinary `>=X.Y` +`quarto-required` ranges. Do not add a fourth numeric component: it is not +valid semver. Identify trial builds by workflow, ref, and SHA instead of the +version string. ### D3. Dist outside the checkout + `99.9.9` sentinel refusal diff --git a/llm-docs/github-actions-test-reporting.md b/llm-docs/github-actions-test-reporting.md new file mode 100644 index 00000000000..2a74a768fe1 --- /dev/null +++ b/llm-docs/github-actions-test-reporting.md @@ -0,0 +1,160 @@ +--- +main_commit: e5850df75 +analyzed_date: 2026-09-15 +key_files: + - .github/workflows/test-smokes.yml + - .github/workflows/test-smokes-parallel.yml + - .github/workflows/test-ff-matrix.yml + - src/tools/github.ts + - tests/test.ts + - tests/gha-grouping.ts + - tests/integration/playwright-tests.test.ts + - tests/tools/check-gha-log.ts +--- + +# GitHub Actions test reporting + +Quarto's test harness groups full-run logs by test file and reports failures +through annotations and the GitHub Actions step summary. Bucketed workflows +already group each bucket and emit bucket-level annotations in their shell +loops, so the harness must not emit competing workflow commands. + +## Ownership + +There are two execution paths: + +- **Harness-owned:** `QUARTO_TESTS_GHA_ORCHESTRATED` is unset. The harness + emits groups, annotations, and step-summary entries. This does not imply a + single `deno test` process: `run-tests.sh`'s timing mode (`QUARTO_TEST_TIMING`, + used by `test-smokes.yml`) invokes separate `deno test` processes for test + files, and invokes `smoke-all.test.ts` separately for each smoke-all document. + It leaves the flag unset, so each invocation is independently harness-owned. +- **Orchestrated:** A workflow loop invokes `run-tests` once per file (or + bucket of files) and sets `QUARTO_TESTS_GHA_ORCHESTRATED=1`. The loop owns + groups and bucket-level annotations; the harness emits only step-summary + entries. A direct test runner can still emit its own file-specific + annotations. + +`harnessOwnsStep()` in `src/tools/github.ts` implements this distinction. +Workflow commands are otherwise gated by `GITHUB_ACTIONS`. + +Step-summary entries are emitted in both paths. A bucketed step can contain many +harness processes, all appending to the same summary file. + +## Log grouping + +GitHub Actions groups cannot nest. Starting a group while another is open +implicitly closes the first, and the later `::endgroup::` leaves subsequent +output ungrouped. Each execution path must therefore have one group owner. + +For harness-owned runs: + +1. The first `test()` registration in a file opens the group before Deno prints + that file's reporter header. The file URL is recovered from the V8 call + stack. +2. The test body opens or corrects the group from `context.origin`, which is + authoritative. This is also the fallback when stack parsing fails. +3. A failure closes the group before teardown, annotation output, and Deno's + `FAILED` line. +4. The file's `unload` handler closes a passing file's group before Deno prints + the next file or the final `ERRORS` and `FAILURES` sections. + +Deno creates a separate module graph for each test file and fires `unload` once +per file. Module-level grouping and summary state is therefore per file, not per +`deno test` process. + +Workflow command markers must start at column 0 and end with a newline. ANSI +color is valid in group bodies but must be removed from annotations and step +summaries. + +## Failure reporting + +`tests/test.ts` reports each harness failure once, after cleanup and teardown. +When a primary test failure and a teardown or cleanup failure both occur, the +report includes both while preserving the primary exception. + +Each failure contributes: + +- A summary table row with a short navigation label, file, test name, and + duration. +- A clustered detail block containing the reproduction command and a bounded + output excerpt. +- A `::error` annotation when the harness owns the step and the annotation + budget allows it. + +Rows are written immediately. Detail blocks are buffered until the file's +`unload` event because GitHub-flavored Markdown ends a table at the first +non-row line. Failures with the same first three non-empty excerpt lines share +one detail block. `pendingClusters` is a module-level map, and Deno gives each +test-file execution its own module graph, so this clustering only merges +failures within one module instance, not across the step. + +Summary writes are best-effort and limited to 512 KiB, leaving margin below +GitHub's 1 MiB per-step limit. Full rows degrade to name-only rows when +necessary. If no candidate fits, the harness emits one truncation notice and +leaves the complete failure record in the step log. Individual excerpts are also +byte-limited so one long line cannot consume the summary budget. + +## Annotation budget and labels + +GitHub displays at most 10 error annotations per step. The harness emits nine +per-test annotations, then one aggregate annotation when the tenth failure is +recorded. + +The count must span test-file module instances. `AnnotationBudget` stores it in +a sidecar file derived from `GITHUB_STEP_SUMMARY`, which is unique to the step +and writable by the runner. Tests run serially, so no locking is needed. If the +sidecar cannot be read or written, reporting remains best-effort and falls back +to local counting. + +Failure labels combine: + +- `RUNNER_OS`, reduced to `L`, `W`, `M`, or `X`; +- `QUARTO_TESTS_GHA_LABEL_TAG`, supplied by the reusable-workflow caller; +- the step-wide failure ordinal. + +The run summary concatenates summaries from multiple jobs. Every reusable +workflow call that can share an OS with another call in the same run must +therefore pass a distinct `label-tag`. Per-OS jobs may share a tag because the +OS prefix already distinguishes them. + +## Workflow contract + +When editing the smoke workflows: + +- Set `QUARTO_TESTS_GHA_ORCHESTRATED=1` on loops that emit their own `::group::` + or `::error` commands. +- Pass `label-tag` at every `test-smokes.yml` and `test-ff-matrix.yml` call + site. +- Keep tags short and alphanumeric. `failureLabel()` discards other characters + and keeps at most eight. +- Do not add `deno test --parallel`; interleaved workflow commands would corrupt + grouping. Parallel support would require buffered, atomic output. + +## Verification + +Unit coverage is in: + +- `tests/unit/gha-grouping.test.ts` +- `tests/unit/github-actions-reporting.test.ts` +- `tests/unit/harness-failure-reporting.test.ts` +- `tests/unit/check-gha-log.test.ts` + +`tests/tools/check-gha-log.ts` validates captured logs for balanced, non-nested +groups, column-zero markers, and failure sections outside groups. Run it after +changing the grouping logic or upgrading Deno: + +```bash +GITHUB_ACTIONS=true ./run-tests.sh | tee log.txt +deno run --allow-read tests/tools/check-gha-log.ts log.txt +``` + +Direct `Deno.test` files that bypass the Quarto harness (e.g. +`playwright-tests.test.ts`) receive none of the harness's own grouping, +annotation, or summary logic. They can still use file-specific reporting or be +wrapped by workflow-level reporting. On GitHub Actions, +`tests/integration/playwright-tests.test.ts` emits its own annotations; when a +bucket loop owns the step, it calls `gha.error` directly instead of using the +step-wide annotation budget. Bucket loops in `test-smokes.yml` also wrap each +bucket in their own `::group::`/`::endgroup::` pair and emit a bucket-level +failure annotation regardless of which files the bucket runs. diff --git a/src/resources/extensions/quarto/version/version.lua b/src/resources/extensions/quarto/version/version.lua index d957070c79d..4f6d8d6890a 100644 --- a/src/resources/extensions/quarto/version/version.lua +++ b/src/resources/extensions/quarto/version/version.lua @@ -1,5 +1,5 @@ return { ['version'] = function(args, kwargs, meta) - return table.concat(quarto.version, '.') + return tostring(quarto.version) end } diff --git a/src/tools/github.ts b/src/tools/github.ts index 947bbdc8cd2..83f81523c30 100644 --- a/src/tools/github.ts +++ b/src/tools/github.ts @@ -141,6 +141,377 @@ export async function group( return await withGroupAsync(title, fn); } +// Test reporting helpers. See llm-docs/github-actions-test-reporting.md. + +// Annotations and step summaries do not render ANSI escapes. +// deno-lint-ignore no-control-regex +const kAnsiPattern = /\x1b\[[0-9;?]*[A-Za-z]/g; +export function stripAnsi(s: string): string { + return s.replace(kAnsiPattern, ""); +} + +// The harness owns workflow commands on CI unless an outer loop claims them. +// Omitting `orchestrated` reads the environment; `null` treats it as unset. +export function harnessOwnsStep( + githubActions: boolean = isGitHubActions(), + orchestrated?: string | null, +): boolean { + const o = orchestrated === undefined + ? Deno.env.get("QUARTO_TESTS_GHA_ORCHESTRATED") + : orchestrated; + return githubActions && !o; +} + +// Deno module state is per test file, so persist the step-wide annotation +// count beside the runner-provided summary file. +export function defaultAnnotationCounterPath(): string | undefined { + const summary = Deno.env.get("GITHUB_STEP_SUMMARY"); + return summary ? `${summary}.qt-annotation-count` : undefined; +} + +export interface AnnotationDecision { + // Step-wide, one-based failure count. + ordinal: number; + // Emit an annotation for this failure. + emitAnnotation: boolean; + // Emit the single aggregate annotation after the per-test limit. + emitAggregate: boolean; +} + +// Omitting `counterPath` uses the step sidecar; `null` uses local state. +export class AnnotationBudget { + private localCount = 0; + // Continue monotonically in this instance after persistence fails. + private persistenceFailed = false; + private readonly counterPath: string | null; + constructor( + private readonly max = 9, + counterPath?: string | null, + ) { + this.counterPath = counterPath === undefined + ? defaultAnnotationCounterPath() ?? null + : counterPath; + } + + private readCount(): number { + if (this.counterPath === null || this.persistenceFailed) { + return this.localCount; + } + try { + return parseInt(Deno.readTextFileSync(this.counterPath), 10) || 0; + } catch { + return 0; + } + } + + private writeCount(n: number): void { + this.localCount = n; + if (this.counterPath === null) { + return; + } + try { + Deno.writeTextFileSync(this.counterPath, String(n)); + } catch { + // Reporting is best-effort and must not replace the original failure. + this.persistenceFailed = true; + } + } + + // The first failure after the per-test limit emits the aggregate annotation. + recordFailure(): AnnotationDecision { + const count = this.readCount() + 1; + this.writeCount(count); + return { + ordinal: count, + emitAnnotation: count <= this.max, + emitAggregate: count === this.max + 1, + }; + } +} + +// Leave margin below GitHub's 1 MiB per-step limit (actions/runner#4337). +export const kStepSummaryBudgetBytes = 512 * 1024; + +// Omitting `path` uses GITHUB_STEP_SUMMARY; `null` or "" disables the write. +// Returns false when the path is unavailable or the best-effort write fails. +export function stepSummary( + markdown: string, + path?: string | null, +): boolean { + const p = path === undefined ? Deno.env.get("GITHUB_STEP_SUMMARY") : path; + if (!p) return false; + try { + Deno.writeTextFileSync(p, markdown, { append: true }); + return true; + } catch { + return false; + } +} + +// Return zero when the summary path is unset, missing, or unreadable. +export function stepSummarySize( + path?: string | null, +): number { + const p = path === undefined ? Deno.env.get("GITHUB_STEP_SUMMARY") : path; + if (!p) return 0; + try { + return Deno.statSync(p).size; + } catch { + return 0; + } +} + +// A line count alone cannot bound serialized data or other very long lines. +export const kExcerptMaxBytes = 8 * 1024; + +function utf8ByteLength(s: string): number { + return new TextEncoder().encode(s).length; +} + +// Drop replacement characters produced by slicing through a code point. +function decodeUtf8Prefix(encoded: Uint8Array): string { + return new TextDecoder().decode(encoded).replace(/�+$/, ""); +} + +// Truncate to at most `maxBytes` of UTF-8, including the marker. +export function truncateUtf8Bytes(s: string, maxBytes: number): string { + const encoded = new TextEncoder().encode(s); + if (encoded.length <= maxBytes) return s; + const marker = "\n…[truncated]"; + const markerBytes = utf8ByteLength(marker); + if (maxBytes <= markerBytes) { + return decodeUtf8Prefix( + new TextEncoder().encode(marker).subarray(0, Math.max(0, maxBytes)), + ); + } + const budget = maxBytes - markerBytes; + return decodeUtf8Prefix(encoded.subarray(0, budget)) + marker; +} + +// The step log remains the complete record when summary content is dropped. +export const kStepSummaryTruncationNotice = + "\n**Step summary truncated. See the step log's ERRORS/FAILURES sections for all failures.**\n"; + +// Reserve room for one truncation notice. +const kStepSummaryContentBudgetBytes = kStepSummaryBudgetBytes - + utf8ByteLength(kStepSummaryTruncationNotice); + +function fitsInStepSummary( + markdown: string, + limitBytes: number, + path: string, +): boolean { + return stepSummarySize(path) + utf8ByteLength(markdown) <= limitBytes; +} + +// Append only when the content fits. The first refusal adds one notice. +export function appendStepSummaryBounded( + markdown: string, + path?: string | null, +): boolean { + const p = path === undefined ? Deno.env.get("GITHUB_STEP_SUMMARY") : path; + if (!p) return false; + if (appendIfFits(markdown, p)) return true; + emitTruncationNotice(p); + return false; +} + +function appendIfFits(markdown: string, path: string): boolean { + if (!fitsInStepSummary(markdown, kStepSummaryContentBudgetBytes, path)) { + return false; + } + return stepSummary(markdown, path); +} + +// The file content coordinates this across test-file module instances. +function emitTruncationNotice(path: string): void { + let existing = ""; + try { + existing = Deno.readTextFileSync(path); + } catch { + // Continue with the write attempt. + } + if ( + !existing.includes(kStepSummaryTruncationNotice) && + fitsInStepSummary( + kStepSummaryTruncationNotice, + kStepSummaryBudgetBytes, + path, + ) + ) { + stepSummary(kStepSummaryTruncationNotice, path); + } +} + +// Try candidates in order before writing the notice, so the notice cannot +// crowd out a smaller fallback. Return the written index, or -1. +export function appendStepSummaryFirstFit( + candidates: string[], + path?: string | null, +): number { + const p = path === undefined ? Deno.env.get("GITHUB_STEP_SUMMARY") : path; + if (!p) return -1; + for (let i = 0; i < candidates.length; i++) { + if (appendIfFits(candidates[i], p)) { + if (i > 0) emitTruncationNotice(p); + return i; + } + } + emitTruncationNotice(p); + return -1; +} + +function htmlEscape(s: string): string { + return s + .replaceAll("&", "&") + .replaceAll("<", "<") + .replaceAll(">", ">"); +} + +// Escape backslashes first so they cannot neutralize later Markdown escapes. +function summaryCell(s: string): string { + return htmlEscape(s) + .replaceAll("\\", "\\\\") + .replaceAll("`", "\\`") + .replaceAll("_", "\\_") + .replaceAll("*", "\\*") + .replaceAll("|", "\\|") + .replace(/\r?\n/g, " "); +} + +function formatDuration(ms: number): string { + return `${(ms / 1000).toFixed(2)}s`; +} + +const kMaxLabelTagChars = 8; + +// Labels are search targets shared by rows, detail blocks, and annotations. +// The workflow tag distinguishes jobs that share an OS and ordinal. +export function failureLabel( + ordinal: number, + runnerOs: string, + tag = "", +): string { + const os = runnerOs.toLowerCase(); + const prefix = os.startsWith("linux") + ? "L" + : os.startsWith("windows") + ? "W" + : os.startsWith("macos") + ? "M" + : "X"; + const jobTag = tag.replace(/[^A-Za-z0-9]/g, "").slice(0, kMaxLabelTagChars); + return `${prefix}${jobTag}-F${ordinal}`; +} + +// Use enough lines to avoid grouping failures with only a generic first line. +export function excerptSignature(excerpt: string): string { + return stripAnsi(excerpt) + .split("\n") + .filter((line) => line.trim().length > 0) + .slice(0, 3) + .join("\n"); +} + +export function summaryTableHeader(): string { + return "| # | Test file | Test | Duration |\n| :-- | :-- | :-- | --: |\n"; +} + +export function summaryTableRow( + label: string, + file: string, + name: string, + durationMs: number, +): string { + return `| ${label} | \`${file}\` | ${summaryCell(name)} | ${ + formatDuration(durationMs) + } |\n`; +} + +// Preserve the failure identity when a full row no longer fits. +export function summaryTableRowNameOnly( + label: string, + file: string, + name: string, +): string { + return `| ${label} | \`${file}\` | ${summaryCell(name)} | |\n`; +} + +export interface ClusterMember { + label: string; + file: string; + testName: string; + repro: string; +} + +export interface FailureCluster { + label: string; + members: ClusterMember[]; + excerpt: string; +} + +// Render one escaped excerpt for failures that share a signature. +export function summaryClusterBlock(cluster: FailureCluster): string { + const first = cluster.members[0]; + const n = cluster.members.length; + const count = n > 1 ? ` (${n} tests)` : ""; + const summaryLabel = `${htmlEscape(first.file)} — ${ + htmlEscape(first.testName).replace(/\r?\n/g, " ") + }${count}`; + let memberList = ""; + if (n > 1) { + memberList = "\n" + cluster.members + .map((m) => + `- ${m.label} · ${htmlEscape(m.file)} (${ + htmlEscape(m.repro) + })` + ) + .join("\n") + + "\n"; + } + const body = htmlEscape(`${first.repro}\n\n${cluster.excerpt}`); + return `\n#### ${cluster.label}\n\n
${summaryLabel}\n${memberList}\n
\n${body}\n
\n
\n\n`; +} + +// Whether the summary has a full row, a name-only row, or no row. A queued +// detail block may still be dropped when it is flushed under the shared cap. +export type SummaryRowOutcome = "detail" | "name-only" | "none"; + +// Exported so callers can reserve lines for teardown or cleanup failures. +export const kAnnotationExcerptLines = 5; + +// Build a bounded annotation with an accurate summary or log pointer. +export function annotationBody( + repro: string, + excerpt: string, + label: string, + outcome: SummaryRowOutcome, + maxLines = kAnnotationExcerptLines, +): string { + const allLines = stripAnsi(truncateUtf8Bytes(excerpt, kExcerptMaxBytes)) + .split("\n") + .filter((line) => line.trim().length > 0); + const truncated = allLines.length > maxLines; + const lines = allLines.slice(0, maxLines); + let pointer: string; + switch (outcome) { + case "detail": + pointer = + `Step summary: ${label}. Details may be truncated; see the step log for the complete failure.`; + break; + case "name-only": + pointer = + `Step summary: ${label} (name only). See the step log for details.`; + break; + case "none": + pointer = + `Not included in the step summary because it exceeded the size limit. See the step log for details.`; + break; + } + const ellipsis = truncated ? "\n…" : ""; + return `${repro}\n\n${lines.join("\n")}${ellipsis}\n${pointer}`; +} + // GitHub API // A Github Release for a Github Repo diff --git a/tests/README.md b/tests/README.md index 4ada5d9bc39..7f6dbcaf6ba 100644 --- a/tests/README.md +++ b/tests/README.md @@ -467,7 +467,7 @@ To run in binary mode locally: ```bash # 1. Build a distribution (after ./configure.sh) cd package/src -./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)+test.$(date +%Y%m%d)" +./quarto-bld prepare-dist --set-version "$(cat ../../version.txt)" cd ../.. # 2. Copy the distribution outside the checkout. An in-repo launcher uses diff --git a/tests/gha-grouping.ts b/tests/gha-grouping.ts new file mode 100644 index 00000000000..e0c8bb8971a --- /dev/null +++ b/tests/gha-grouping.ts @@ -0,0 +1,61 @@ +/* + * gha-grouping.ts + * + * Per-file GitHub Actions grouping for harness-owned test runs. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { endGroup, harnessOwnsStep, startGroup } from "../src/tools/github.ts"; + +// Injected emitters keep the state machine independent of the environment. +export class GroupEmitter { + private openFile: string | undefined = undefined; + + constructor( + private readonly gate: () => boolean, + private readonly onStart: (title: string) => void, + private readonly onEnd: () => void, + ) {} + + // Keep at most one group open and reuse it across a file's tests. + enterFile(file: string): void { + if (!this.gate()) return; + if (this.openFile === file) return; + if (this.openFile !== undefined) this.onEnd(); + this.onStart(file); + this.openFile = file; + } + + // Closing an already closed emitter is a no-op. + close(): void { + if (!this.gate()) return; + if (this.openFile !== undefined) { + this.onEnd(); + this.openFile = undefined; + } + } + + currentFile(): string | undefined { + return this.openFile; + } +} + +// Return the first `.test.ts` file URL in a V8 stack, without its location. +export function testFileUrlFromStack( + stack: string | undefined, +): string | undefined { + if (stack === undefined) return undefined; + const match = stack.match(/file:\/\/\/?\S*?\.test\.ts/); + return match?.[0]; +} + +const emitter = new GroupEmitter(harnessOwnsStep, startGroup, endGroup); + +export function enterTestFileGroup(file: string): void { + emitter.enterFile(file); +} + +export function closeTestFileGroup(): void { + emitter.close(); +} diff --git a/tests/integration/playwright-tests.test.ts b/tests/integration/playwright-tests.test.ts index dbf191095b3..a6dfb7a1174 100644 --- a/tests/integration/playwright-tests.test.ts +++ b/tests/integration/playwright-tests.test.ts @@ -2,7 +2,6 @@ * playwright-tests.test.ts * * Copyright (C) 2022 Posit Software, PBC - * */ import { expandGlobSync } from "../../src/core/deno/expand-glob.ts"; @@ -21,6 +20,29 @@ import { join, relative } from "../../src/deno_ral/path.ts"; import { existsSync } from "../../src/deno_ral/fs.ts"; import * as gha from "../../src/tools/github.ts"; +// Share the step-wide annotation budget when the harness owns this step. +const annotationBudget = new gha.AnnotationBudget(); + +function reportFailure( + message: string, + properties?: gha.AnnotationProperties, +) { + if (!gha.harnessOwnsStep()) { + gha.error(message, properties); + return; + } + const decision = annotationBudget.recordFailure(); + if (decision.emitAnnotation) { + gha.error(message, properties); + } else if (decision.emitAggregate) { + gha.error( + "Additional failures were not annotated because GitHub limits " + + "annotations per step. See the step log for all failures.", + { title: "More test failures" }, + ); + } +} + async function fullInit() { await initYamlIntelligenceResourcesFromFilesystem(); } @@ -57,8 +79,8 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { { pathSuffix: "docs/playwright/embed-resources/issue-11860/main.qmd", options: ["--output-dir=inner"], - } - ] + }, + ]; for (const { path: fileName } of globOutput) { const input = relative(Deno.cwd(), fileName); @@ -83,7 +105,7 @@ if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { }); if (!result.success) { - gha.error(`Failed to render ${input}`) + reportFailure(`Failed to render ${input}`); if (result.stdout) console.log(result.stdout); if (result.stderr) console.error(result.stderr); throw new Error(`Render failed with code ${result.code}`); @@ -100,38 +122,49 @@ Deno.test({ fn: async () => { try { // run playwright - const res = await execProcess({ - cmd: isWindows ? "npx.cmd" : "npx", - args: ["playwright", "test", "--ignore-snapshots"], - cwd: "integration/playwright", - }, - undefined, // stdin - undefined, // mergeOutput - undefined, // stderrFilter - true // respectStreams - write directly to stderr/stdout + const res = await execProcess( + { + cmd: isWindows ? "npx.cmd" : "npx", + args: ["playwright", "test", "--ignore-snapshots"], + cwd: "integration/playwright", + }, + undefined, // stdin + undefined, // mergeOutput + undefined, // stderrFilter + true, // respectStreams - write directly to stderr/stdout ); if (!res.success) { - if (gha.isGitHubActions() && Deno.env.get("GITHUB_REPOSITORY") && Deno.env.get("GITHUB_RUN_ID")) { - const runUrl = `https://github.com/${Deno.env.get("GITHUB_REPOSITORY")}/actions/runs/${Deno.env.get("GITHUB_RUN_ID")}`; - gha.error( + if ( + gha.isGitHubActions() && Deno.env.get("GITHUB_REPOSITORY") && + Deno.env.get("GITHUB_RUN_ID") + ) { + const runUrl = `https://github.com/${ + Deno.env.get("GITHUB_REPOSITORY") + }/actions/runs/${Deno.env.get("GITHUB_RUN_ID")}`; + reportFailure( `Some tests failed. Download report uploaded as artifact at ${runUrl}`, { file: "playwright-tests.test.ts", - title: "Playwright tests" - } + title: "Playwright tests", + }, ); } - fail("Failed tests with playwright. Look at playwright report for more details.") + fail( + "Failed tests with playwright. Look at playwright report for more details.", + ); } - } finally { // skip cleanoutput if requested - if (Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_CLEANOUTPUT") === "true" || Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true") { + if ( + Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_CLEANOUTPUT") === "true" || + Deno.env.get("QUARTO_PLAYWRIGHT_TESTS_SKIP_RENDER") === "true" + ) { console.log("Skipping cleanoutput of test documents."); - } else + } else { for (const fileName of fileNames) { cleanoutput(fileName, "html"); } + } } - } + }, }); diff --git a/tests/test.ts b/tests/test.ts index 8f3fcae774f..31d66c3967c 100644 --- a/tests/test.ts +++ b/tests/test.ts @@ -1,23 +1,78 @@ /* -* test.ts -* -* Copyright (C) 2020-2022 Posit Software, PBC -* -*/ + * test.ts + * + * Copyright (C) 2020-2022 Posit Software, PBC + */ import { existsSync, safeRemoveSync } from "../src/deno_ral/fs.ts"; import { AssertionError, fail } from "testing/asserts"; import { warning } from "../src/deno_ral/log.ts"; import { initDenoDom } from "../src/core/deno-dom.ts"; -import { cleanupLogger, initializeLogger, flushLoggers, logError, LogLevel, LogFormat } from "../src/core/log.ts"; +import { + cleanupLogger, + flushLoggers, + initializeLogger, + logError, + LogFormat, + LogLevel, +} from "../src/core/log.ts"; import { appendLogError, isBinaryMode, runQuarto } from "./quarto-cmd.ts"; import { join } from "../src/deno_ral/path.ts"; import * as colors from "fmt/colors"; import { runningInCI } from "../src/core/ci-info.ts"; -import { relative, fromFileUrl } from "../src/deno_ral/path.ts"; +import { fromFileUrl, relative } from "../src/deno_ral/path.ts"; import { quartoConfig } from "../src/core/quarto.ts"; import { isWindows } from "../src/deno_ral/platform.ts"; - +import { + annotationBody, + AnnotationBudget, + appendStepSummaryBounded, + appendStepSummaryFirstFit, + error as ghError, + excerptSignature, + type FailureCluster, + failureLabel, + harnessOwnsStep, + isGitHubActions, + kAnnotationExcerptLines, + kExcerptMaxBytes, + stripAnsi, + summaryClusterBlock, + type SummaryRowOutcome, + summaryTableHeader, + summaryTableRow, + summaryTableRowNameOnly, + truncateUtf8Bytes, +} from "../src/tools/github.ts"; +import { + closeTestFileGroup, + enterTestFileGroup, + testFileUrlFromStack, +} from "./gha-grouping.ts"; + +// GitHub Actions reporting. Deno evaluates this module once per test file, so +// module state is per file; AnnotationBudget persists the step-wide count. +const kExcerptLines = 20; +// Reserve a separator, banner, and one secondary-failure line. +const kFinallyReservedLines = 3; +// Annotations omit the separator from their smaller line budget. +const kFinallyAnnotationReservedLines = 2; +const annotationBudget = new AnnotationBudget(); +let summaryHeaderEmitted = false; +// Opening at registration includes Deno's file header in the group. +let registrationGroupAttempted = false; +// Detail blocks must follow all table rows, so queue per-file clusters. +const pendingClusters = new Map(); + +if (isGitHubActions()) { + globalThis.addEventListener("unload", () => { + // Keep the next file and Deno's final failure sections outside this group. + closeTestFileGroup(); + for (const cluster of pendingClusters.values()) { + if (!appendStepSummaryBounded(summaryClusterBlock(cluster))) break; + } + }); +} export interface TestLogConfig { // Path to log file @@ -44,7 +99,7 @@ export interface TestDescriptor { // type of test type: "smoke" | "unit"; - + // Optional logging configuration logConfig?: TestLogConfig; } @@ -54,7 +109,8 @@ export interface TestContext { // Checks that prereqs for the test are met (async conditional skip) // - Returns false: Test is SKIPPED with warning message (not failed) - // - Throws/rejects: Test is SKIPPED (initialization failed gracefully) + // - Throws/rejects: Test FAILS (propagates to the outer catch as the + // primary failure, same as any other lifecycle error) // Use cases: // - Tool availability checks (e.g., which("rsvg-convert")) // - Initialization that might fail (e.g., schema loading) @@ -96,7 +152,10 @@ export interface TestContext { } // Allow to merge test contexts in Tests helpers -export function mergeTestContexts(baseContext: TestContext, additionalContext?: TestContext): TestContext { +export function mergeTestContexts( + baseContext: TestContext, + additionalContext?: TestContext, +): TestContext { if (!additionalContext) { return baseContext; } @@ -107,7 +166,8 @@ export function mergeTestContexts(baseContext: TestContext, additionalContext?: // combine prereq conditions prereq: async () => { const baseResult = !baseContext.prereq || await baseContext.prereq(); - const additionalResult = !additionalContext.prereq || await additionalContext.prereq(); + const additionalResult = !additionalContext.prereq || + await additionalContext.prereq(); return baseResult && additionalResult; }, // run teardowns in reverse order @@ -124,7 +184,8 @@ export function mergeTestContexts(baseContext: TestContext, additionalContext?: cwd: additionalContext.cwd || baseContext.cwd, // merge sanitize options sanitize: { - resources: additionalContext.sanitize?.resources ?? baseContext.sanitize?.resources, + resources: additionalContext.sanitize?.resources ?? + baseContext.sanitize?.resources, ops: additionalContext.sanitize?.ops ?? baseContext.sanitize?.ops, exit: additionalContext.sanitize?.exit ?? baseContext.sanitize?.exit, }, @@ -198,7 +259,15 @@ export function unitTest( name: `${name}`, verify: async (_outputs: ExecuteOutput[]) => { const timeout = new Promise((_resolve, reject) => { - setTimeout(() => reject(new AssertionError(`timed out after 2 minutes. Something may be wrong with verify function in the test '${name}'.`)), 120000); + setTimeout( + () => + reject( + new AssertionError( + `timed out after 2 minutes. Something may be wrong with verify function in the test '${name}'.`, + ), + ), + 120000, + ); }); await Promise.race([ver(), timeout]); }, @@ -207,7 +276,226 @@ export function unitTest( }); } +// Resolve an origin to an absolute path and a path relative to `tests/`. +// Bare `deno test` runs may lack QUARTO_BIN_PATH, so fall back to Deno.cwd(). +export function testFileFromOrigin( + origin: string, + resolveBinPath: () => string = quartoConfig.binPath, +): { + absPath: string; + relPath: string; +} { + const absPath = isWindows ? fromFileUrl(origin) : (new URL(origin)).pathname; + let relPath: string; + try { + const quartoRoot = join(resolveBinPath(), "..", "..", ".."); + relPath = relative(join(quartoRoot, "tests"), absPath); + } catch { + relPath = relative(Deno.cwd(), absPath); + } + return { absPath, relPath }; +} + +// context.origin is unavailable during registration; use the first test-file +// stack frame. The test body later corrects or opens the authoritative group. +function testFileFromStack(): string | undefined { + const url = testFileUrlFromStack(new Error().stack); + if (url === undefined) return undefined; + return testFileFromOrigin(url).relPath.replaceAll("\\", "/"); +} + +const kTeardownBanner = "TEARDOWN ALSO FAILED:"; + +const kCleanupBanner = "CLEANUP ALSO FAILED:"; + +function bannerFor(phase: "teardown" | "cleanup"): string { + return phase === "teardown" ? kTeardownBanner : kCleanupBanner; +} + +// Tests may throw values other than Error. +function describeThrow(value: unknown): { message: string; stack: string } { + if (value instanceof Error) { + return { message: value.message, stack: value.stack ?? "" }; + } + return { message: String(value), stack: "" }; +} + +// Boxing preserves a thrown `undefined` and identifies the failing phase. +interface Thrown { + value: unknown; + phase: "teardown" | "cleanup"; +} + +interface PrimaryFailure { + value: unknown; + message: string; + stack: string; + logMessages?: ExecuteOutput[]; +} + +interface FailureContext { + testName: string; + origin: string; + testStart: number; +} + +// Report once after teardown. Diagnostics are best-effort and must not replace +// the test failure already in flight. +function reportFailure( + primary: PrimaryFailure | undefined, + finallyFailure: Thrown | undefined, + ctx: FailureContext, +): void { + if (!isGitHubActions()) return; + try { + const fwd = (p: string) => p.replaceAll("\\", "/"); + const { absPath, relPath } = testFileFromOrigin(ctx.origin); + const command = isWindows ? "./run-tests.ps1" : "./run-tests.sh"; + // Smoke-all failures should navigate to the document, not the harness. + let reproPath = fwd(relPath); + if (fwd(absPath).endsWith("/smoke/smoke-all.test.ts")) { + const m = fwd(ctx.testName).match( + /(\S+\.(?:qmd|ipynb|md))(?=\s|$)/, + ); + if (m) { + reproPath = m[1]; + } + } + const annotationFile = `tests/${reproPath}`; + const repro = `${command} ${reproPath}`; + + const rawExcerpt: string[] = []; + if (primary) { + // Cap bytes before lines so the truncation marker uses a reserved line. + rawExcerpt.push( + finallyFailure + ? truncateUtf8Bytes(primary.message, kExcerptMaxBytes / 2) + .split("\n") + .slice(0, kExcerptLines - kFinallyReservedLines) + .join("\n") + : primary.message, + ); + } + if (finallyFailure) { + if (rawExcerpt.length > 0) { + rawExcerpt.push(""); + } + rawExcerpt.push( + bannerFor(finallyFailure.phase), + describeThrow(finallyFailure.value).message, + ); + } + if (primary?.stack) { + rawExcerpt.push(primary.stack); + } else if (finallyFailure) { + // A teardown-only failure needs its own source location. + const stack = describeThrow(finallyFailure.value).stack; + if (stack) rawExcerpt.push(stack); + } + const logMessages = primary?.logMessages; + if (logMessages && logMessages.length > 0) { + rawExcerpt.push("OUTPUT:"); + for (const out of logMessages) { + for (const part of out.msg.split("\n")) { + rawExcerpt.push(" " + part); + } + } + } + const excerpt = truncateUtf8Bytes( + stripAnsi(rawExcerpt.join("\n")) + .split("\n") + .slice(0, kExcerptLines) + .join("\n"), + kExcerptMaxBytes, + ); + + // Use a smaller reserved excerpt so annotations retain secondary failures. + const annotationExcerpt = finallyFailure + ? [ + ...truncateUtf8Bytes( + stripAnsi(primary?.message ?? ""), + kExcerptMaxBytes / 2, + ) + .split("\n") + .filter((line) => line.trim().length > 0) + .slice(0, kAnnotationExcerptLines - kFinallyAnnotationReservedLines), + bannerFor(finallyFailure.phase), + describeThrow(finallyFailure.value).message, + ].join("\n") + : excerpt; + + // All CI paths need a step-wide ordinal for summary navigation. + const decision = annotationBudget.recordFailure(); + const label = failureLabel( + decision.ordinal, + Deno.env.get("RUNNER_OS") ?? "", + Deno.env.get("QUARTO_TESTS_GHA_LABEL_TAG") ?? "", + ); + + // Rows take priority over detail blocks and degrade before being dropped. + if (!summaryHeaderEmitted) { + appendStepSummaryBounded(summaryTableHeader()); + summaryHeaderEmitted = true; + } + const durationMs = Math.round(performance.now() - ctx.testStart); + const rowIndex = appendStepSummaryFirstFit([ + summaryTableRow(label, annotationFile, ctx.testName, durationMs), + summaryTableRowNameOnly(label, annotationFile, ctx.testName), + ]); + const rowOutcome: SummaryRowOutcome = rowIndex === 0 + ? "detail" + : rowIndex === 1 + ? "name-only" + : "none"; + if (rowOutcome === "detail") { + const signature = excerptSignature(excerpt); + const member = { + label, + file: annotationFile, + testName: ctx.testName, + repro, + }; + const existing = pendingClusters.get(signature); + if (existing) { + existing.members.push(member); + } else { + pendingClusters.set(signature, { + label, + members: [member], + excerpt, + }); + } + } + // Bucket loops own their annotations; full runs use the harness budget. + if (harnessOwnsStep()) { + if (decision.emitAnnotation) { + ghError(annotationBody(repro, annotationExcerpt, label, rowOutcome), { + file: annotationFile, + title: `${label} · ${ctx.testName}`, + }); + } else if (decision.emitAggregate) { + ghError( + "Additional failures were not annotated because GitHub limits " + + "annotations per step. See the step log for all failures.", + { title: "More test failures" }, + ); + } + } + } catch { + // Preserve the original failure. + } +} + export function test(test: TestDescriptor) { + // Open early enough to include Deno's file header. + if (!registrationGroupAttempted && harnessOwnsStep()) { + registrationGroupAttempted = true; + const file = testFileFromStack(); + if (file !== undefined) { + enterTestFileGroup(file); + } + } + const testName = test.context.name ? `[${test.type}] > ${test.name} (${test.context.name})` : `[${test.type}] > ${test.name}`; @@ -222,189 +510,240 @@ export function test(test: TestDescriptor) { const args: Deno.TestDefinition = { name: testName, async fn(context) { - await initDenoDom(); - const runTest = !test.context.prereq || await test.context.prereq(); - if (runTest) { - const wd = Deno.cwd(); - - // The child owns log capture in binary mode. - const binMode = isBinaryMode(); - - let cleanedup = false; - const cleanupLogOnce = async () => { - if (!cleanedup && !binMode) { - await cleanupLogger(); - cleanedup = true; - } - }; + // Correct or open the group from the authoritative test origin. + if (harnessOwnsStep()) { + enterTestFileGroup( + testFileFromOrigin(context.origin).relPath.replaceAll("\\", "/"), + ); + } + // Start timing before lifecycle hooks can fail. + const testStart = performance.now(); + let primary: PrimaryFailure | undefined; + // Keep the first cleanup or teardown failure. + let finallyFailure: Thrown | undefined; + try { + await initDenoDom(); + const runTest = !test.context.prereq || await test.context.prereq(); + if (runTest) { + const wd = Deno.cwd(); + + // The child owns log capture in binary mode. + const binMode = isBinaryMode(); + + let cleanedup = false; + const cleanupLogOnce = async () => { + if (!cleanedup && !binMode) { + await cleanupLogger(); + cleanedup = true; + } + }; - let log: string | undefined; - let logTarget: string | undefined; - let handlers: Awaited> | undefined; + let log: string | undefined; + let logTarget: string | undefined; + let handlers: + | Awaited> + | undefined; - const logOutput = (path?: string) => { - if (path && existsSync(path)) { - return readExecuteOutput(path); - } else { - return undefined; - } - }; - let lastVerify; + const logOutput = (path?: string) => { + if (path && existsSync(path)) { + return readExecuteOutput(path); + } else { + return undefined; + } + }; + let lastVerify; + // Delay the decorated failure until teardown has run. + let failureOutput: string | undefined; - try { - // Keep setup and cwd changes inside the cleanup scope. - if (test.context?.cwd) { - Deno.chdir(test.context.cwd()); - } + try { + // Keep setup and cwd changes inside the finalization scope. + if (test.context?.cwd) { + Deno.chdir(test.context.cwd()); + } - if (test.context.setup) { - await test.context.setup(); - } + if (test.context.setup) { + await test.context.setup(); + } - // Capture the output. Started only after setup, so a setup that - // renders its own baseline (e.g. building a freeze cache) doesn't - // attribute its output to the execute() run that verify() inspects. - log = Deno.makeTempFileSync({ suffix: ".json" }); - logTarget = test.logConfig?.log || log; - handlers = binMode ? undefined : await initializeLogger({ - log: logTarget, - level: test.logConfig?.level || "INFO", - format: test.logConfig?.format || "json-stream", - quiet: true, - }); + // Capture the output. Started only after setup, so a setup that + // renders its own baseline (e.g. building a freeze cache) doesn't + // attribute its output to the execute() run that verify() inspects. + log = Deno.makeTempFileSync({ suffix: ".json" }); + logTarget = test.logConfig?.log || log; + handlers = binMode ? undefined : await initializeLogger({ + log: logTarget, + level: test.logConfig?.level || "INFO", + format: test.logConfig?.format || "json-stream", + quiet: true, + }); - try { - await test.execute(logTarget); - } catch (e) { - if (binMode) { - // Append directly because binary mode has no harness logger. - const message = e instanceof Error - ? `${e.message}\n${e.stack ?? ""}` - : String(e); - appendLogError(logTarget, message); - } else { - logError(e); + try { + await test.execute(logTarget); + } catch (e) { + if (binMode) { + // Append directly because binary mode has no harness logger. + const message = e instanceof Error + ? `${e.message}\n${e.stack ?? ""}` + : String(e); + appendLogError(logTarget, message); + } else { + logError(e); + } } - } - // Cleanup the output logging - await cleanupLogOnce(); + // Cleanup the output logging + await cleanupLogOnce(); - if (handlers) { - flushLoggers(handlers); - } + if (handlers) { + flushLoggers(handlers); + } - // Both logging modes write to logTarget; a missing log is a failure. - const testOutput = logOutput(logTarget); - if (testOutput === undefined) { - fail(`test log file is missing: ${logTarget}`); - } else { - for (const ver of test.verify) { - lastVerify = ver; - if (userSession) { - const verifyMsg = "[verify] > " + ver.name; - console.log(userSession ? colors.dim(verifyMsg) : verifyMsg); + // Both logging modes write to logTarget; a missing log is a failure. + const testOutput = logOutput(logTarget); + if (testOutput === undefined) { + fail(`test log file is missing: ${logTarget}`); + } else { + for (const ver of test.verify) { + lastVerify = ver; + if (userSession) { + const verifyMsg = "[verify] > " + ver.name; + console.log(userSession ? colors.dim(verifyMsg) : verifyMsg); + } + await ver.verify(testOutput); } - await ver.verify(testOutput); } - } - } catch (ex) { - if (!(ex instanceof Error)) throw ex; - - const border = "-".repeat(80); - const coloredName = userSession - ? colors.brightGreen(colors.italic(testName)) - : testName; - - // Compute an inset based upon the testName - const offset = testName.indexOf(">"); - - // Form the test runner command - const absPath = isWindows - ? fromFileUrl(context.origin) - : (new URL(context.origin)).pathname; - - const quartoRoot = join(quartoConfig.binPath(), "..", "..", ".."); - const relPath = relative( - join(quartoRoot, "tests"), - absPath, - ); - const command = isWindows - ? "./run-tests.ps1" - : "./run-tests.sh"; - const testCommand = `${ - offset > 0 ? " ".repeat(offset + 2) : "" - }${command} ${relPath}`; - const coloredTestCommand = userSession - ? colors.brightGreen(testCommand) - : testCommand; - - const verifyFailed = `[verify] > ${ - lastVerify ? lastVerify.name : "unknown" - }`; - const coloredVerify = userSession - ? colors.brightGreen(verifyFailed) - : verifyFailed; - - // Preserve the primary failure if the log is malformed. - let logMessages: ExecuteOutput[] | undefined; - try { - logMessages = logOutput(logTarget); - } catch { - logMessages = undefined; - } + } catch (ex) { + // Keep teardown, annotations, and failure output visible. + closeTestFileGroup(); + + const { message, stack } = describeThrow(ex); + + const border = "-".repeat(80); + const coloredName = userSession + ? colors.brightGreen(colors.italic(testName)) + : testName; + + // Compute an inset based upon the testName + const offset = testName.indexOf(">"); + + // Form the test runner command + const { relPath } = testFileFromOrigin(context.origin); + const command = isWindows ? "./run-tests.ps1" : "./run-tests.sh"; + const testCommand = `${ + offset > 0 ? " ".repeat(offset + 2) : "" + }${command} ${relPath}`; + const coloredTestCommand = userSession + ? colors.brightGreen(testCommand) + : testCommand; + + const verifyFailed = `[verify] > ${ + lastVerify ? lastVerify.name : "unknown" + }`; + const coloredVerify = userSession + ? colors.brightGreen(verifyFailed) + : verifyFailed; + + // Preserve the primary failure if the log is malformed. + let logMessages: ExecuteOutput[] | undefined; + try { + logMessages = logOutput(logTarget); + } catch { + logMessages = undefined; + } - // Create distinctive failure marker for easy log navigation - // This helps users find the failure when clicking GitHub Actions annotations - const failureMarker = `━━━ TEST FAILURE: ${testName}`; - const coloredFailureMarker = userSession - ? colors.red(colors.bold(failureMarker)) - : failureMarker; - - const output: string[] = [ - "", - "", - coloredFailureMarker, - border, - coloredName, - coloredTestCommand, - "", - coloredVerify, - "", - ex.message, - ex.stack ?? "", - "", - ]; - - if (logMessages && logMessages.length > 0) { - output.push("OUTPUT:"); - logMessages.forEach((out) => { - const parts = out.msg.split("\n"); - parts.forEach((part) => { - output.push(" " + part); + // Add a stable log navigation marker. + const failureMarker = `━━━ TEST FAILURE: ${testName}`; + const coloredFailureMarker = userSession + ? colors.red(colors.bold(failureMarker)) + : failureMarker; + + const output: string[] = [ + "", + "", + coloredFailureMarker, + border, + coloredName, + coloredTestCommand, + "", + coloredVerify, + "", + message, + stack, + "", + ]; + + if (logMessages && logMessages.length > 0) { + output.push("OUTPUT:"); + logMessages.forEach((out) => { + const parts = out.msg.split("\n"); + parts.forEach((part) => { + output.push(" " + part); + }); }); - }); + } + + primary = { value: ex, message, stack, logMessages }; + failureOutput = output.join("\n"); + } finally { + // Capture cleanup failures so teardown and cwd restoration still run. + try { + if (log) { + safeRemoveSync(log); + } + } catch (e) { + finallyFailure ??= { value: e, phase: "cleanup" }; + } + try { + await cleanupLogOnce(); + } catch (e) { + finallyFailure ??= { value: e, phase: "cleanup" }; + } + // Restore cwd even when teardown fails. + try { + if (test.context.teardown) { + await test.context.teardown(); + } + } catch (e) { + finallyFailure ??= { value: e, phase: "teardown" }; + } finally { + if (test.context?.cwd) { + Deno.chdir(wd); + } + } } - fail(output.join("\n")); - } finally { - if (log) { - safeRemoveSync(log); + if (failureOutput !== undefined && finallyFailure === undefined) { + fail(failureOutput); } - await cleanupLogOnce(); - // Restore the cwd even when teardown fails. - try { - if (test.context.teardown) { - await test.context.teardown(); - } - } finally { - if (test.context?.cwd) { - Deno.chdir(wd); + if (finallyFailure !== undefined) { + if (failureOutput === undefined) { + throw finallyFailure.value; } + // Report the primary and finalization failures together. + const combined = new AssertionError( + `${failureOutput}\n\n${bannerFor(finallyFailure.phase)}\n${ + describeThrow(finallyFailure.value).message + }`, + ); + combined.cause = finallyFailure.value; + throw combined; } + } else { + warning(`Skipped - ${test.name}`); } - } else { - warning(`Skipped - ${test.name}`); + } catch (e) { + // Lifecycle failures can bypass the inner close. + closeTestFileGroup(); + if (primary === undefined && finallyFailure === undefined) { + const { message, stack } = describeThrow(e); + primary = { value: e, message, stack }; + } + reportFailure(primary, finallyFailure, { + testName, + origin: context.origin, + testStart, + }); + throw e; } }, ignore, diff --git a/tests/tools/check-gha-log.ts b/tests/tools/check-gha-log.ts new file mode 100644 index 00000000000..741aabae349 --- /dev/null +++ b/tests/tools/check-gha-log.ts @@ -0,0 +1,134 @@ +#!/usr/bin/env -S deno run --allow-read +/* + * check-gha-log.ts + * + * Checks captured GitHub Actions logs for harness grouping violations: + * + * 1. Groups are balanced and do not nest. + * 2. Workflow command markers start at column 0. + * 3. Harness `FAILED` lines and final failure sections remain ungrouped. + * + * Usage: + * GITHUB_ACTIONS=true ./run-tests.sh | tee log.txt + * deno run --allow-read tests/tools/check-gha-log.ts log.txt + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { stripAnsi } from "../../src/tools/github.ts"; + +export interface Violation { + line: number; + message: string; + text: string; +} + +const kGroupOpen = "::group::"; +const kGroupClose = "::endgroup::"; +// A harness test name is `[smoke] > ...` / `[unit] > ...` (see test() in +// tests/test.ts); Deno prints its result as ` ... FAILED ()`. +const kHarnessFailed = /^\[(smoke|unit)\] > .* \.\.\. FAILED/; +const kSectionHeader = /^(ERRORS|FAILURES)$/; +const kMarkerAnywhere = /::(group|endgroup)::/; + +export function checkLog(content: string): Violation[] { + const violations: Violation[] = []; + const lines = content.split("\n"); + + let depth = 0; + let openGroupLine = 0; + let openGroupTitle = ""; + + lines.forEach((raw, i) => { + const n = i + 1; + const line = stripAnsi(raw).replace(/\r$/, ""); + + const isGroupOpen = line.startsWith(kGroupOpen); + // The runner accepts trailing whitespace on a close marker. + const isGroupClose = line.trimEnd() === kGroupClose; + + // Indented markers are visible text, not workflow commands. + if (!isGroupOpen && !isGroupClose && kMarkerAnywhere.test(line)) { + violations.push({ + line: n, + message: "group marker not at column 0", + text: raw, + }); + return; + } + + if (isGroupOpen) { + if (depth > 0) { + violations.push({ + line: n, + message: `nested ::group:: — group opened at line ${openGroupLine} ` + + `(${openGroupTitle}) was not closed first`, + text: raw, + }); + } + depth++; + openGroupLine = n; + openGroupTitle = line.slice(kGroupOpen.length); + return; + } + + if (isGroupClose) { + // A stray close marker is harmless. + if (depth > 0) depth--; + return; + } + + if (depth > 0) { + if (kHarnessFailed.test(line)) { + violations.push({ + line: n, + message: + `harness FAILED result line inside group ${openGroupTitle} ` + + `(opened at line ${openGroupLine})`, + text: raw, + }); + } else if (kSectionHeader.test(line.trim())) { + violations.push({ + line: n, + message: + `${line.trim()} section header inside group ${openGroupTitle} ` + + `(opened at line ${openGroupLine})`, + text: raw, + }); + } + } + }); + + if (depth > 0) { + violations.push({ + line: lines.length, + message: `group left open at end of log: ${openGroupTitle} ` + + `(opened at line ${openGroupLine})`, + text: "", + }); + } + + return violations; +} + +if (import.meta.main) { + const path = Deno.args[0]; + if (!path) { + console.error("usage: check-gha-log.ts "); + Deno.exit(2); + } + const content = Deno.readTextFileSync(path); + const violations = checkLog(content); + if (violations.length === 0) { + console.log(`OK: ${path} has valid GitHub Actions grouping.`); + Deno.exit(0); + } + console.error( + `FAIL: ${path} has ${violations.length} grouping violation(s):`, + ); + for (const v of violations) { + console.error(` line ${v.line}: ${v.message}`); + if (v.text) console.error(` | ${v.text}`); + } + Deno.exit(1); +} diff --git a/tests/unit/check-gha-log.test.ts b/tests/unit/check-gha-log.test.ts new file mode 100644 index 00000000000..100c9218bee --- /dev/null +++ b/tests/unit/check-gha-log.test.ts @@ -0,0 +1,82 @@ +/* + * check-gha-log.test.ts + * + * Tests for the GitHub Actions log-grouping checker. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { unitTest } from "../test.ts"; +import { assertEquals } from "testing/asserts"; +import { checkLog } from "../tools/check-gha-log.ts"; + +// deno-lint-ignore require-await +unitTest( + "check-gha-log - ::endgroup:: with trailing whitespace is a clean close", + async () => { + const log = [ + "::group::a", + "some output", + "::endgroup:: ", + ].join("\n"); + assertEquals(checkLog(log), []); + }, +); + +// deno-lint-ignore require-await +unitTest( + "check-gha-log - trailing-whitespace close still decrements depth for the next group", + async () => { + const log = [ + "::group::a", + "::endgroup:: ", + "::group::b", + "::endgroup::", + ].join("\n"); + assertEquals(checkLog(log), []); + }, +); + +// deno-lint-ignore require-await +unitTest( + "check-gha-log - leading-whitespace ::endgroup:: is still a column-0 violation", + async () => { + const log = [ + "::group::a", + " ::endgroup::", + ].join("\n"); + const violations = checkLog(log); + assertEquals(violations.length, 2); + assertEquals(violations[0].message, "group marker not at column 0"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "check-gha-log - a genuinely nested ::group:: is still reported", + async () => { + const log = [ + "::group::a", + "::group::b", + "::endgroup::", + "::endgroup::", + ].join("\n"); + const violations = checkLog(log); + assertEquals(violations.length, 1); + assertEquals(violations[0].message.startsWith("nested ::group::"), true); + }, +); + +// deno-lint-ignore require-await +unitTest( + "check-gha-log - a group left open at end of log is still reported", + async () => { + const log = ["::group::a", "some output"].join("\n"); + const violations = checkLog(log); + assertEquals(violations.length, 1); + assertEquals( + violations[0].message.startsWith("group left open at end of log"), + true, + ); + }, +); diff --git a/tests/unit/gha-grouping.test.ts b/tests/unit/gha-grouping.test.ts new file mode 100644 index 00000000000..4bd54119068 --- /dev/null +++ b/tests/unit/gha-grouping.test.ts @@ -0,0 +1,240 @@ +/* + * gha-grouping.test.ts + * + * Tests for per-file GitHub Actions grouping. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { unitTest } from "../test.ts"; +import { assert, assertEquals } from "testing/asserts"; +import { GroupEmitter, testFileUrlFromStack } from "../gha-grouping.ts"; +import { harnessOwnsStep } from "../../src/tools/github.ts"; + +class Recorder { + readonly markers: string[] = []; + depth = 0; + maxDepth = 0; + + start = (title: string) => { + this.markers.push(`group:${title}`); + this.depth++; + this.maxDepth = Math.max(this.maxDepth, this.depth); + }; + end = () => { + this.markers.push("endgroup"); + this.depth--; + }; + + emitter(gate: () => boolean = () => true): GroupEmitter { + return new GroupEmitter(gate, this.start, this.end); + } +} + +// deno-lint-ignore require-await +unitTest("gha-grouping - opens a group on first file", async () => { + const rec = new Recorder(); + const e = rec.emitter(); + e.enterFile("smoke/a.test.ts"); + assertEquals(rec.markers, ["group:smoke/a.test.ts"]); + assertEquals(e.currentFile(), "smoke/a.test.ts"); +}); + +// deno-lint-ignore require-await +unitTest("gha-grouping - same file keeps one group open", async () => { + const rec = new Recorder(); + const e = rec.emitter(); + e.enterFile("smoke/a.test.ts"); + e.enterFile("smoke/a.test.ts"); + e.enterFile("smoke/a.test.ts"); + assertEquals(rec.markers, ["group:smoke/a.test.ts"]); +}); + +// deno-lint-ignore require-await +unitTest("gha-grouping - file change closes then opens", async () => { + const rec = new Recorder(); + const e = rec.emitter(); + e.enterFile("smoke/a.test.ts"); + e.enterFile("smoke/b.test.ts"); + assertEquals(rec.markers, [ + "group:smoke/a.test.ts", + "endgroup", + "group:smoke/b.test.ts", + ]); + assertEquals(rec.maxDepth, 1); + assertEquals(rec.depth, 1); +}); + +// deno-lint-ignore require-await +unitTest("gha-grouping - close is defensive when nothing is open", async () => { + const rec = new Recorder(); + const e = rec.emitter(); + e.close(); + assertEquals(rec.markers, []); + assertEquals(e.currentFile(), undefined); +}); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - failure-path close, then same-file reopen", + async () => { + const rec = new Recorder(); + const e = rec.emitter(); + e.enterFile("smoke/a.test.ts"); + e.close(); + assertEquals(e.currentFile(), undefined); + e.enterFile("smoke/a.test.ts"); + assertEquals(rec.markers, [ + "group:smoke/a.test.ts", + "endgroup", + "group:smoke/a.test.ts", + ]); + assertEquals(rec.maxDepth, 1); + }, +); + +// deno-lint-ignore require-await +unitTest("gha-grouping - double close never double-emits", async () => { + const rec = new Recorder(); + const e = rec.emitter(); + e.enterFile("smoke/a.test.ts"); + e.close(); + e.close(); + assertEquals(rec.markers, ["group:smoke/a.test.ts", "endgroup"]); +}); + +// deno-lint-ignore require-await +unitTest("gha-grouping - gate off: no emission, no state change", async () => { + const rec = new Recorder(); + const e = rec.emitter(() => false); + e.enterFile("smoke/a.test.ts"); + e.enterFile("smoke/b.test.ts"); + e.close(); + assertEquals(rec.markers, []); + assertEquals(e.currentFile(), undefined); +}); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - single-open-group invariant across many files", + async () => { + const rec = new Recorder(); + const e = rec.emitter(); + for (const f of ["a", "b", "c", "d"]) { + e.enterFile(`smoke/${f}.test.ts`); + } + e.close(); + assertEquals(rec.maxDepth, 1); + assertEquals(rec.depth, 0); + const opens = rec.markers.filter((m) => m.startsWith("group:")).length; + const closes = rec.markers.filter((m) => m === "endgroup").length; + assertEquals(opens, closes); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - ownership gate honors both env dimensions", + async () => { + assertEquals(harnessOwnsStep(true, null), true); + assertEquals(harnessOwnsStep(true, ""), true); + assertEquals(harnessOwnsStep(true, "1"), false); + assertEquals(harnessOwnsStep(false, null), false); + assertEquals(harnessOwnsStep(false, "1"), false); + + const rec = new Recorder(); + const orchestrated = rec.emitter(() => harnessOwnsStep(true, "1")); + orchestrated.enterFile("smoke/a.test.ts"); + assertEquals(rec.markers, []); + assert(rec.maxDepth === 0); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - stack: Linux bare frame yields the test-file URL", + async () => { + const stack = [ + "Error", + " at testFileUrlFromStack (file:///home/runner/quarto-cli/tests/gha-grouping.ts:30:10)", + " at test (file:///home/runner/quarto-cli/tests/test.ts:340:20)", + " at file:///home/runner/quarto-cli/tests/unit/foo.test.ts:12:1", + ].join("\n"); + assertEquals( + testFileUrlFromStack(stack), + "file:///home/runner/quarto-cli/tests/unit/foo.test.ts", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - stack: Windows drive-letter frame yields the test-file URL", + async () => { + const stack = [ + "Error", + " at test (file:///C:/Users/x/quarto-cli/tests/test.ts:340:20)", + " at file:///C:/Users/x/quarto-cli/tests/smoke/render/bar.test.ts:3:5", + ].join("\n"); + assertEquals( + testFileUrlFromStack(stack), + "file:///C:/Users/x/quarto-cli/tests/smoke/render/bar.test.ts", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - stack: function-wrapped frame in parentheses", + async () => { + const stack = [ + "Error", + " at Object.fn (file:///home/runner/quarto-cli/tests/unit/baz.test.ts:9:1)", + ].join("\n"); + assertEquals( + testFileUrlFromStack(stack), + "file:///home/runner/quarto-cli/tests/unit/baz.test.ts", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - stack: first test-file frame wins over later ones", + async () => { + const stack = [ + "Error", + " at testFileUrlFromStack (file:///home/runner/quarto-cli/tests/gha-grouping.ts:30:10)", + " at test (file:///home/runner/quarto-cli/tests/test.ts:340:20)", + " at file:///home/runner/quarto-cli/tests/unit/first.test.ts:5:1", + " at file:///home/runner/quarto-cli/tests/unit/second.test.ts:8:1", + ].join("\n"); + assertEquals( + testFileUrlFromStack(stack), + "file:///home/runner/quarto-cli/tests/unit/first.test.ts", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - stack: no test-file frame yields undefined", + async () => { + const stack = [ + "Error", + " at testFileUrlFromStack (file:///home/runner/quarto-cli/tests/gha-grouping.ts:30:10)", + " at test (file:///home/runner/quarto-cli/tests/test.ts:340:20)", + ].join("\n"); + assertEquals(testFileUrlFromStack(stack), undefined); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-grouping - stack: missing or malformed stack yields undefined", + async () => { + assertEquals(testFileUrlFromStack(undefined), undefined); + assertEquals(testFileUrlFromStack(""), undefined); + assertEquals(testFileUrlFromStack("not a stack trace at all"), undefined); + }, +); diff --git a/tests/unit/github-actions-reporting.test.ts b/tests/unit/github-actions-reporting.test.ts new file mode 100644 index 00000000000..fbe2bed4317 --- /dev/null +++ b/tests/unit/github-actions-reporting.test.ts @@ -0,0 +1,841 @@ +/* + * github-actions-reporting.test.ts + * + * Tests for GitHub Actions test-reporting helpers. + * + * Copyright (C) 2026 Posit Software, PBC + */ + +import { unitTest } from "../test.ts"; +import { assert, assertEquals } from "testing/asserts"; +import { join } from "../../src/deno_ral/path.ts"; +import { + annotationBody, + AnnotationBudget, + appendStepSummaryBounded, + appendStepSummaryFirstFit, + escapeData, + escapeProperty, + excerptSignature, + failureLabel, + harnessOwnsStep, + kExcerptMaxBytes, + kStepSummaryBudgetBytes, + kStepSummaryTruncationNotice, + stepSummary, + stepSummarySize, + stripAnsi, + summaryClusterBlock, + summaryTableHeader, + summaryTableRow, + summaryTableRowNameOnly, + truncateUtf8Bytes, +} from "../../src/tools/github.ts"; + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotation budget caps at 9 then aggregates once", + async () => { + const budget = new AnnotationBudget(9, null); + const decisions = []; + for (let i = 0; i < 12; i++) { + decisions.push(budget.recordFailure()); + } + assertEquals(decisions.filter((d) => d.emitAnnotation).length, 9); + assertEquals(decisions.slice(0, 9).every((d) => d.emitAnnotation), true); + assertEquals(decisions.map((d) => d.emitAggregate), [ + false, + false, + false, + false, + false, + false, + false, + false, + false, + true, + false, + false, + ]); + assertEquals(decisions.map((d) => d.ordinal), [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + ]); + }, +); + +// deno-lint-ignore require-await +unitTest("gha-reporting - annotation budget honors a custom cap", async () => { + const budget = new AnnotationBudget(2, null); + assertEquals(budget.recordFailure(), { + ordinal: 1, + emitAnnotation: true, + emitAggregate: false, + }); + assertEquals(budget.recordFailure(), { + ordinal: 2, + emitAnnotation: true, + emitAggregate: false, + }); + assertEquals(budget.recordFailure(), { + ordinal: 3, + emitAnnotation: false, + emitAggregate: true, + }); + assertEquals(budget.recordFailure(), { + ordinal: 4, + emitAnnotation: false, + emitAggregate: false, + }); +}); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotation budget is step-wide across module instances", + async () => { + const counter = Deno.makeTempFileSync({ suffix: ".count" }); + Deno.removeSync(counter); // budget must cope with a not-yet-created file + try { + const fileA = new AnnotationBudget(3, counter); + const fileB = new AnnotationBudget(3, counter); + assertEquals(fileA.recordFailure(), { + ordinal: 1, + emitAnnotation: true, + emitAggregate: false, + }); + assertEquals(fileA.recordFailure(), { + ordinal: 2, + emitAnnotation: true, + emitAggregate: false, + }); + assertEquals(fileB.recordFailure(), { + ordinal: 3, + emitAnnotation: true, + emitAggregate: false, + }); + assertEquals(fileB.recordFailure(), { + ordinal: 4, + emitAnnotation: false, + emitAggregate: true, + }); + assertEquals(fileA.recordFailure(), { + ordinal: 5, + emitAnnotation: false, + emitAggregate: false, + }); + assertEquals(fileB.recordFailure(), { + ordinal: 6, + emitAnnotation: false, + emitAggregate: false, + }); + } finally { + try { + Deno.removeSync(counter); + } catch { + // The counter may not have been created. + } + } + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - failureLabel prefixes the ordinal with the runner OS", + async () => { + assertEquals(failureLabel(7, "Linux"), "L-F7"); + assertEquals(failureLabel(7, "Windows"), "W-F7"); + assertEquals(failureLabel(7, "macOS"), "M-F7"); + assertEquals(failureLabel(1, ""), "X-F1"); + assertEquals(failureLabel(3, "FreeBSD"), "X-F3"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - failureLabel discriminates same-OS jobs by workflow tag", + async () => { + assertEquals(failureLabel(1, "Linux", "b01"), "Lb01-F1"); + assertEquals(failureLabel(1, "Linux", "b02"), "Lb02-F1"); + assert( + failureLabel(1, "Linux", "b01") !== failureLabel(1, "Linux", "b02"), + "same OS, same ordinal, different leg → different label", + ); + assertEquals(failureLabel(12, "Windows", "relsmk"), "Wrelsmk-F12"); + assertEquals(failureLabel(3, "macOS", "ngtsmk"), "Mngtsmk-F3"); + assertEquals(failureLabel(7, "Linux", "b07"), "Lb07-F7"); + assertEquals(failureLabel(7, "Linux", ""), "L-F7"); + assertEquals(failureLabel(7, "Linux"), "L-F7"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - failureLabel sanitizes the workflow tag", + async () => { + assertEquals(failureLabel(2, "Linux", "smoke | tests"), "Lsmoketes-F2"); + assertEquals(failureLabel(2, "Linux", "a*b_c-d"), "Labcd-F2"); + assertEquals(failureLabel(2, "Linux", "line\nbreak"), "Llinebrea-F2"); + assertEquals(failureLabel(2, "Linux", "0123456789"), "L01234567-F2"); + assertEquals(failureLabel(2, "Linux", "***"), "L-F2"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - excerptSignature is the first 3 non-empty ANSI-stripped lines", + async () => { + const excerpt = "\x1b[31mone\x1b[0m\n\n two \n\nthree\nfour\nfive"; + assertEquals(excerptSignature(excerpt), "one\n two \nthree"); + assertEquals(excerptSignature("only\n\n"), "only"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - identical-error excerpts share a signature; distinct ones don't", + async () => { + const a = "AssertionError: x\n at foo\n at bar\n detail-a"; + const b = "AssertionError: x\n at foo\n at bar\n detail-b"; // differs past line 3 + const c = "TypeError: y\n at baz\n at qux"; + assertEquals( + excerptSignature(a), + excerptSignature(b), + "same first 3 lines cluster", + ); + assert( + excerptSignature(a) !== excerptSignature(c), + "different errors do not cluster", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotationBody trims to repro + 5 lines + summary pointer", + async () => { + const excerpt = "l1\nl2\nl3\nl4\nl5\nl6\nl7"; + const body = annotationBody( + "./run-tests.sh x.qmd", + excerpt, + "L-F7", + "detail", + ); + assert( + body.startsWith("./run-tests.sh x.qmd\n\n"), + "repro then a blank line", + ); + assert( + body.includes("l1\nl2\nl3\nl4\nl5"), + "first five excerpt lines kept", + ); + assert(!body.includes("l6"), "sixth line dropped"); + assert(body.includes("…"), "ellipsis marks the truncation"); + assert(body.trimEnd().includes("L-F7"), "pointer line carries the label"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotationBody pointer: a queued cluster gets honest, unconditional wording", + async () => { + const body = annotationBody( + "./run-tests.sh x.qmd", + "boom", + "L-F7", + "detail", + ); + assert(body.includes("L-F7"), "pointer carries the label"); + assert( + !body.includes("Full output"), + "does not promise complete output exists", + ); + assert(body.includes("step log"), "points the reader at the step log"); + assert( + !body.includes("…"), + "a short excerpt that fit under maxLines was not truncated, so no ellipsis is warranted", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotationBody pointer: a degraded (name-only) row gets a distinct, accurate wording", + async () => { + const body = annotationBody( + "./run-tests.sh x.qmd", + "boom", + "L-F8", + "name-only", + ); + assert(body.includes("L-F8")); + assert( + !body.includes("Full output"), + "no promise that a detail block exists", + ); + assert(body.includes("step log"), "points the reader at the step log"); + assert( + body !== annotationBody("./run-tests.sh x.qmd", "boom", "L-F8", "detail"), + "the two cases produce different wording", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotationBody pointer: no row at all gets wording that names no summary target", + async () => { + const body = annotationBody("./run-tests.sh x.qmd", "boom", "L-F9", "none"); + assert( + !body.includes("L-F9"), + "does not point the reader at a summary row under this label", + ); + assert(!body.includes("Failure row"), "does not claim any row exists"); + assert( + !body.includes("Full output"), + "does not promise complete output exists", + ); + assert(body.includes("step log"), "points the reader at the step log"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotationBody produces three distinct bodies for the three outcomes", + async () => { + const bodies = (["detail", "name-only", "none"] as const).map((outcome) => + annotationBody("./run-tests.sh x.qmd", "boom", "L-F10", outcome) + ); + assertEquals(new Set(bodies).size, 3, "each outcome has its own wording"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - annotationBody bounds a huge excerpt by bytes even if the caller didn't", + async () => { + const huge = "z".repeat(300 * 1024); + const body = annotationBody("./run-tests.sh x.qmd", huge, "L-F9", "detail"); + assert(!body.includes(huge), "the full huge excerpt is not embedded whole"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - truncateUtf8Bytes leaves content under the cap untouched", + async () => { + assertEquals(truncateUtf8Bytes("short", 100), "short"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - truncateUtf8Bytes bounds a huge single line by UTF-8 bytes", + async () => { + const huge = "z".repeat(1000); + const truncated = truncateUtf8Bytes(huge, 100); + assert( + new TextEncoder().encode(truncated).length <= 100, + "result stays within the byte budget", + ); + assert(truncated.includes("truncated"), "marks that truncation happened"); + assert(truncated.length < huge.length, "shorter than the input"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - truncateUtf8Bytes cuts on a UTF-8 boundary, not mid-codepoint", + async () => { + // The 10-byte content budget splits a three-byte code point. + const s = "€".repeat(50); + const truncated = truncateUtf8Bytes(s, 25); + assert( + !truncated.includes("�"), + "no replacement character from a split code point", + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - truncateUtf8Bytes bounds the marker itself when maxBytes is below the marker's own length", + async () => { + for (const maxBytes of [0, 1, 5, 10, 14]) { + const truncated = truncateUtf8Bytes("z".repeat(1000), maxBytes); + assert( + new TextEncoder().encode(truncated).length <= maxBytes, + `maxBytes=${maxBytes}: result must not exceed the requested budget`, + ); + } + }, +); + +// deno-lint-ignore require-await +unitTest("gha-reporting - harnessOwnsStep gate", async () => { + assertEquals(harnessOwnsStep(true, null), true); + assertEquals(harnessOwnsStep(true, ""), true); + assertEquals(harnessOwnsStep(true, "1"), false); + assertEquals(harnessOwnsStep(false, null), false); + assertEquals(harnessOwnsStep(false, "1"), false); +}); + +// deno-lint-ignore require-await +unitTest("gha-reporting - stripAnsi removes color codes", async () => { + assertEquals(stripAnsi("\x1b[31mred\x1b[0m"), "red"); + assertEquals( + stripAnsi("\x1b[1m\x1b[32mbold green\x1b[0m done"), + "bold green done", + ); + assertEquals(stripAnsi("no color here"), "no color here"); + assertEquals(stripAnsi("line1\n\x1b[31mline2\x1b[0m"), "line1\nline2"); +}); + +unitTest( + "gha-reporting - step summary size budget and degrade path", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + assertEquals(stepSummarySize(tmp), 0); + + stepSummary("x".repeat(100), tmp); + assertEquals(stepSummarySize(tmp), 100); + assert( + stepSummarySize(tmp) < kStepSummaryBudgetBytes, + "still under budget", + ); + + stepSummary("y".repeat(kStepSummaryBudgetBytes), tmp); + assert( + stepSummarySize(tmp) > kStepSummaryBudgetBytes, + "now over budget → callers degrade to name-only rows", + ); + + stepSummary("ignored", ""); + stepSummary("ignored", null); + assertEquals(stepSummarySize(""), 0); + assertEquals(stepSummarySize(null), 0); + } finally { + Deno.removeSync(tmp); + } + }, +); + +unitTest( + "gha-reporting - appendStepSummaryBounded writes when content fits and reports success", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + assertEquals(appendStepSummaryBounded("| row |\n", tmp), true); + assertEquals(Deno.readTextFileSync(tmp), "| row |\n"); + } finally { + Deno.removeSync(tmp); + } + }, +); + +unitTest( + "gha-reporting - appendStepSummaryBounded refuses content that would exceed the budget, and reports failure", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + Deno.writeTextFileSync(tmp, "x".repeat(kStepSummaryBudgetBytes)); + assertEquals(appendStepSummaryBounded("more content", tmp), false); + } finally { + Deno.removeSync(tmp); + } + }, +); + +unitTest( + "gha-reporting - appendStepSummaryBounded emits the truncation notice exactly once, even across repeated refusals", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + const noticeBytes = new TextEncoder().encode(kStepSummaryTruncationNotice) + .length; + Deno.writeTextFileSync( + tmp, + "x".repeat(kStepSummaryBudgetBytes - noticeBytes - 500), + ); + assertEquals(appendStepSummaryBounded("y".repeat(1000), tmp), false); + const afterFirst = Deno.readTextFileSync(tmp); + assert( + afterFirst.includes(kStepSummaryTruncationNotice), + "the notice was written on the first refusal", + ); + assertEquals(appendStepSummaryBounded("z".repeat(1000), tmp), false); + assertEquals(appendStepSummaryBounded("w".repeat(1000), tmp), false); + const afterMore = Deno.readTextFileSync(tmp); + const occurrences = afterMore.split(kStepSummaryTruncationNotice).length - + 1; + assertEquals(occurrences, 1, "the notice is emitted exactly once"); + } finally { + Deno.removeSync(tmp); + } + }, +); + +unitTest( + "gha-reporting - appendStepSummaryBounded never lets the file exceed the configured limit", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + for (let i = 0; i < 50; i++) { + appendStepSummaryBounded("z".repeat(20 * 1024) + "\n", tmp); + } + assert( + stepSummarySize(tmp) <= kStepSummaryBudgetBytes, + "size never exceeds the budget regardless of how many writes are attempted", + ); + } finally { + Deno.removeSync(tmp); + } + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - appendStepSummaryBounded no-ops when the path is unset", + async () => { + assertEquals(appendStepSummaryBounded("ignored", null), false); + assertEquals(appendStepSummaryBounded("ignored", ""), false); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - appendStepSummaryBounded reports failure when the write itself fails, not just when the budget check refuses", + async () => { + const dir = Deno.makeTempDirSync({ prefix: "quarto-summary" }); + try { + const unwritable = join(dir, "no-such-dir", "summary.md"); + assertEquals(appendStepSummaryBounded("| row |\n", unwritable), false); + } finally { + Deno.removeSync(dir, { recursive: true }); + } + }, +); + +unitTest( + "gha-reporting - appendStepSummaryFirstFit writes a smaller fallback that the notice would otherwise crowd out", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + const fullRow = summaryTableRow("L-F1", "tests/x.qmd", "some test", 100); + const nameOnlyRow = summaryTableRowNameOnly( + "L-F1", + "tests/x.qmd", + "some test", + ); + const byteLen = (s: string) => new TextEncoder().encode(s).length; + const noticeBytes = byteLen(kStepSummaryTruncationNotice); + const contentBudget = kStepSummaryBudgetBytes - noticeBytes; + const size = contentBudget - byteLen(nameOnlyRow); + Deno.writeTextFileSync(tmp, "x".repeat(size)); + + const index = appendStepSummaryFirstFit([fullRow, nameOnlyRow], tmp); + assertEquals(index, 1, "the second (smaller) candidate was written"); + const content = Deno.readTextFileSync(tmp); + assert( + content.includes(nameOnlyRow), + "the name-only row fit before the notice and must survive", + ); + assert( + content.includes(kStepSummaryTruncationNotice), + "the notice is still emitted, since the preferred candidate was dropped", + ); + } finally { + Deno.removeSync(tmp); + } + }, +); + +unitTest( + "gha-reporting - appendStepSummaryFirstFit returns 0 and emits no notice when the first candidate fits", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + const index = appendStepSummaryFirstFit(["small", "smaller"], tmp); + assertEquals(index, 0); + const content = Deno.readTextFileSync(tmp); + assertEquals(content, "small"); + assert(!content.includes(kStepSummaryTruncationNotice)); + } finally { + Deno.removeSync(tmp); + } + }, +); + +unitTest( + "gha-reporting - appendStepSummaryFirstFit returns -1 and emits the notice when nothing fits", + async () => { + const tmp = Deno.makeTempFileSync({ suffix: ".md" }); + try { + const noticeBytes = new TextEncoder().encode(kStepSummaryTruncationNotice) + .length; + Deno.writeTextFileSync( + tmp, + "x".repeat(kStepSummaryBudgetBytes - noticeBytes), + ); + const index = appendStepSummaryFirstFit( + ["CANDIDATE_ONE_PAYLOAD", "CANDIDATE_TWO_PAYLOAD"], + tmp, + ); + assertEquals(index, -1); + const content = Deno.readTextFileSync(tmp); + assert(content.includes(kStepSummaryTruncationNotice)); + assert( + !content.includes("CANDIDATE_ONE_PAYLOAD") && + !content.includes("CANDIDATE_TWO_PAYLOAD"), + "neither candidate was written", + ); + } finally { + Deno.removeSync(tmp); + } + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - appendStepSummaryFirstFit no-ops when the path is unset", + async () => { + assertEquals(appendStepSummaryFirstFit(["a", "b"], null), -1); + assertEquals(appendStepSummaryFirstFit(["a", "b"], ""), -1); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - kExcerptMaxBytes is well under the step summary budget", + async () => { + assert(kExcerptMaxBytes < kStepSummaryBudgetBytes / 4); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - summary table carries a plain-text label column", + async () => { + const header = summaryTableHeader(); + assert( + header.startsWith("| # | Test file | Test | Duration |"), + "leading # column", + ); + + const row = summaryTableRow( + "L-F7", + "tests/docs/smoke-all/x.qmd", + "a|b\nc ", + 1234, + ); + assert(row.startsWith("| L-F7 |"), "plain label in the # column"); + assert(!row.includes("](#"), "no fragment-link syntax"); + assert(row.includes("\\|"), "pipe escaped"); + assert(!row.includes("b\nc"), "newline collapsed"); + assert(row.includes("<tag>"), "angle brackets escaped"); + assert(row.includes("1.23s"), "duration formatted"); + + const emphasisRow = summaryTableRow( + "L-F11", + "tests/x.qmd", + "test_foo_bar with `code` and *emph* ", + 500, + ); + assert( + emphasisRow.includes("test\\_foo\\_bar"), + "underscores escaped so they don't trigger emphasis", + ); + assert( + emphasisRow.includes("\\`code\\`"), + "backticks escaped so they don't open a code span", + ); + assert( + emphasisRow.includes("\\*emph\\*"), + "asterisks escaped so they don't trigger emphasis", + ); + assert( + emphasisRow.includes("<tag>"), + "angle brackets still escaped alongside markdown escaping", + ); + + const nameOnly = summaryTableRowNameOnly( + "L-F8", + "tests/docs/smoke-all/x.qmd", + "a|b", + ); + assert( + nameOnly.startsWith("| L-F8 |"), + "name-only row carries the label too", + ); + assert(nameOnly.includes("\\|"), "pipe escaped in name-only row"); + assert(nameOnly.trimEnd().endsWith("| |"), "empty duration cell"); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - summaryCell escapes a pre-existing backslash before escaping markdown punctuation", + async () => { + const row = summaryTableRow("L-F12", "tests/x.qmd", "\\_foo_", 100); + const expected = "\\".repeat(3) + "_foo" + "\\_"; + assert( + row.includes(expected), + `expected the pre-existing backslash to be escaped independently, got: ${row}`, + ); + }, +); + +// deno-lint-ignore require-await +unitTest( + "gha-reporting - single-member cluster renders a labeled detail block", + async () => { + const block = summaryClusterBlock({ + label: "L-F3", + members: [{ + label: "L-F3", + file: "docs/smoke-all/x.qmd", + testName: "[smoke] > quarto render docs/smoke-all/x.qmd ", + repro: "./run-tests.sh docs/smoke-all/x.qmd", + }], + excerpt: "boom