Regression coverage for real-use bugs, and fixes for their live siblings

.github/workflows/ci.yml

@@ -52,6 +52,16 @@ jobs:
       - run: bun install --frozen-lockfile
       - run: bun run test:integration
 
+  test-compiled-binaries:
+    name: Compiled Binary Smoke Tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+      - uses: oven-sh/setup-bun@v2
+      - run: bun install --frozen-lockfile
+      - run: make build
+      - run: bun run test:smoke
+
   security:
     name: Security Audit
     runs-on: ubuntu-latest

docs/project-discovery.md

@@ -30,6 +30,7 @@
 - Test (unit): `bun run vitest run`
 - Test (integration): `bun run vitest run --config vitest.integration.config.ts`
 - Test (all): `make test`
+- Test (compiled binary smoke): `make build && bun run test:smoke`
 - Build: `make build`
 - Dev server: `make dev`
 - Test file pattern: `*.test.ts`, `*.integration.test.ts`, `*.unit.test.ts`

docs/sandbox-integration-package.md

@@ -110,7 +110,7 @@ Primary execution function. Builds and spawns the command `sbx exec claude-skill
 async function createSandbox(repoRoot: string, extraWorkspaces: string[] = []): Promise<void>
 ```
 
-Checks whether the sandbox already exists via an internal `sandboxExists()` helper (runs `sbx ls --quiet`). If found, prints a help message to stderr explaining how to recreate it, and returns early. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude <repoRoot> [<extraWorkspace>:ro ...]` with inherited stdio for interactive OAuth login. Prints progress messages to stderr.
+Checks whether the sandbox already exists via an internal `sandboxExists()` helper (runs `sbx ls --quiet`). If found, prints a help message to stderr explaining how to recreate it, and returns early. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude <repoRoot> [<extraWorkspace>:ro ...]` with inherited stdio for interactive OAuth login. Prints progress messages to stderr. Throws `SandboxError` if `sbx run` exits non-zero.
 
 `extraWorkspaces` are mounted read-only after `repoRoot` (`<path>:ro`); any already inside `repoRoot` are skipped. The CLI passes the directory holding `sandbox-run.sh` and `sandbox-extract.sh` (`sandboxScriptsDir` from `@testdouble/claude-integration`). `execInSandbox` runs those scripts by their host path, and the sandbox only sees host paths under a mounted workspace, so without this mount every test run fails whenever the target repo is not the skillwalker repo.
 

docs/sandbox-integration.md

@@ -171,7 +171,7 @@ When a scaffold path is provided, it copies the scaffold into a fresh temp direc
 export async function createSandbox(repoRoot: string, extraWorkspaces: string[] = []): Promise<void>
 ```
 
-Checks if the sandbox already exists via an internal `sandboxExists()` helper. If it does, prints a help message to stderr and returns. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude <repoRoot> [<extraWorkspace>:ro ...]` with inherited stdio for interactive OAuth login.
+Checks if the sandbox already exists via an internal `sandboxExists()` helper. If it does, prints a help message to stderr and returns. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude <repoRoot> [<extraWorkspace>:ro ...]` with inherited stdio for interactive OAuth login. If `sbx run` exits non-zero, it throws `SandboxError` instead of reporting the sandbox as ready.
 
 `extraWorkspaces` are mounted read-only after `repoRoot` (`<path>:ro`); any already inside `repoRoot` are skipped. The CLI passes the directory holding `sandbox-run.sh` and `sandbox-extract.sh` (`sandboxScriptsDir` from `@testdouble/claude-integration`). `execInSandbox` runs those scripts by their host path, and the sandbox only sees host paths under a mounted workspace, so without this mount every test run fails whenever the target repo is not the skillwalker repo.
 

docs/web.md

@@ -16,7 +16,8 @@ Change this package when you need to touch the dashboard's Hono API server, the
 - Compiled as a standalone Bun executable (`skillwalker-web`) with embedded client assets via Bun's `{ type: 'file' }` imports
 
 Key files:
-- `packages/web/src/server/index.ts` — Server entry point, CLI arg parsing, route registration, embedded asset serving
+- `packages/web/src/server/index.ts` — Server entry point, CLI arg parsing, embedded asset serving
+- `packages/web/src/server/app.ts` — `createApp(dataDir)`: the Hono app with the API routes and `jsonErrorHandler`
 - `packages/web/src/client/index.tsx` — Client entry point, router and page registration
 - `packages/web/src/server/routes/test-runs.ts` — Test run list and detail API endpoints
 - `packages/web/src/server/routes/scil.ts` — SCIL history and detail API endpoints
@@ -68,7 +69,8 @@ flowchart TB
 ### Backend
 | File | Purpose |
 |------|---------|
-| `packages/web/src/server/index.ts` | Server entry point — Yargs CLI, Hono app, route registration, embedded static asset serving, SPA fallback |
+| `packages/web/src/server/index.ts` | Server entry point — Yargs CLI, embedded static asset serving, SPA fallback |
+| `packages/web/src/server/app.ts` | `createApp(dataDir)` — Hono app with the API routes and `jsonErrorHandler` |
 | `packages/web/src/server/routes/test-runs.ts` | `getTestRuns` and `getTestRunById` handlers delegating to `queryTestRunSummaries` / `queryTestRunDetails` |
 | `packages/web/src/server/routes/scil.ts` | `getScilHistory` and `getScilRunById` handlers delegating to `queryScilHistory` / `queryScilRunDetails` |
 | `packages/web/src/server/routes/analytics.ts` | `getPerTestAnalytics` handler with optional `?eval=` query param filter |
@@ -209,7 +211,7 @@ interface ScilSummaryRow {
 
 #### Server Startup and Asset Embedding
 
-The server entry (`packages/web/src/server/index.ts`) uses Yargs to parse `--port` and `--data-dir` CLI arguments. It registers the API routes, a `jsonErrorHandler` via `app.onError` so unexpected errors reach the client as JSON, and the static asset routes. The client build output (`dist/client/`) is embedded using Bun's `import ... with { type: 'file' }` syntax, which resolves to `$bunfs` paths in compiled standalone executables. A SPA fallback (`/*`) serves `index.html` for all unmatched paths, enabling client-side routing.
+The server entry (`packages/web/src/server/index.ts`) uses Yargs to parse `--port` and `--data-dir` CLI arguments. It builds the app with `createApp(dataDir)` from `app.ts`, which registers the API routes and a `jsonErrorHandler` via `app.onError` so unexpected errors reach the client as JSON. The entry then adds the static asset routes. The client build output (`dist/client/`) is embedded using Bun's `import ... with { type: 'file' }` syntax, which resolves to `$bunfs` paths in compiled standalone executables. A SPA fallback (`/*`) serves `index.html` for all unmatched paths, enabling client-side routing.
 
 #### Route Handler Pattern
 
@@ -429,6 +431,7 @@ flowchart TB
 - `packages/web/src/server/routes/scil.test.ts` / `acil.test.ts` — Test the SCIL and ACIL history and detail handlers, including 404 for malformed run IDs
 - `packages/web/src/server/routes/error-handler.test.ts` — Tests that `jsonErrorHandler` logs the error and returns a JSON 500
 - `packages/web/src/server/routes/analytics.test.ts` — Tests `getPerTestAnalytics` including eval filter behavior
+- `packages/web/src/server/app.integration.test.ts` — Sends real requests through `createApp` against Parquet built from real JSONL, with nothing mocked. Covers whole-number scores, accuracies, and costs, which DuckDB returns as `BigInt` unless the query casts them, and an empty data directory
 
 ### Frontend
 - `packages/web/src/client/lib/fetch-json.test.ts` — Tests `fetchJson` with OK, non-OK JSON, and non-OK plain-text responses, using `vi.stubGlobal('fetch', ...)`

package.json

@@ -8,6 +8,7 @@
     "test:watch": "vitest",
     "test:integration": "vitest run --config vitest.integration.config.ts",
     "test:all": "vitest run --config vitest.all.config.ts",
+    "test:smoke": "vitest run --config vitest.smoke.config.ts",
     "lint": "biome lint",
     "format:check": "biome format",
     "check": "biome check",

packages/cli/src/command-registration.integration.test.ts

@@ -1,11 +1,18 @@
 import { spawnSync } from 'node:child_process'
+import { chmod, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import path from 'node:path'
 import { fileURLToPath } from 'node:url'
-import { describe, expect, it } from 'vitest'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
 
 const entryPoint = fileURLToPath(new URL('../index.ts', import.meta.url))
 
 function runCli(...args: string[]): { status: number; output: string } {
-  const result = spawnSync('bun', [entryPoint, ...args], { encoding: 'utf8' })
+  return runCliWithEnv(process.env, ...args)
+}
+
+function runCliWithEnv(env: NodeJS.ProcessEnv, ...args: string[]): { status: number; output: string } {
+  const result = spawnSync('bun', [entryPoint, ...args], { encoding: 'utf8', env })
   return { status: result.status ?? 1, output: `${result.stdout}${result.stderr}` }
 }
 
@@ -66,4 +73,30 @@ describe('command handler errors', () => {
     expect(output).not.toContain('Options:')
     expect(output).not.toContain('RunNotFoundError')
   })
+
+  describe('when sbx fails', () => {
+    let fakeBinDir: string
+
+    beforeEach(async () => {
+      fakeBinDir = await mkdtemp(path.join(tmpdir(), 'skillwalker-fake-sbx-'))
+      const fakeSbx = path.join(fakeBinDir, 'sbx')
+      await writeFile(fakeSbx, '#!/bin/sh\necho "You are not logged in." >&2\nexit 1\n', 'utf8')
+      await chmod(fakeSbx, 0o755)
+    })
+
+    afterEach(async () => {
+      await rm(fakeBinDir, { recursive: true, force: true })
+    })
+
+    it('prints a sandbox error as a single Error line without help text or a stack trace', () => {
+      const env = { ...process.env, PATH: `${fakeBinDir}${path.delimiter}${process.env.PATH}` }
+
+      const { status, output } = runCliWithEnv(env, 'sandbox', 'shell')
+
+      expect(status).toBe(1)
+      expect(output).toMatch(/^Error: Unable to list sandboxes with sbx/m)
+      expect(output).not.toContain('Options:')
+      expect(output).not.toMatch(/^\s+at /m)
+    })
+  })
 })

packages/cli/src/compiled-binary.smoke.test.ts

@@ -0,0 +1,79 @@
+import { type ChildProcess, spawn, spawnSync } from 'node:child_process'
+import { existsSync } from 'node:fs'
+import { rm } from 'node:fs/promises'
+import path from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { makeTmpDir, writeRunFixture } from '@testdouble/skillwalker-data/src/analytics-test-helpers.js'
+import { afterEach, beforeEach, describe, expect, it } from 'vitest'
+
+// Runs the binaries `make build` produces. Unit tests stub the Bun runtime, so only
+// a compiled binary can show the DuckDB native addon actually loads.
+const buildDir = fileURLToPath(new URL('../../../build/', import.meta.url))
+const cliBinary = path.join(buildDir, 'skillwalker')
+const webBinary = path.join(buildDir, 'skillwalker-web')
+
+const TEST_RUN_ID = '20260101T100001'
+const WEB_PORT = 39099
+
+async function waitForServer(url: string, proc: ChildProcess, timeoutMs = 10000): Promise<Response> {
+  const deadline = Date.now() + timeoutMs
+  while (Date.now() < deadline) {
+    if (proc.exitCode !== null) throw new Error(`skillwalker-web exited early with code ${proc.exitCode}`)
+    try {
+      return await fetch(url)
+    } catch {
+      await new Promise((resolve) => setTimeout(resolve, 100))
+    }
+  }
+  throw new Error(`skillwalker-web did not answer ${url} within ${timeoutMs}ms`)
+}
+
+// ─── test lifecycle ───────────────────────────────────────────────────────────
+
+let tmpDir: string
+let outputDir: string
+let dataDir: string
+
+beforeEach(async () => {
+  tmpDir = await makeTmpDir()
+  outputDir = path.join(tmpDir, 'output')
+  dataDir = path.join(tmpDir, 'analytics')
+  await writeRunFixture({ outputDir, testRunId: TEST_RUN_ID, eval: 'smoke', testName: 'smoke test' })
+})
+
+afterEach(async () => {
+  await rm(tmpDir, { recursive: true, force: true })
+})
+
+// ─── compiled binaries ────────────────────────────────────────────────────────
+
+describe('compiled skillwalker binary', () => {
+  it('imports run output into parquet with the bundled DuckDB addon', () => {
+    const result = spawnSync(cliBinary, ['update-analytics-data', '--output-dir', outputDir, '--data-dir', dataDir], {
+      encoding: 'utf8',
+    })
+    const output = `${result.stdout}${result.stderr}`
+
+    expect(output).not.toContain('Cannot find module')
+    expect(result.status).toBe(0)
+    expect(existsSync(path.join(dataDir, 'test-run.parquet'))).toBe(true)
+  })
+})
+
+describe('compiled skillwalker-web binary', () => {
+  let server: ChildProcess | undefined
+
+  afterEach(() => {
+    server?.kill()
+  })
+
+  it('serves test runs queried through the bundled DuckDB addon', async () => {
+    spawnSync(cliBinary, ['update-analytics-data', '--output-dir', outputDir, '--data-dir', dataDir])
+    server = spawn(webBinary, ['--port', String(WEB_PORT), '--data-dir', dataDir])
+
+    const res = await waitForServer(`http://localhost:${WEB_PORT}/api/test-runs`, server)
+
+    expect(res.status).toBe(200)
+    expect((await res.json()).runs).toHaveLength(1)
+  })
+})

packages/data/src/analytics.integration.test.ts

@@ -356,6 +356,17 @@ describe('queryPerTest', () => {
     const rows = await queryPerTest(dataDir)
     expect(rows[0].total_cost_usd).toBe(0.12)
   })
+
+  it('returns total_cost_usd as a number when every run cost a whole number', async () => {
+    const outputDir = path.join(tmpDir, 'output')
+    const dataDir = path.join(tmpDir, 'analytics')
+    await writeRunFixture({ outputDir, testRunId: '20260101T100001', eval: 's', testName: 'test one', totalCostUsd: 0 })
+    await updateAllParquet({ outputDir, dataDir })
+
+    const rows = await queryPerTest(dataDir)
+
+    expect(rows[0].total_cost_usd).toBe(0)
+  })
 })
 
 // ─── queryTestRunSummaries ────────────────────────────────────────────────────
@@ -482,6 +493,17 @@ describe('queryTestRunDetails', () => {
     const details = await queryTestRunDetails(dataDir, '20260101T000008')
     expect(details.summary[0].total_cost_usd).toBe(0.1235)
   })
+
+  it('returns total_cost_usd as a number in details when every run cost a whole number', async () => {
+    const outputDir = path.join(tmpDir, 'output')
+    const dataDir = path.join(tmpDir, 'analytics')
+    await writeRunFixture({ outputDir, testRunId: '20260101T100001', eval: 's', testName: 'test one', totalCostUsd: 0 })
+    await updateAllParquet({ outputDir, dataDir })
+
+    const details = await queryTestRunDetails(dataDir, '20260101T100001')
+
+    expect(details.summary[0].total_cost_usd).toBe(0)
+  })
 })
 
 // ─── queryPerTest — JOIN edge cases ──────────────────────────────────────────
@@ -806,6 +828,21 @@ describe('queryScilHistory', () => {
       best_train_accuracy: 1.0,
     })
   })
+
+  it('returns best_train_accuracy as a number when every iteration scored a whole number', async () => {
+    const outputDir = path.join(tmpDir, 'output')
+    const dataDir = path.join(tmpDir, 'analytics')
+    await writeScilRunFixture({
+      outputDir,
+      runId: '20260101T200001',
+      iterations: [makeScilIterationRecord({ trainAccuracy: 1 })],
+    })
+    await updateAllParquet({ outputDir, dataDir })
+
+    const rows = await queryScilHistory(dataDir)
+
+    expect(rows[0].best_train_accuracy).toBe(1)
+  })
 })
 
 // ─── SCIL: queryScilRunDetails ───────────────────────────────────────────────
@@ -1488,6 +1525,21 @@ describe('queryAcilHistory', () => {
     })
   })
 
+  it('returns best_train_accuracy as a number when every iteration scored a whole number', async () => {
+    const outputDir = path.join(tmpDir, 'output')
+    const dataDir = path.join(tmpDir, 'analytics')
+    await writeAcilRunFixture({
+      outputDir,
+      runId: '20260101T300001',
+      iterations: [makeAcilIterationRecord({ trainAccuracy: 1 })],
+    })
+    await updateAllParquet({ outputDir, dataDir })
+
+    const rows = await queryAcilHistory(dataDir)
+
+    expect(rows[0].best_train_accuracy).toBe(1)
+  })
+
   it('returns multiple runs ordered by test_run_id DESC', async () => {
     const outputDir = path.join(tmpDir, 'output')
     const dataDir = path.join(tmpDir, 'analytics')

packages/data/src/analytics.ts

@@ -345,7 +345,7 @@ export async function queryPerTest(dataDir: string): Promise<PerTestRow[]> {
         c.test.name AS test_name,
         c.eval,
         e.all_expectations_passed,
-        ROUND(r.total_cost_usd, 2) AS total_cost_usd,
+        CAST(ROUND(r.total_cost_usd, 2) AS DOUBLE) AS total_cost_usd,
         CAST(r.num_turns AS INTEGER) AS num_turns,
         CAST(r.usage.input_tokens AS INTEGER) AS input_tokens,
         CAST(r.usage.output_tokens AS INTEGER) AS output_tokens
@@ -464,7 +464,7 @@ export async function queryTestRunDetails(dataDir: string, testRunId: string): P
         r.is_error,
         e.all_expectations_passed,
         r.result,
-        ROUND(r.total_cost_usd, 4) AS total_cost_usd,
+        CAST(ROUND(r.total_cost_usd, 4) AS DOUBLE) AS total_cost_usd,
         CAST(r.num_turns AS INTEGER) AS num_turns,
         CAST(r.usage.input_tokens AS INTEGER) AS input_tokens,
         CAST(r.usage.output_tokens AS INTEGER) AS output_tokens

packages/data/src/run-status.ts

@@ -44,7 +44,7 @@ export async function queryScilHistory(dataDir: string): Promise<ScilHistoryRow[
         i.test_run_id,
         i.skill_file,
         CAST(MAX(i.iteration) AS INTEGER) AS iteration_count,
-        MAX(i.trainAccuracy) AS best_train_accuracy
+        CAST(MAX(i.trainAccuracy) AS DOUBLE) AS best_train_accuracy
       FROM read_parquet('${dataDir}/scil-iteration.parquet') i
       GROUP BY i.test_run_id, i.skill_file
       ORDER BY i.test_run_id DESC
@@ -105,7 +105,7 @@ export async function queryAcilHistory(dataDir: string): Promise<AcilHistoryRow[
         i.test_run_id,
         i.agent_file,
         CAST(MAX(i.iteration) AS INTEGER) AS iteration_count,
-        MAX(i.trainAccuracy) AS best_train_accuracy
+        CAST(MAX(i.trainAccuracy) AS DOUBLE) AS best_train_accuracy
       FROM read_parquet('${dataDir}/acil-iteration.parquet') i
       GROUP BY i.test_run_id, i.agent_file
       ORDER BY i.test_run_id DESC