From 95c871440ae26211c6a513ed3198ba67cc516ff6 Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:18:08 -0600 Subject: [PATCH 1/9] feat(cli): add a build-time version with a dev fallback --- packages/cli/src/version.test.ts | 8 ++++++++ packages/cli/src/version.ts | 6 ++++++ 2 files changed, 14 insertions(+) create mode 100644 packages/cli/src/version.test.ts create mode 100644 packages/cli/src/version.ts diff --git a/packages/cli/src/version.test.ts b/packages/cli/src/version.test.ts new file mode 100644 index 0000000..b939950 --- /dev/null +++ b/packages/cli/src/version.test.ts @@ -0,0 +1,8 @@ +import { describe, expect, it } from 'vitest' +import { skillwalkerVersion } from './version.js' + +describe('skillwalkerVersion', () => { + it('reports dev when running from source without a build-time version', () => { + expect(skillwalkerVersion).toBe('dev') + }) +}) diff --git a/packages/cli/src/version.ts b/packages/cli/src/version.ts new file mode 100644 index 0000000..07bf2f5 --- /dev/null +++ b/packages/cli/src/version.ts @@ -0,0 +1,6 @@ +// Replaced with the release version by scripts/build.ts through Bun.build's +// `define`. Running from source, nothing defines it, and `typeof` on an +// undeclared name is "undefined" rather than a ReferenceError. +declare const SKILLWALKER_VERSION: string | undefined + +export const skillwalkerVersion = typeof SKILLWALKER_VERSION === 'string' ? SKILLWALKER_VERSION : 'dev' From 63c8a438ed47e8ca63f4ff19fa900c34409fff41 Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:18:50 -0600 Subject: [PATCH 2/9] feat(cli): report the package version from --version scripts/build.ts defines SKILLWALKER_VERSION from packages/cli/package.json, so the compiled binary prints it instead of yargs' "unknown". --- packages/cli/index.ts | 2 ++ packages/cli/src/compiled-binary.smoke.test.ts | 13 ++++++++++++- scripts/build.ts | 4 ++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/cli/index.ts b/packages/cli/index.ts index 304e7d8..0af596b 100644 --- a/packages/cli/index.ts +++ b/packages/cli/index.ts @@ -3,10 +3,12 @@ import { SandboxError } from '@testdouble/sandbox-integration' import { SkillwalkerError } from '@testdouble/skillwalker-execution' import yargs from 'yargs' import { hideBin } from 'yargs/helpers' +import { skillwalkerVersion } from './src/version.js' try { await yargs(hideBin(process.argv)) .scriptName('skillwalker') + .version(skillwalkerVersion) .command(await import('./src/commands/test-run.js')) .command(await import('./src/commands/test-eval.js')) .command(await import('./src/commands/sandbox.js')) diff --git a/packages/cli/src/compiled-binary.smoke.test.ts b/packages/cli/src/compiled-binary.smoke.test.ts index eb36b07..0e23a11 100644 --- a/packages/cli/src/compiled-binary.smoke.test.ts +++ b/packages/cli/src/compiled-binary.smoke.test.ts @@ -1,5 +1,5 @@ import { type ChildProcess, spawn, spawnSync } from 'node:child_process' -import { existsSync } from 'node:fs' +import { existsSync, readFileSync } from 'node:fs' import { rm } from 'node:fs/promises' import path from 'node:path' import { fileURLToPath } from 'node:url' @@ -12,6 +12,10 @@ const buildDir = fileURLToPath(new URL('../../../build/', import.meta.url)) const cliBinary = path.join(buildDir, 'skillwalker') const webBinary = path.join(buildDir, 'skillwalker-web') +const cliPackageVersion: string = JSON.parse( + readFileSync(fileURLToPath(new URL('../package.json', import.meta.url)), 'utf8'), +).version + const TEST_RUN_ID = '20260101T100001' const WEB_PORT = 39099 @@ -48,6 +52,13 @@ afterEach(async () => { // ─── compiled binaries ──────────────────────────────────────────────────────── describe('compiled skillwalker binary', () => { + it('reports the CLI package version', () => { + const result = spawnSync(cliBinary, ['--version'], { encoding: 'utf8' }) + + expect(result.status).toBe(0) + expect(result.stdout.trim()).toBe(cliPackageVersion) + }) + it('imports run output into parquet with the bundled DuckDB addon', () => { const result = spawnSync(cliBinary, ['update-analytics-data', '--output-dir', outputDir, '--data-dir', dataDir], { encoding: 'utf8', diff --git a/scripts/build.ts b/scripts/build.ts index e2eeaa6..cc461c2 100755 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -106,6 +106,9 @@ async function copyDuckdbNativeFiles(nativeDir: string): Promise { } } +// The release workflow checks that the pushed tag matches this version +const { version } = await Bun.file(path.join(ROOT, 'packages/cli/package.json')).json() + await mkdir(BUILD_DIR, { recursive: true }) for (const target of COMPILE_TARGETS) { @@ -114,6 +117,7 @@ for (const target of COMPILE_TARGETS) { target: 'bun', compile: { outfile: path.join(BUILD_DIR, target.outfile) }, plugins: [duckdbSidecarPlugin], + define: { SKILLWALKER_VERSION: JSON.stringify(version) }, }) if (!result.success) { From 760a40d2456335ddcd59c55cdef9245d9b340d9d Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:20:07 -0600 Subject: [PATCH 3/9] feat(claude-integration): let SKILLWALKER_SCRIPTS_DIR locate the sandbox scripts A Homebrew install keeps the binary in a versioned Cellar folder, so the scripts folder mounted into the sandbox would change on every upgrade. The override lets an installer point at a folder whose path stays the same. --- .../src/sandbox-scripts.test.ts | 58 +++++++++++++++++++ .../claude-integration/src/sandbox-scripts.ts | 47 ++++++++++++--- 2 files changed, 97 insertions(+), 8 deletions(-) create mode 100644 packages/claude-integration/src/sandbox-scripts.test.ts diff --git a/packages/claude-integration/src/sandbox-scripts.test.ts b/packages/claude-integration/src/sandbox-scripts.test.ts new file mode 100644 index 0000000..e6274ad --- /dev/null +++ b/packages/claude-integration/src/sandbox-scripts.test.ts @@ -0,0 +1,58 @@ +import { mkdtemp, rm, writeFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import path from 'node:path' +import { fileURLToPath } from 'node:url' +import { afterEach, beforeEach, describe, expect, it } from 'vitest' +import { resolveSandboxScripts } from './sandbox-scripts.js' + +const packageDir = fileURLToPath(new URL('..', import.meta.url)) + +let scriptsDir: string + +beforeEach(async () => { + scriptsDir = await mkdtemp(path.join(tmpdir(), 'sandbox-scripts-')) +}) + +afterEach(async () => { + await rm(scriptsDir, { recursive: true, force: true }) +}) + +describe('resolveSandboxScripts', () => { + it('uses the scripts bundled with the package when SKILLWALKER_SCRIPTS_DIR is unset', () => { + const scripts = resolveSandboxScripts({}) + + expect(scripts.runScript).toBe(path.join(packageDir, 'sandbox-run.sh')) + expect(scripts.extractScript).toBe(path.join(packageDir, 'sandbox-extract.sh')) + expect(scripts.scriptsDir).toBe(path.resolve(packageDir)) + }) + + it('uses the scripts in SKILLWALKER_SCRIPTS_DIR when it is set', async () => { + await writeFile(path.join(scriptsDir, 'sandbox-run.sh'), '') + await writeFile(path.join(scriptsDir, 'sandbox-extract.sh'), '') + + const scripts = resolveSandboxScripts({ SKILLWALKER_SCRIPTS_DIR: scriptsDir }) + + expect(scripts).toEqual({ + runScript: path.join(scriptsDir, 'sandbox-run.sh'), + extractScript: path.join(scriptsDir, 'sandbox-extract.sh'), + scriptsDir, + }) + }) + + it('resolves a relative SKILLWALKER_SCRIPTS_DIR to an absolute folder for the sandbox mount', async () => { + await writeFile(path.join(scriptsDir, 'sandbox-run.sh'), '') + await writeFile(path.join(scriptsDir, 'sandbox-extract.sh'), '') + + const scripts = resolveSandboxScripts({ SKILLWALKER_SCRIPTS_DIR: path.relative(process.cwd(), scriptsDir) }) + + expect(scripts.scriptsDir).toBe(scriptsDir) + }) + + it('names SKILLWALKER_SCRIPTS_DIR when that folder is missing a script', async () => { + await writeFile(path.join(scriptsDir, 'sandbox-run.sh'), '') + + expect(() => resolveSandboxScripts({ SKILLWALKER_SCRIPTS_DIR: scriptsDir })).toThrow( + `SKILLWALKER_SCRIPTS_DIR is set to ${scriptsDir}, but it has no sandbox-extract.sh`, + ) + }) +}) diff --git a/packages/claude-integration/src/sandbox-scripts.ts b/packages/claude-integration/src/sandbox-scripts.ts index fa247d6..12e076c 100644 --- a/packages/claude-integration/src/sandbox-scripts.ts +++ b/packages/claude-integration/src/sandbox-scripts.ts @@ -1,12 +1,43 @@ +import fs from 'node:fs' import path from 'node:path' import { resolveRelativePath } from '@testdouble/bun-helpers' -export const sandboxRunScript = resolveRelativePath(import.meta, '../sandbox-run.sh', 'sandbox-run.sh') -export const sandboxExtractScript = resolveRelativePath(import.meta, '../sandbox-extract.sh', 'sandbox-extract.sh') +export interface SandboxScripts { + runScript: string + extractScript: string + /** + * Directory holding the scripts `sbx exec` runs by their host path. The sandbox + * only sees host paths under a mounted workspace, so this directory must be + * mounted alongside the target repo. + */ + scriptsDir: string +} -/** - * Directory holding the scripts `sbx exec` runs by their host path. The sandbox - * only sees host paths under a mounted workspace, so this directory must be - * mounted alongside the target repo. - */ -export const sandboxScriptsDir = path.dirname(sandboxRunScript) +function overrideScript(dir: string, name: string): string { + const script = path.join(dir, name) + if (!fs.existsSync(script)) { + throw new Error(`SKILLWALKER_SCRIPTS_DIR is set to ${dir}, but it has no ${name}`) + } + return script +} + +export function resolveSandboxScripts(env: NodeJS.ProcessEnv): SandboxScripts { + if (env.SKILLWALKER_SCRIPTS_DIR) { + const overrideDir = path.resolve(env.SKILLWALKER_SCRIPTS_DIR) + return { + runScript: overrideScript(overrideDir, 'sandbox-run.sh'), + extractScript: overrideScript(overrideDir, 'sandbox-extract.sh'), + scriptsDir: overrideDir, + } + } + + const runScript = resolveRelativePath(import.meta, '../sandbox-run.sh', 'sandbox-run.sh') + const extractScript = resolveRelativePath(import.meta, '../sandbox-extract.sh', 'sandbox-extract.sh') + return { runScript, extractScript, scriptsDir: path.dirname(runScript) } +} + +const scripts = resolveSandboxScripts(process.env) + +export const sandboxRunScript = scripts.runScript +export const sandboxExtractScript = scripts.extractScript +export const sandboxScriptsDir = scripts.scriptsDir From 84eba02c112588f542c6c2e5d43fdcb8ce3d8eea Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:21:08 -0600 Subject: [PATCH 4/9] fix(sandbox): name the skillwalker command, not ./build/skillwalker, in errors A Homebrew install has no ./build folder, so the retry hints pointed at a path that does not exist. --- packages/sandbox-integration/src/lifecycle.test.ts | 11 +++++++++-- packages/sandbox-integration/src/lifecycle.ts | 6 +++--- packages/sandbox-integration/src/sandbox.test.ts | 4 ++-- packages/sandbox-integration/src/sandbox.ts | 4 ++-- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/packages/sandbox-integration/src/lifecycle.test.ts b/packages/sandbox-integration/src/lifecycle.test.ts index fbb805c..e564d8f 100644 --- a/packages/sandbox-integration/src/lifecycle.test.ts +++ b/packages/sandbox-integration/src/lifecycle.test.ts @@ -76,6 +76,7 @@ describe('createSandbox', () => { await createSandbox('/repo/root') expect(stderrSpy).toHaveBeenCalledWith(expect.stringContaining('already exists')) + expect(stderrSpy).toHaveBeenCalledWith(' skillwalker sandbox create\n') expect((globalThis as any).Bun.spawn).toHaveBeenCalledTimes(1) stderrSpy.mockRestore() @@ -122,7 +123,10 @@ describe('createSandbox', () => { const { createSandbox } = await import('./lifecycle.js') - await expect(createSandbox('/repo/root')).rejects.toBeInstanceOf(SandboxError) + const result = createSandbox('/repo/root') + + await expect(result).rejects.toBeInstanceOf(SandboxError) + await expect(result).rejects.toThrow('Retry with `skillwalker sandbox create`.') expect(stderrSpy).not.toHaveBeenCalledWith(expect.stringContaining('is ready')) }) @@ -275,7 +279,10 @@ describe('updateSandbox', () => { const stderrSpy = vi.spyOn(process.stderr, 'write').mockImplementation(() => true) const { updateSandbox } = await import('./lifecycle.js') - await expect(updateSandbox('/repo/root')).rejects.toThrow(SandboxError) + const result = updateSandbox('/repo/root') + + await expect(result).rejects.toThrow(SandboxError) + await expect(result).rejects.toThrow('Retry with `skillwalker sandbox update`.') stderrSpy.mockRestore() }) diff --git a/packages/sandbox-integration/src/lifecycle.ts b/packages/sandbox-integration/src/lifecycle.ts index a978b45..c77b963 100644 --- a/packages/sandbox-integration/src/lifecycle.ts +++ b/packages/sandbox-integration/src/lifecycle.ts @@ -73,7 +73,7 @@ async function removeTemplateImage(imageId: string): Promise { if (exitCode === 0 || output.includes(TEMPLATE_ALREADY_REMOVED_MESSAGE)) return throw new SandboxError( - `sbx template rm ${imageId} failed (exit code ${exitCode ?? 1}): ${output}\nRetry with \`./build/skillwalker sandbox update\`.`, + `sbx template rm ${imageId} failed (exit code ${exitCode ?? 1}): ${output}\nRetry with \`skillwalker sandbox update\`.`, exitCode, ) } @@ -93,7 +93,7 @@ export async function createSandbox(repoRoot: string, extraWorkspaces: string[] if (await sandboxExists()) { process.stderr.write(`Sandbox "${SANDBOX_NAME}" already exists. To recreate, run:\n`) process.stderr.write(` sbx rm --force ${SANDBOX_NAME}\n`) - process.stderr.write(` ./build/skillwalker sandbox create\n`) + process.stderr.write(` skillwalker sandbox create\n`) return } @@ -109,7 +109,7 @@ export async function createSandbox(repoRoot: string, extraWorkspaces: string[] if (runProc.exitCode !== 0) { throw new SandboxError( - `sbx run failed (exit code ${runProc.exitCode ?? 1}). The sandbox was not created.\nRetry with \`./build/skillwalker sandbox create\`.`, + `sbx run failed (exit code ${runProc.exitCode ?? 1}). The sandbox was not created.\nRetry with \`skillwalker sandbox create\`.`, runProc.exitCode, ) } diff --git a/packages/sandbox-integration/src/sandbox.test.ts b/packages/sandbox-integration/src/sandbox.test.ts index 7fa744b..7f47a9a 100644 --- a/packages/sandbox-integration/src/sandbox.test.ts +++ b/packages/sandbox-integration/src/sandbox.test.ts @@ -54,7 +54,7 @@ describe('ensureSandboxExists', () => { mockSbxLs(JSON.stringify({ sandboxes: null })) const { ensureSandboxExists } = await import('./sandbox.js') - await expect(ensureSandboxExists()).rejects.toThrow(/not found.*sandbox create/) + await expect(ensureSandboxExists()).rejects.toThrow("Run 'skillwalker sandbox create' first.") }) it('resolves when every required path is under a mounted workspace', async () => { @@ -94,7 +94,7 @@ describe('ensureSandboxExists', () => { }) const { ensureSandboxExists } = await import('./sandbox.js') - await expect(ensureSandboxExists()).rejects.toThrow(/sbx login/) + await expect(ensureSandboxExists()).rejects.toThrow('Run `sbx login`, then retry `skillwalker sandbox create`.') }) it('throws SandboxError when sbx is missing', async () => { diff --git a/packages/sandbox-integration/src/sandbox.ts b/packages/sandbox-integration/src/sandbox.ts index 13ad62b..f6d9094 100644 --- a/packages/sandbox-integration/src/sandbox.ts +++ b/packages/sandbox-integration/src/sandbox.ts @@ -35,7 +35,7 @@ async function runSbxLs(args: string[]): Promise { if (proc.exitCode !== 0) { throw new SandboxError( - `Unable to list sandboxes with sbx (exit code ${proc.exitCode ?? 1}): ${stdout}${stderr}\nRun \`sbx login\`, then retry \`./build/skillwalker sandbox create\`.`, + `Unable to list sandboxes with sbx (exit code ${proc.exitCode ?? 1}): ${stdout}${stderr}\nRun \`sbx login\`, then retry \`skillwalker sandbox create\`.`, proc.exitCode, ) } @@ -74,7 +74,7 @@ export async function ensureSandboxExists(requiredPaths: string[] = []): Promise const sandbox = (await listSandboxes()).find(({ name }) => name === SANDBOX_NAME) if (!sandbox) { - throw new SandboxError(`Sandbox "${SANDBOX_NAME}" not found. Run './build/skillwalker sandbox create' first.`, null) + throw new SandboxError(`Sandbox "${SANDBOX_NAME}" not found. Run 'skillwalker sandbox create' first.`, null) } for (const requiredPath of requiredPaths) { From e24b21c8f37778c22a13252d38025fed9317dfaa Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:21:50 -0600 Subject: [PATCH 5/9] fix(build): ad-hoc re-sign compiled binaries on macOS bun build --compile appends its bundle after the linker signs the file, so codesign --verify rejected both binaries. Strip the stale signature, sign ad hoc, and verify, then smoke test the result on macOS. --- .../cli/src/compiled-binary.smoke.test.ts | 10 ++++++++ scripts/build.ts | 24 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/packages/cli/src/compiled-binary.smoke.test.ts b/packages/cli/src/compiled-binary.smoke.test.ts index 0e23a11..d6469e0 100644 --- a/packages/cli/src/compiled-binary.smoke.test.ts +++ b/packages/cli/src/compiled-binary.smoke.test.ts @@ -71,6 +71,16 @@ describe('compiled skillwalker binary', () => { }) }) +describe('compiled binary signatures', () => { + // A binary whose signature fails to verify is killed on launch by recent macOS releases + it.skipIf(process.platform !== 'darwin').each([cliBinary, webBinary])('%s passes codesign verification', (binary) => { + const result = spawnSync('codesign', ['--verify', binary], { encoding: 'utf8' }) + + expect(result.stderr).toBe('') + expect(result.status).toBe(0) + }) +}) + describe('compiled skillwalker-web binary', () => { let server: ChildProcess | undefined diff --git a/scripts/build.ts b/scripts/build.ts index cc461c2..8070138 100755 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -88,6 +88,25 @@ function resolveDuckdbNativeDir(): string { throw new Error(`No DuckDB native bindings installed for ${platformArch}. Tried: ${candidates.join(', ')}`) } +function codesign(args: string[]): void { + const result = Bun.spawnSync(['codesign', ...args], { stderr: 'pipe' }) + if (result.exitCode !== 0) { + throw new Error(`codesign ${args.join(' ')} failed: ${result.stderr.toString().trim()}`) + } +} + +/** + * `bun build --compile` appends the bundle to an executable the linker has + * already signed, which leaves a signature that no longer matches the file, and + * recent macOS releases kill such a binary on launch. The stale signature has to + * be stripped before an ad-hoc one can replace it. + */ +function adHocSign(binary: string): void { + codesign(['--remove-signature', binary]) + codesign(['--force', '--sign', '-', binary]) + codesign(['--verify', binary]) +} + /** Copies the native files, skipping any that are already in place unchanged. */ async function copyDuckdbNativeFiles(nativeDir: string): Promise { const artifacts = (await readdir(nativeDir)).filter((name) => !BINDINGS_METADATA_FILES.has(name)) @@ -126,6 +145,11 @@ for (const target of COMPILE_TARGETS) { } console.log(` compiled build/${target.outfile}`) + + if (process.platform === 'darwin') { + adHocSign(path.join(BUILD_DIR, target.outfile)) + console.log(` signed build/${target.outfile}`) + } } await copyDuckdbNativeFiles(resolveDuckdbNativeDir()) From a28edaa4767607074585e9bd1fb188916d7ed81c Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:22:32 -0600 Subject: [PATCH 6/9] test: smoke test the compiled CLI in a Homebrew-style layout Cover running through a bin symlink into libexec from another directory, and reading the sandbox scripts from SKILLWALKER_SCRIPTS_DIR. --- .../cli/src/compiled-binary.smoke.test.ts | 60 ++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/compiled-binary.smoke.test.ts b/packages/cli/src/compiled-binary.smoke.test.ts index d6469e0..fe6ea78 100644 --- a/packages/cli/src/compiled-binary.smoke.test.ts +++ b/packages/cli/src/compiled-binary.smoke.test.ts @@ -1,6 +1,6 @@ import { type ChildProcess, spawn, spawnSync } from 'node:child_process' import { existsSync, readFileSync } from 'node:fs' -import { rm } from 'node:fs/promises' +import { cp, mkdir, readdir, rm, symlink, writeFile } from 'node:fs/promises' import path from 'node:path' import { fileURLToPath } from 'node:url' import { makeTmpDir, writeRunFixture } from '@testdouble/skillwalker-data/src/analytics-test-helpers.js' @@ -32,6 +32,23 @@ async function waitForServer(url: string, proc: ChildProcess, timeoutMs = 10000) throw new Error(`skillwalker-web did not answer ${url} within ${timeoutMs}ms`) } +/** + * Lays the build out the way Homebrew installs it: real files in libexec/ and a + * relative symlink to the CLI in bin/. + */ +async function installLikeHomebrew(prefix: string): Promise { + const libexec = path.join(prefix, 'libexec') + await mkdir(libexec, { recursive: true }) + for (const file of await readdir(buildDir)) { + await cp(path.join(buildDir, file), path.join(libexec, file)) + } + + const bin = path.join(prefix, 'bin') + await mkdir(bin) + await symlink('../libexec/skillwalker', path.join(bin, 'skillwalker')) + return path.join(bin, 'skillwalker') +} + // ─── test lifecycle ─────────────────────────────────────────────────────────── let tmpDir: string @@ -71,6 +88,47 @@ describe('compiled skillwalker binary', () => { }) }) +describe('compiled skillwalker binary installed like Homebrew', () => { + it('loads its sidecar files through a bin symlink run from another directory', async () => { + const linkedBinary = await installLikeHomebrew(path.join(tmpDir, 'prefix')) + + const result = spawnSync(linkedBinary, ['update-analytics-data', '--output-dir', outputDir, '--data-dir', dataDir], { + cwd: tmpDir, + encoding: 'utf8', + }) + + expect(result.status).toBe(0) + expect(existsSync(path.join(dataDir, 'test-run.parquet'))).toBe(true) + }) + + it('reads the sandbox scripts from SKILLWALKER_SCRIPTS_DIR', async () => { + const scriptsDir = path.join(tmpDir, 'sandbox-scripts') + await mkdir(scriptsDir) + await writeFile(path.join(scriptsDir, 'sandbox-run.sh'), '') + await writeFile(path.join(scriptsDir, 'sandbox-extract.sh'), '') + + const result = spawnSync(cliBinary, ['--help'], { + encoding: 'utf8', + env: { ...process.env, SKILLWALKER_SCRIPTS_DIR: scriptsDir }, + }) + + expect(result.status).toBe(0) + }) + + it('names SKILLWALKER_SCRIPTS_DIR when that folder has no sandbox scripts', async () => { + const scriptsDir = path.join(tmpDir, 'empty-scripts') + await mkdir(scriptsDir) + + const result = spawnSync(cliBinary, ['--help'], { + encoding: 'utf8', + env: { ...process.env, SKILLWALKER_SCRIPTS_DIR: scriptsDir }, + }) + + expect(result.status).not.toBe(0) + expect(result.stderr).toContain(`SKILLWALKER_SCRIPTS_DIR is set to ${scriptsDir}, but it has no sandbox-run.sh`) + }) +}) + describe('compiled binary signatures', () => { // A binary whose signature fails to verify is killed on launch by recent macOS releases it.skipIf(process.platform !== 'darwin').each([cliBinary, webBinary])('%s passes codesign verification', (binary) => { From 470b5bd1c215bc34fb7383dc6401373aa97d6b59 Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:23:20 -0600 Subject: [PATCH 7/9] ci: build and draft a macOS release on version tags Pushing a v* tag checks it against packages/cli/package.json, builds and smoke tests arm64 and x86_64 binaries, and attaches the tarballs and their checksums to a draft GitHub Release. CI also smoke tests the binaries on macOS, where the codesign check runs. --- .github/workflows/ci.yml | 10 +++++ .github/workflows/release.yml | 73 +++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 .github/workflows/release.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd167d9..988f18e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -62,6 +62,16 @@ jobs: - run: make build - run: bun run test:smoke + test-compiled-binaries-macos: + name: Compiled Binary Smoke Tests (macOS) + runs-on: macos-15 + steps: + - uses: actions/checkout@v7 + - uses: oven-sh/setup-bun@v2 + - run: bun install --frozen-lockfile + - run: make build + - run: bun run test:smoke + security: name: Security Audit runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..1a09e48 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,73 @@ +name: Release + +on: + push: + tags: ['v*'] + +jobs: + check-version: + name: Check Tag Matches Version + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - name: Compare the tag with packages/cli/package.json + run: | + package_version=$(jq -r .version packages/cli/package.json) + if [ "${GITHUB_REF_NAME#v}" != "$package_version" ]; then + echo "::error::Tag $GITHUB_REF_NAME does not match packages/cli/package.json version $package_version" + exit 1 + fi + + build: + name: Build macOS ${{ matrix.arch }} + needs: check-version + runs-on: ${{ matrix.runner }} + strategy: + matrix: + include: + - runner: macos-15 + arch: arm64 + - runner: macos-15-intel + arch: x86_64 + steps: + - uses: actions/checkout@v7 + - uses: oven-sh/setup-bun@v2 + with: + bun-version-file: package.json + - run: bun install --frozen-lockfile + - run: make build + - run: bun run test:smoke + - name: Package the build folder + run: | + version=${GITHUB_REF_NAME#v} + name="skillwalker-$version-darwin-${{ matrix.arch }}" + mkdir -p "dist/$name" + cp build/skillwalker build/skillwalker-web build/duckdb.node build/libduckdb.dylib \ + build/sandbox-run.sh build/sandbox-extract.sh "dist/$name/" + tar -czf "dist/$name.tar.gz" -C dist "$name" + shasum -a 256 "dist/$name.tar.gz" | tee "dist/$name.tar.gz.sha256" + - uses: actions/upload-artifact@v4 + with: + name: darwin-${{ matrix.arch }} + path: dist/*.tar.gz* + + release: + name: Draft GitHub Release + needs: build + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - uses: actions/download-artifact@v4 + with: + path: dist + merge-multiple: true + - name: Create a draft release with the archives + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "$GITHUB_REF_NAME" dist/* \ + --repo "$GITHUB_REPOSITORY" \ + --draft \ + --generate-notes \ + --title "Skillwalker $GITHUB_REF_NAME" From 9c1774891a228caff83c44e05a5a970b6e8ee299 Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:24:15 -0600 Subject: [PATCH 8/9] docs: describe --version, build signing, releases, and SKILLWALKER_SCRIPTS_DIR --- docs/claude-integration.md | 14 +++++++------- docs/cli.md | 18 +++++++++++++++++- docs/sandbox-integration-package.md | 6 +++--- docs/sandbox-integration.md | 4 ++-- 4 files changed, 29 insertions(+), 13 deletions(-) diff --git a/docs/claude-integration.md b/docs/claude-integration.md index 1d8558a..1cb981c 100644 --- a/docs/claude-integration.md +++ b/docs/claude-integration.md @@ -121,15 +121,15 @@ The `--print` flag is always placed last in the argument array, after all other ### Sandbox Script Resolution -The `sandbox-run.sh` script path is resolved at module load time using `resolveRelativePath` from `@testdouble/bun-helpers`. This utility handles cross-runtime path resolution across Bun, Vitest, and the compiled binaries. In a compiled binary the script is read from beside the executable, where `scripts/build.ts` copies it: +The `sandbox-run.sh` and `sandbox-extract.sh` paths are resolved once, at module load, by `resolveSandboxScripts(process.env)` in `packages/claude-integration/src/sandbox-scripts.ts`. + +When `SKILLWALKER_SCRIPTS_DIR` is set, both scripts are read from that folder, resolved to an absolute path. If either script is missing there, loading the module throws an error naming the variable, so every command fails at startup rather than at the first sandbox run. An installer uses this to keep the scripts in a folder whose path stays the same across upgrades: `sandbox create` mounts `sandboxScriptsDir` into the sandbox, and a Homebrew install's own folder changes with each version. + +Otherwise the paths come from `resolveRelativePath` in `@testdouble/bun-helpers`, which handles cross-runtime path resolution across Bun, Vitest, and the compiled binaries. In a compiled binary the scripts are read from beside the executable, where `scripts/build.ts` copies them: ```typescript -// packages/claude-integration/src/run-claude.ts -const sandboxRunScript = resolveRelativePath( - import.meta, - '../sandbox-run.sh', - 'sandbox-run.sh', -) +// packages/claude-integration/src/sandbox-scripts.ts +const runScript = resolveRelativePath(import.meta, '../sandbox-run.sh', 'sandbox-run.sh') ``` ### Sandbox Script Behavior diff --git a/docs/cli.md b/docs/cli.md index 1c2b723..345f504 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -6,7 +6,7 @@ This page documents the CLI boundary: the six top-level commands the `skillwalke The `@testdouble/skillwalker-cli` package is the command-line entry point for Skillwalker. It is a thin Yargs wrapper that parses arguments, resolves paths from `process.cwd()`, and delegates all pipeline orchestration to `@testdouble/skillwalker-execution`. -- **Last Updated:** 2026-05-15 +- **Last Updated:** 2026-09-23 - **Authors:** - River Bailey (river.bailey@testdouble.com) @@ -122,10 +122,26 @@ The CLI catches `SkillwalkerError` at the top level (`index.ts`) and writes the | `--apply` | `scil`, `acil` | Auto-apply best description | `false` | | `--output-dir` | `update-analytics-data` | Path to test output directory | `tests/output/` | | `--data-dir` | `update-analytics-data` | Path to analytics data directory | `tests/analytics/` | +| `--version` | all | Print the CLI version: `packages/cli/package.json`'s version in a compiled binary, `dev` from source | — | + +| Environment variable | Description | Default | +|----------------------|-------------|---------| +| `SKILLWALKER_SCRIPTS_DIR` | Folder holding `sandbox-run.sh` and `sandbox-extract.sh`. An installer sets it so the folder mounted into the sandbox keeps the same path across upgrades. See [Claude Integration](./claude-integration.md#sandbox-script-resolution). | beside the executable | + +### Build and Release + +`make build` runs `scripts/build.ts`, which compiles `skillwalker` and `skillwalker-web` into `build/` and copies the DuckDB native files and sandbox scripts beside them. Two details matter for a distributable build: + +- **Version.** The build passes `packages/cli/package.json`'s `version` to `Bun.build` as the `SKILLWALKER_VERSION` define. `packages/cli/src/version.ts` reads it and falls back to `dev` when running from source. +- **Signing.** On macOS, `bun build --compile` appends the bundle after the linker has signed the executable, so the signature no longer matches the file and recent macOS releases kill it on launch. The build strips that signature, signs each binary ad hoc, and runs `codesign --verify`, failing the build if any step fails. + +Pushing a `v*` tag runs `.github/workflows/release.yml`. It fails unless the tag matches `packages/cli/package.json`'s version, then builds and smoke tests on arm64 and x86_64 macOS runners. Each build folder is packaged as `skillwalker--darwin-.tar.gz` with a `.sha256` file, and all of them are attached to a draft GitHub Release. ## Testing - `packages/cli/src/paths.test.ts` — Tests `createPathConfig` and `getAllEvals` +- `packages/cli/src/version.test.ts` — Tests the `dev` version fallback when running from source +- `packages/cli/src/compiled-binary.smoke.test.ts` — Runs the binaries in `build/` (`make build && bun run test:smoke`): `--version`, DuckDB loading, a Homebrew-style `bin` symlink into `libexec`, `SKILLWALKER_SCRIPTS_DIR`, and `codesign --verify` on macOS - `packages/cli/src/commands/test-run.test.ts` — Tests `test-run` command builder and handler - `packages/cli/src/commands/test-eval.test.ts` — Tests `test-eval` command builder and handler - `packages/cli/src/commands/scil.test.ts` — Tests `scil` command builder and handler diff --git a/docs/sandbox-integration-package.md b/docs/sandbox-integration-package.md index 4ce17b3..effecfc 100644 --- a/docs/sandbox-integration-package.md +++ b/docs/sandbox-integration-package.md @@ -72,7 +72,7 @@ class SandboxError extends Error { async function ensureSandboxExists(requiredPaths: string[] = []): Promise ``` -Pre-flight check that the sandbox exists and mounts what the run needs. Runs `sbx ls --json`, finds the entry named `SANDBOX_NAME`, and checks that every path in `requiredPaths` is inside one of its workspaces (a trailing `:ro` on a listed workspace is ignored). `runEvals` and the SCIL and ACIL loops pass `[sandboxScriptsDir]`. A sandbox created before the scripts mount was added keeps its old workspaces, so this check fails it before any test runs instead of at the first `sbx exec`. Throws `SandboxError` with `exitCode: null` if the sandbox is not found, with a message directing the user to run `./build/skillwalker sandbox create`. +Pre-flight check that the sandbox exists and mounts what the run needs. Runs `sbx ls --json`, finds the entry named `SANDBOX_NAME`, and checks that every path in `requiredPaths` is inside one of its workspaces (a trailing `:ro` on a listed workspace is ignored). `runEvals` and the SCIL and ACIL loops pass `[sandboxScriptsDir]`. A sandbox created before the scripts mount was added keeps its old workspaces, so this check fails it before any test runs instead of at the first `sbx exec`. Throws `SandboxError` with `exitCode: null` if the sandbox is not found, with a message directing the user to run `skillwalker sandbox create`. **Consumers:** - `cli/src/commands/test-run.ts` -- before the per-eval test loop @@ -112,7 +112,7 @@ async function createSandbox(repoRoot: string, extraWorkspaces: string[] = []): Checks whether the sandbox already exists via an internal `sandboxExists()` helper (runs `sbx ls --quiet`). If found, prints a help message to stderr explaining how to recreate it, and returns early. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude [:ro ...]` with inherited stdio for interactive OAuth login. Prints progress messages to stderr. Throws `SandboxError` if `sbx run` exits non-zero. -`extraWorkspaces` are mounted read-only after `repoRoot` (`:ro`); any already inside `repoRoot` are skipped. The CLI passes the directory holding `sandbox-run.sh` and `sandbox-extract.sh` (`sandboxScriptsDir` from `@testdouble/claude-integration`). `execInSandbox` runs those scripts by their host path, and the sandbox only sees host paths under a mounted workspace, so without this mount every test run fails whenever the target repo is not the skillwalker repo. +`extraWorkspaces` are mounted read-only after `repoRoot` (`:ro`); any already inside `repoRoot` are skipped. The CLI passes the directory holding `sandbox-run.sh` and `sandbox-extract.sh` (`sandboxScriptsDir` from `@testdouble/claude-integration`, or `SKILLWALKER_SCRIPTS_DIR` when set). `execInSandbox` runs those scripts by their host path, and the sandbox only sees host paths under a mounted workspace, so without this mount every test run fails whenever the target repo is not the skillwalker repo. **Consumer:** `cli/src/commands/sandbox/create.ts` @@ -210,7 +210,7 @@ flowchart TB | Scenario | Error Type | Behavior | |----------|------------|----------| -| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `./build/skillwalker sandbox create` | +| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `skillwalker sandbox create` | | Required path not mounted, checked by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown naming the unmounted path, with a hint to run `skillwalker sandbox update` from the target repo | | `sbx rm` fails | `SandboxError` (exitCode: process code) | Thrown with stdout+stderr in message | | `sbx exec` prints `OCI runtime exec failed` (exits 0) | `SandboxError` (exitCode: process code) | Thrown with the sbx output and a hint to run `skillwalker sandbox update` from the target repo | diff --git a/docs/sandbox-integration.md b/docs/sandbox-integration.md index 2525204..808b65e 100644 --- a/docs/sandbox-integration.md +++ b/docs/sandbox-integration.md @@ -173,7 +173,7 @@ export async function createSandbox(repoRoot: string, extraWorkspaces: string[] Checks if the sandbox already exists via an internal `sandboxExists()` helper. If it does, prints a help message to stderr and returns. Otherwise, spawns `sbx run --name claude-skills-skillwalker claude [:ro ...]` with inherited stdio for interactive OAuth login. If `sbx run` exits non-zero, it throws `SandboxError` instead of reporting the sandbox as ready. -`extraWorkspaces` are mounted read-only after `repoRoot` (`:ro`); any already inside `repoRoot` are skipped. The CLI passes the directory holding `sandbox-run.sh` and `sandbox-extract.sh` (`sandboxScriptsDir` from `@testdouble/claude-integration`). `execInSandbox` runs those scripts by their host path, and the sandbox only sees host paths under a mounted workspace, so without this mount every test run fails whenever the target repo is not the skillwalker repo. +`extraWorkspaces` are mounted read-only after `repoRoot` (`:ro`); any already inside `repoRoot` are skipped. The CLI passes the directory holding `sandbox-run.sh` and `sandbox-extract.sh` (`sandboxScriptsDir` from `@testdouble/claude-integration`, or `SKILLWALKER_SCRIPTS_DIR` when set). `execInSandbox` runs those scripts by their host path, and the sandbox only sees host paths under a mounted workspace, so without this mount every test run fails whenever the target repo is not the skillwalker repo. Called by `commands/sandbox/create.ts`. @@ -226,7 +226,7 @@ See [Cross-Runtime Meta Property Resolution](coding-standards/cross-runtime-meta | Scenario | Error Type | Behavior | |----------|------------|----------| -| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `./build/skillwalker sandbox create` | +| Sandbox not found by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown with message suggesting `skillwalker sandbox create` | | Required path not mounted, checked by `ensureSandboxExists` | `SandboxError` (exitCode: `null`) | Thrown naming the unmounted path, with a hint to run `skillwalker sandbox update` from the target repo | | `sbx rm` fails | `SandboxError` (exitCode: process code) | Thrown with stdout+stderr in message | | `sbx exec` prints `OCI runtime exec failed` (exits 0) | `SandboxError` (exitCode: process code) | Thrown with the sbx output and a hint to run `skillwalker sandbox update` from the target repo | From 98a8b2d251d7cc6281e63ed9439c1c89cb723d9a Mon Sep 17 00:00:00 2001 From: River Bailey Date: Wed, 23 Sep 2026 07:24:57 -0600 Subject: [PATCH 9/9] style: format the compiled binary smoke test --- packages/cli/src/compiled-binary.smoke.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/cli/src/compiled-binary.smoke.test.ts b/packages/cli/src/compiled-binary.smoke.test.ts index fe6ea78..dbed3a7 100644 --- a/packages/cli/src/compiled-binary.smoke.test.ts +++ b/packages/cli/src/compiled-binary.smoke.test.ts @@ -92,10 +92,14 @@ describe('compiled skillwalker binary installed like Homebrew', () => { it('loads its sidecar files through a bin symlink run from another directory', async () => { const linkedBinary = await installLikeHomebrew(path.join(tmpDir, 'prefix')) - const result = spawnSync(linkedBinary, ['update-analytics-data', '--output-dir', outputDir, '--data-dir', dataDir], { - cwd: tmpDir, - encoding: 'utf8', - }) + const result = spawnSync( + linkedBinary, + ['update-analytics-data', '--output-dir', outputDir, '--data-dir', dataDir], + { + cwd: tmpDir, + encoding: 'utf8', + }, + ) expect(result.status).toBe(0) expect(existsSync(path.join(dataDir, 'test-run.parquet'))).toBe(true)