diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9eb2ca5852..6db740b719 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -199,7 +199,13 @@ jobs: - { target: cloud, shard: 2/4, shard-name: 2of4 } - { target: cloud, shard: 3/4, shard-name: 3of4 } - { target: cloud, shard: 4/4, shard-name: 4of4 } - - target: selfhost + # Selfhost shards the same way: each shard is its own runner booting + # its own fresh instance (own port block + data dir), so the + # project's shared-bootstrap-admin assumption stays intact per shard + # and `fileParallelism: false` still serializes within a shard. + - { target: selfhost, shard: 1/3, shard-name: 1of3 } + - { target: selfhost, shard: 2/3, shard-name: 2of3 } + - { target: selfhost, shard: 3/3, shard-name: 3of3 } runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 steps: @@ -254,7 +260,7 @@ jobs: - name: Run selfhost scenarios if: matrix.target == 'selfhost' - run: bunx vitest run --project selfhost --retry=2 + run: bunx vitest run --project selfhost --retry=2 ${{ matrix.shard && format('--shard={0}', matrix.shard) || '' }} working-directory: e2e # Failed runs keep their trace.zip / session.mp4 / step screenshots in diff --git a/apps/host-selfhost/src/config.ts b/apps/host-selfhost/src/config.ts index e0cd282d52..dae13a5e16 100644 --- a/apps/host-selfhost/src/config.ts +++ b/apps/host-selfhost/src/config.ts @@ -43,6 +43,14 @@ export interface SelfHostConfig { readonly organizationName: string; /** URL slug for org-prefixed console paths (`//policies`). */ readonly orgSlug: string; + /** + * Sandbox execution budget passed to the QuickJS runtime, or undefined for + * the runtime's own default (5 minutes). An operator knob in principle, but + * its real consumer is the e2e harness, which shrinks it to seconds so the + * sandbox-deadline scenario proves its race without waiting out real + * minutes (the same pattern as MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS on cloud). + */ + readonly sandboxTimeoutMs: number | undefined; } export const resolveDataDir = (): string => @@ -148,9 +156,26 @@ export const loadConfig = (): SelfHostConfig => { bootstrapAdminName: process.env.EXECUTOR_BOOTSTRAP_ADMIN_NAME ?? "Admin", organizationName: process.env.EXECUTOR_ORG_NAME ?? "Default", orgSlug: resolveOrgSlug(), + sandboxTimeoutMs: resolveSandboxTimeoutMs(), }; }; +// A malformed value is refused rather than silently ignored: an operator who +// sets the knob and typos it should find out at boot, not by watching a +// runaway execution use the 5-minute default. +const resolveSandboxTimeoutMs = (): number | undefined => { + const raw = process.env.EXECUTOR_SANDBOX_TIMEOUT_MS; + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) { + // oxlint-disable-next-line executor/no-try-catch-or-throw, executor/no-error-constructor -- boundary: refuse to boot on a malformed operator knob + throw new Error( + `EXECUTOR_SANDBOX_TIMEOUT_MS ${JSON.stringify(raw)} is not a positive number of milliseconds`, + ); + } + return Math.floor(parsed); +}; + // The org slug doubles as a URL segment (`//policies`), so an // operator-set value must fit the shared grammar and avoid reserved root // segments (api, mcp, login, …) — a colliding slug would shadow real routes. diff --git a/apps/host-selfhost/src/execution.ts b/apps/host-selfhost/src/execution.ts index 270ffc4f8e..aa2ffee536 100644 --- a/apps/host-selfhost/src/execution.ts +++ b/apps/host-selfhost/src/execution.ts @@ -65,7 +65,12 @@ export const SelfHostHostConfig: Layer.Layer = Layer.sync(HostConfig export const SelfHostCodeExecutorProvider: Layer.Layer = Layer.sync( CodeExecutorProvider, - () => makeQuickJsExecutor(), + () => { + const { sandboxTimeoutMs } = loadConfig(); + return makeQuickJsExecutor( + sandboxTimeoutMs === undefined ? {} : { timeoutMs: sandboxTimeoutMs }, + ); + }, ); /** diff --git a/e2e/scenarios/resume-after-sandbox-deadline.test.ts b/e2e/scenarios/resume-after-sandbox-deadline.test.ts index 42c9154424..47bccc6235 100644 --- a/e2e/scenarios/resume-after-sandbox-deadline.test.ts +++ b/e2e/scenarios/resume-after-sandbox-deadline.test.ts @@ -8,14 +8,18 @@ // unknown execution. // // The journey drives exactly that shape: ONE execution with TWO approval -// gates. The first approval is granted late in its window (~3.5 min), so the -// second pause's window reaches well past the sandbox's 5-minute mark. The -// second approval arrives ~5.75 min after execution start — inside its OWN -// advertised window, but past the old absolute deadline. Deliberately slow -// (~6 min): the elapsed time IS the subject under test. A single-pause -// variant cannot express this cross-target — hosts that advertise a -// 4-minute window would expire it legitimately before the sandbox clock -// even matters. +// gates. The first approval is granted late (70% of the sandbox budget in), +// so the second pause's window reaches well past the budget. The second +// approval arrives at ~115% of the budget after execution start — inside its +// OWN advertised window, but past the old absolute deadline. The subject is +// that RATIO, not any absolute duration, so the delays scale off the budget +// the target was booted with: selfhost boots with a seconds-long +// EXECUTOR_SANDBOX_TIMEOUT_MS (setup/sandbox-timeout.ts) and proves the race +// in ~25s; a target on the production 5-minute budget runs the original +// ~6-minute journey (the elapsed time IS the subject — nothing is mocked). A +// single-pause variant cannot express this cross-target — hosts that +// advertise a 4-minute window would expire it legitimately before the +// sandbox clock even matters. // // The gate is `policies.create`'s own `requiresApproval` annotation // (hermetic, same device as policy-tool-approval.test.ts); both approvals @@ -29,15 +33,23 @@ import { composePluginApi } from "@executor-js/api/server"; import { scenario } from "../src/scenario"; import { Api, Mcp, Target } from "../src/services"; import { configuredMcpPausedSessionIdleTimeoutMs } from "../setup/mcp-session-timeouts"; +import { configuredSandboxTimeoutMs } from "../setup/sandbox-timeout"; const coreApi = composePluginApi([] as const); -// Grant the first approval at 3.5 min — late but inside its 4-minute window. -// The second pause then opens a fresh window reaching ~7.5 min. -const FIRST_APPROVAL_DELAY_MS = 3.5 * 60_000; -// Grant the second approval 2.25 min later: ~5.75 min after execution start, -// past the sandbox's 5-minute budget but inside the second window. -const SECOND_APPROVAL_DELAY_MS = 2.25 * 60_000; +const SANDBOX_BUDGET_MS = configuredSandboxTimeoutMs(); + +// Grant the first approval at 70% of the budget — late but inside its window +// (was 3.5 of 5 min). The second pause then opens a fresh window reaching +// past the budget. +const FIRST_APPROVAL_DELAY_MS = 0.7 * SANDBOX_BUDGET_MS; +// Grant the second approval 45% of the budget later: ~115% of the budget +// after execution start, past the sandbox clock but inside the second window +// (was 2.25 of 5 min → ~5.75 min total). +const SECOND_APPROVAL_DELAY_MS = 0.45 * SANDBOX_BUDGET_MS; +// The whole journey plus scheduling slack, for the idle-window guard and the +// vitest timeout. +const JOURNEY_MS = FIRST_APPROVAL_DELAY_MS + SECOND_APPROVAL_DELAY_MS; /** Sandbox code that creates two policies through the approval-gated core * tool. Patterns are unique-per-run and match no real tool, so the rules are @@ -56,19 +68,22 @@ const second = await tools.executor.coreTools.policies.create({ return JSON.stringify({ first: first.ok, second: second.ok }); `; -// The journey spans ~6 real minutes of paused waiting, so the host must keep -// the paused session alive that long. The suite's default e2e override shrinks +// The journey spans the whole paused waiting time, so the host must keep the +// paused session alive that long. The suite's default e2e override shrinks // the paused-session idle teardown to seconds (to keep teardown tests fast), // which would evict the session mid-scenario for reasons unrelated to the -// clock under test — require the production-like window instead. +// clock under test — require a window that outlasts the journey instead. +// With a shrunken sandbox budget the journey shrinks too, so even the short +// e2e idle window can suffice; the guard compares the two rather than +// hardcoding either. const PAUSED_IDLE_WINDOW_TOO_SHORT = - configuredMcpPausedSessionIdleTimeoutMs() < 8 * 60_000 - ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ~6-minute journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= 480000 to run it` + configuredMcpPausedSessionIdleTimeoutMs() < JOURNEY_MS + 60_000 + ? `the target's paused-session idle teardown (${configuredMcpPausedSessionIdleTimeoutMs()}ms) evicts the session before this ${Math.round(JOURNEY_MS / 1000)}s journey completes; boot the target with MCP_PAUSED_SESSION_IDLE_TIMEOUT_MS >= ${JOURNEY_MS + 60_000} or a smaller E2E_SANDBOX_TIMEOUT_MS to run it` : undefined; scenario( "MCP · chained approvals granted within their windows survive the sandbox clock", - { timeout: 480_000, skip: PAUSED_IDLE_WINDOW_TOO_SHORT }, + { timeout: Math.max(120_000, JOURNEY_MS + 120_000), skip: PAUSED_IDLE_WINDOW_TOO_SHORT }, Effect.gen(function* () { const target = yield* Target; const apiSurface = yield* Api; diff --git a/e2e/setup/sandbox-timeout.ts b/e2e/setup/sandbox-timeout.ts new file mode 100644 index 0000000000..51a85df37d --- /dev/null +++ b/e2e/setup/sandbox-timeout.ts @@ -0,0 +1,28 @@ +// The sandbox execution budget shared between a target's boot env and the +// sandbox-deadline scenario, so they cannot drift apart (same pattern as +// execution-limits.ts). The scenario proves a RATIO — approvals granted +// inside their own windows survive an execution that outlives the sandbox's +// absolute budget — so the budget's magnitude is free to shrink: on selfhost +// the boot recipe passes E2E_SANDBOX_TIMEOUT_MS through to the server as +// EXECUTOR_SANDBOX_TIMEOUT_MS and the scenario scales its approval delays to +// match, turning a ~6-minute real-time wait into seconds. Targets that cannot +// shrink the budget (cloud's dynamic-worker deadline is not env-tunable) run +// against the production default and skip via their paused-session window +// guard instead. +export const E2E_SANDBOX_TIMEOUT_MS = 20_000; + +export const SANDBOX_TIMEOUT_ENV = "E2E_SANDBOX_TIMEOUT_MS"; + +const PRODUCTION_SANDBOX_TIMEOUT_MS = 5 * 60_000; + +const positiveMilliseconds = (raw: string | undefined): number | undefined => { + if (!raw) return undefined; + const parsed = Number(raw); + if (!Number.isFinite(parsed) || parsed <= 0) return undefined; + return Math.floor(parsed); +}; + +/** The sandbox budget the current target enforces: the harness override when + * the target was booted with one, else the production default. */ +export const configuredSandboxTimeoutMs = (): number => + positiveMilliseconds(process.env[SANDBOX_TIMEOUT_ENV]) ?? PRODUCTION_SANDBOX_TIMEOUT_MS; diff --git a/e2e/setup/selfhost.boot.ts b/e2e/setup/selfhost.boot.ts index 6f0d743d7b..a705e4e543 100644 --- a/e2e/setup/selfhost.boot.ts +++ b/e2e/setup/selfhost.boot.ts @@ -21,6 +21,9 @@ export interface SelfhostBootOptions { /** vite --host (e.g. "0.0.0.0" to be tailnet-reachable). */ readonly host?: string; readonly logFile?: string; + /** Shrink the sandbox execution budget (EXECUTOR_SANDBOX_TIMEOUT_MS) so + * deadline scenarios prove their race in seconds. Omit for production. */ + readonly sandboxTimeoutMs?: number; } export const bootSelfhost = async (options: SelfhostBootOptions): Promise => { @@ -51,6 +54,9 @@ export const bootSelfhost = async (options: SelfhostBootOptions): Promise Promise) | void> { [{ envVar: "E2E_SELFHOST_PORT", offset: 4, label: "selfhost vite dev" }], async (ports) => { const port = ports.E2E_SELFHOST_PORT!; + // Shrink the sandbox execution budget and publish the value to the test + // workers (spawned after this globalsetup, so they inherit the env): the + // sandbox-deadline scenario reads it to scale its approval delays. + process.env[SANDBOX_TIMEOUT_ENV] = String(E2E_SANDBOX_TIMEOUT_MS); // Fresh data dir per suite run — hermetic; in-suite isolation comes from // fresh identities, not resets (bootSelfhost wipes it). const procs = await bootSelfhost({ @@ -44,6 +49,7 @@ export default async function setup(): Promise<(() => Promise) | void> { webBaseUrl: `http://localhost:${port}`, admin: SELFHOST_ADMIN, logFile: bootLogFile, + sandboxTimeoutMs: E2E_SANDBOX_TIMEOUT_MS, }); return { teardown: procs.teardown, value: procs }; },