[Fix] Destroy compute sandboxes when task runs are canceled

apps/api/src/handlers/tasks/cancelTask.ts

@@ -15,6 +15,7 @@ import {
   isExitedRunStatus,
 } from '@roomote/types';
 import { captureTaskSettled } from '@roomote/telemetry/server';
+import { destroyCanceledTaskRunSandbox } from '@roomote/sdk/server';
 
 import type { Variables } from '../../types';
 import type { McpAuth } from '../mcp/middleware';
@@ -88,6 +89,14 @@ export async function cancelTask(
 
     if (canceledRun) {
       void captureTaskSettled(canceledRun.id, 'canceled');
+
+      // This cancel writes the terminal state directly (it never reaches
+      // finishRun), so tear down any attached sandbox here or it keeps
+      // running against the provider's capacity until its TTL.
+      await destroyCanceledTaskRunSandbox({
+        runId: canceledRun.id,
+        logPrefix: 'cancelTask',
+      });
     }
 
     return c.json({ success: true });

apps/api/src/handlers/tasks/task-stop.ts

@@ -1,5 +1,8 @@
 import { TRPCClientError } from '@trpc/client';
-import { withSandboxServerRpcClient } from '@roomote/sdk/server';
+import {
+  destroyCanceledTaskRunSandbox,
+  withSandboxServerRpcClient,
+} from '@roomote/sdk/server';
 import {
   and,
   cancelTaskRunDirect,
@@ -73,6 +76,14 @@ async function cancelTaskRunBeforeSandbox(runId: number): Promise<boolean> {
   const canceled = await cancelTaskRunDirect({ runId });
   if (canceled) {
     void captureTaskSettled(runId, 'canceled');
+
+    // Usually a no-op (pre-sandbox runs have no machine), but a spawn racing
+    // this cancel may already have stamped machineId without a reachable
+    // sandbox server — destroy it rather than leaking it until provider TTL.
+    await destroyCanceledTaskRunSandbox({
+      runId,
+      logPrefix: 'stopTaskRun',
+    });
   }
   return canceled;
 }

apps/bullmq/src/scheduled-jobs/sleep-check.ts

@@ -35,6 +35,7 @@ import {
   type ComputeProviderClient,
 } from '@roomote/compute-providers';
 import {
+  claimMachineDestroy,
   createSnapshot,
   finishRun,
   refreshTaskTitleOnCompletion,
@@ -1432,6 +1433,24 @@ async function destroyInstanceWithAudit(
     );
   }
 
+  // Serialize with the cancel-finalization teardown path: both destroyers
+  // record their final usage row only after the provider call returns, so the
+  // redis claim is the only atomic arbiter for a live race on this machine.
+  // The lease renews until settled below, so a slow provider delete cannot
+  // outlive it.
+  const claim = await claimMachineDestroy({
+    provider: job.vendor ?? 'docker',
+    machineId: job.machineId!,
+    owner: logPrefix,
+  });
+
+  if (claim.outcome === 'held') {
+    console.log(
+      `[${logPrefix}] Skipping destroyInstance for ${job.machineId}: another destroyer holds the teardown claim`,
+    );
+    return;
+  }
+
   const recordMutation = createComputeProviderMutationEventRecorder(
     db,
     {
@@ -1453,6 +1472,10 @@ async function destroyInstanceWithAudit(
   try {
     const result = await client.destroyInstance({ instanceId: job.machineId! });
 
+    // Success: stop renewing and let the claim expire naturally — the
+    // residual TTL keeps guarding against a duplicate delete.
+    claim.finish();
+
     await tryRecordComputeProviderUsage({
       runId: job.id,
       lifecycleAction: 'destroy',
@@ -1465,6 +1488,9 @@ async function destroyInstanceWithAudit(
       logPrefix,
     });
   } catch (error) {
+    // Give the claim back (token-conditional, so a successor that took over
+    // after a lapsed lease is unaffected) so teardown can be retried.
+    await claim.release();
     await recordMutation({
       provider: job.vendor ?? 'docker',
       operation: 'destroy_instance',

apps/controller/src/BaseController.ts

@@ -35,7 +35,7 @@ import {
   sql,
 } from '@roomote/db/server';
 import { dequeueTaskRun } from '@roomote/cloud-agents/server';
-import { finishRun } from '@roomote/sdk/server';
+import { destroyCanceledTaskRunSandbox, finishRun } from '@roomote/sdk/server';
 
 import { getOrphanedTaskRun } from './orphaned-task-runs';
 import {
@@ -399,11 +399,52 @@ export abstract class BaseController {
         sandboxTimeoutMs,
         provider,
       );
+
+      // A cancel that lands mid-provision beats the machine stamp: the run row
+      // is already terminal, so no later finalize path will tear the fresh
+      // sandbox down. Re-check once the spawn settled and destroy if so.
+      await this.destroySandboxIfCanceledDuringSpawn(taskRun.id);
     } catch (error) {
       await this.handleSpawnTaskRunError(taskRun, error);
     }
   }
 
+  private async destroySandboxIfCanceledDuringSpawn(
+    runId: number,
+  ): Promise<void> {
+    try {
+      const latestRun = await db.query.taskRuns.findFirst({
+        where: eq(taskRuns.id, runId),
+        columns: {
+          status: true,
+          canceledAt: true,
+        },
+      });
+
+      if (
+        !latestRun ||
+        (!latestRun.canceledAt && latestRun.status !== RunStatus.Canceled)
+      ) {
+        return;
+      }
+
+      console.warn(
+        `[BaseController] Task run #${runId} was canceled during spawn; destroying its sandbox`,
+      );
+
+      await destroyCanceledTaskRunSandbox({
+        runId,
+        logPrefix: 'BaseController',
+      });
+    } catch (error) {
+      console.warn(
+        `[BaseController] Failed post-spawn cancel check for task run #${runId}: ${
+          error instanceof Error ? error.message : String(error)
+        }`,
+      );
+    }
+  }
+
   private spawnWorkerInBackground(taskRun: TaskRun): boolean {
     if (this.inFlightSpawns.has(taskRun.id)) {
       console.warn(

apps/web/src/lib/task-run-errors.ts

@@ -32,6 +32,11 @@ const DOCKER_RELEASE_ARCHIVE_MISSING =
   /Docker worker release archive does not exist|Docker provider requires a local worker release archive/i;
 const DOCKER_FETCH_FAILED_IN_LOGS =
   /Job\s+<\s*unknown\s*>\s*failed:\s*fetch failed|❌[^\n]*failed:\s*fetch failed/i;
+// Roomote Cloud compute broker capacity rejection (code sandbox_limit_reached).
+const SANDBOX_LIMIT_REACHED =
+  /tenant sandbox limit was reached|sandbox_limit_reached/i;
+const SANDBOX_LIMIT_REACHED_MESSAGE =
+  "Roomote couldn't start a new sandbox because this deployment is already running its maximum number of concurrent sandboxes. Wait for an active task to finish, or stop one you no longer need, then try again.";
 
 function parseOpenAiAdminErrorBody(
   body: string,
@@ -222,6 +227,10 @@ export function getTaskRunErrorDisplayMessage(
     return workspacePreparationMessage;
   }
 
+  if (SANDBOX_LIMIT_REACHED.test(stripped)) {
+    return SANDBOX_LIMIT_REACHED_MESSAGE;
+  }
+
   // The persisted category is authoritative; text inference covers runs
   // that failed before error codes existed.
   const dockerFriendly =

apps/web/src/trpc/commands/task-runs/index.ts

@@ -31,6 +31,7 @@ import {
   tasks,
 } from '@roomote/db/server';
 import { SlackNotifier } from '@roomote/slack';
+import { destroyCanceledTaskRunSandbox } from '@roomote/sdk/server';
 
 import type { UserAuthSuccess } from '@/types';
 import { Env, getArtifactById, getRepositories } from '@/lib/server';
@@ -494,6 +495,14 @@ export async function cancelTaskRunCommand(
 
       if (canceledRun) {
         void captureTaskSettled(canceledRun.id, 'canceled');
+
+        // This cancel writes the terminal state directly (it never reaches
+        // finishRun), so tear down any attached sandbox here or it keeps
+        // running against the provider's capacity until its TTL.
+        await destroyCanceledTaskRunSandbox({
+          runId: canceledRun.id,
+          logPrefix: 'cancelTaskRunCommand',
+        });
       }
     }