feat(run-engine): trigger tasks pinned to an external deployment id (#4664)

The SDK discovers an external deployment id at runtime (explicit
TRIGGER_EXTERNAL_DEPLOYMENT_ID always; platform commit-SHA variables and
generic fallbacks when TRIGGER_AUTOMATIC_SKEW_VERSION_PROTECTION=1) and
sends it alongside lockToVersion; the server resolves precedence
(version > external id > current). An id held by a deployed deployment
pins the run to that worker; an in-flight or unknown id parks the run in
PENDING_VERSION with the id in TaskRun.annotations, wakes it pinned when
a deployment carrying the id finalizes (ClickHouse candidates, Postgres
authoritative), and expires it after a deadline that re-checks Postgres
before acting. Parking outranks delaying and preserves delayUntil. The
id is projected to ClickHouse task_runs_v2.external_deployment_id during
replication. Redis cache for id-to-worker resolution, guarded
version-aware writes.

Ids are not unique. Several deployments can hold one id - a --force
rebuild is the ordinary way to get there - so resolution always picks
the highest version among the candidates, never the newest by timestamp.
The rule is applied identically on both paths that can bind a run to a
worker: resolveExternalDeployment at trigger time, and
PendingVersionSystem when a landing deployment wakes a parked run.
Version comparison is numeric on the counter half, so 20260807.10
outranks 20260807.9.

A run whose id never lands expires at the deadline with
EXTERNAL_DEPLOYMENT_NOT_FOUND and an error naming the id it waited for,
which is what a failed build or a typo looks like from the caller.
Default deadline is one hour (EXTERNAL_DEPLOYMENT_PARK_DEADLINE_MS).

Debounce registration happens in both the parked and the delayed branch
through one helper, so a debounced run that parks still binds its
debounce key; without it every later trigger for the same key created
another parked run, and all of them executed when the deployment landed.
The two DELAYED-only status checks in DebounceSystem also accept
PENDING_VERSION, without which the lock-contention fallback would
rethrow a 5xx the SDK retries and amplifies, and the fast path would
push every trigger on a parked key through the redlock.

Resolution is skipped in development. A dev environment cannot hold a
WorkerDeployment - trigger dev registers a BackgroundWorker with nothing
behind it, and deploy --env refuses dev - so an external deployment id
there could only ever park, and the parked run then expired against the
dev TTL while a connected dev worker sat idle. The id is still annotated
so the dashboard shows what the app sent (TRI-13000).

Author: 0ski

.changeset/trigger-external-deployment-id.md

@@ -0,0 +1,6 @@
+---
+"@trigger.dev/core": patch
+"@trigger.dev/sdk": patch
+---
+
+Pin runs to the deployment your calling code came from, so an old release never triggers tasks from a new one: set `TRIGGER_EXTERNAL_DEPLOYMENT_ID` to the id you deployed with, or `TRIGGER_AUTOMATIC_SKEW_VERSION_PROTECTION=1` to detect the commit automatically on Vercel and most CI systems. Runs triggered before that deployment finishes building wait for it, then start pinned.

apps/webapp/app/env.server.ts

@@ -488,6 +488,31 @@ const EnvironmentSchema = z
       .default(process.env.REDIS_TLS_DISABLED ?? "false"),
     TASK_META_CACHE_CURRENT_ENV_TTL_SECONDS: z.coerce.number().default(86400),
 
+    EXTERNAL_DEPLOYMENT_CACHE_REDIS_HOST: z
+      .string()
+      .optional()
+      .transform((v) => v ?? process.env.REDIS_HOST),
+    EXTERNAL_DEPLOYMENT_CACHE_REDIS_PORT: z.coerce
+      .number()
+      .optional()
+      .transform(
+        (v) => v ?? (process.env.REDIS_PORT ? parseInt(process.env.REDIS_PORT) : undefined)
+      ),
+    EXTERNAL_DEPLOYMENT_CACHE_REDIS_USERNAME: z
+      .string()
+      .optional()
+      .transform((v) => v ?? process.env.REDIS_USERNAME),
+    EXTERNAL_DEPLOYMENT_CACHE_REDIS_PASSWORD: z
+      .string()
+      .optional()
+      .transform((v) => v ?? process.env.REDIS_PASSWORD),
+    EXTERNAL_DEPLOYMENT_CACHE_REDIS_TLS_DISABLED: z
+      .string()
+      .default(process.env.REDIS_TLS_DISABLED ?? "false"),
+    EXTERNAL_DEPLOYMENT_CACHE_TTL_SECONDS: z.coerce.number().default(2592000),
+    EXTERNAL_DEPLOYMENT_CACHE_MISSING_TTL_SECONDS: z.coerce.number().default(20),
+    EXTERNAL_DEPLOYMENT_PARK_DEADLINE_MS: z.coerce.number().default(3600000),
+
     // Runs-list empty-state check: how far back the ClickHouse "does this env have any run"
     // probe looks. Bounds the prove-absence partition scan. 0 = unbounded ("any run ever").
     RUN_LIST_HAS_RUNS_LOOKBACK_DAYS: z.coerce.number().default(30),

apps/webapp/app/runEngine/services/triggerTask.server.ts

@@ -74,6 +74,9 @@ import {
 import { mollifyTrigger } from "~/v3/mollifier/mollifierMollify.server";
 import { QueueSizeLimitExceededError, ServiceValidationError } from "~/v3/services/common.server";
 import { runStore } from "~/v3/runStore.server";
+import type { ExternalDeploymentCache } from "~/services/externalDeploymentCache.server";
+import { externalDeploymentCacheInstance } from "~/services/externalDeploymentCacheInstance.server";
+import { resolveExternalDeployment } from "~/v3/services/resolveExternalDeployment.server";
 
 class NoopTriggerRacepointSystem implements TriggerRacepointSystem {
   async waitForRacepoint(options: { racepoint: TriggerRacepoints; id: string }): Promise<void> {
@@ -101,6 +104,7 @@ export class RunEngineTriggerTaskService {
   private readonly evaluateGate: MollifierEvaluateGate;
   private readonly getMollifierBuffer: MollifierGetBuffer;
   private readonly isMollifierGloballyEnabled: () => boolean;
+  private readonly externalDeploymentCache: ExternalDeploymentCache;
 
   constructor(opts: {
     prisma: PrismaClientOrTransaction;
@@ -117,6 +121,7 @@ export class RunEngineTriggerTaskService {
     evaluateGate?: MollifierEvaluateGate;
     getMollifierBuffer?: MollifierGetBuffer;
     isMollifierGloballyEnabled?: () => boolean;
+    externalDeploymentCache?: ExternalDeploymentCache;
   }) {
     this.prisma = opts.prisma;
     this.engine = opts.engine;
@@ -134,6 +139,7 @@ export class RunEngineTriggerTaskService {
     this.getMollifierBuffer = opts.getMollifierBuffer ?? defaultGetMollifierBuffer;
     this.isMollifierGloballyEnabled =
       opts.isMollifierGloballyEnabled ?? (() => env.TRIGGER_MOLLIFIER_ENABLED === "1");
+    this.externalDeploymentCache = opts.externalDeploymentCache ?? externalDeploymentCacheInstance;
   }
 
   /**
@@ -394,7 +400,7 @@ export class RunEngineTriggerTaskService {
             });
           }
 
-          const lockedToBackgroundWorker = body.options?.lockToVersion
+          const explicitlyLockedToBackgroundWorker = body.options?.lockToVersion
             ? await this.prisma.backgroundWorker.findFirst({
                 where: {
                   projectId: environment.projectId,
@@ -410,6 +416,34 @@ export class RunEngineTriggerTaskService {
               })
             : undefined;
 
+          const externalDeploymentId = body.options?.lockToVersion
+            ? undefined
+            : body.options?.externalDeploymentId;
+
+          const externalDeploymentResolution =
+            externalDeploymentId && environment.type !== "DEVELOPMENT"
+              ? await resolveExternalDeployment({
+                  prisma: this.prisma,
+                  environmentId: environment.id,
+                  externalDeploymentId,
+                  cache: this.externalDeploymentCache,
+                })
+              : undefined;
+
+          const lockedToBackgroundWorker =
+            explicitlyLockedToBackgroundWorker ??
+            (externalDeploymentResolution?.outcome === "deployed"
+              ? {
+                  id: externalDeploymentResolution.worker.workerId,
+                  version: externalDeploymentResolution.worker.version,
+                  sdkVersion: externalDeploymentResolution.worker.sdkVersion,
+                  cliVersion: externalDeploymentResolution.worker.cliVersion,
+                }
+              : undefined);
+
+          const parkedOnExternalDeploymentId =
+            externalDeploymentResolution?.outcome === "park" ? externalDeploymentId : undefined;
+
           const { queueName, lockedQueueId, taskTtl, taskKind } =
             await this.queueConcern.resolveQueueProperties(
               triggerRequest,
@@ -503,6 +537,7 @@ export class RunEngineTriggerTaskService {
             rootTriggerSource: parentAnnotations?.rootTriggerSource ?? triggerSource,
             rootScheduleId: parentAnnotations?.rootScheduleId || options.scheduleId || undefined,
             taskKind: taskKind ?? "STANDARD",
+            externalDeploymentId,
           };
 
           // Route runs in a scheduled lineage (the scheduled run itself and every
@@ -638,6 +673,7 @@ export class RunEngineTriggerTaskService {
                       depth,
                       parentRun: parentRun ?? undefined,
                       annotations,
+                      parkedOnExternalDeploymentId,
                       planType,
                       taskId,
                       payloadPacket,
@@ -717,6 +753,7 @@ export class RunEngineTriggerTaskService {
                   depth,
                   parentRun: parentRun ?? undefined,
                   annotations,
+                  parkedOnExternalDeploymentId,
                   planType,
                   taskId,
                   payloadPacket,
@@ -892,7 +929,9 @@ export class RunEngineTriggerTaskService {
       triggerAction: string;
       rootTriggerSource: string;
       rootScheduleId?: string | undefined;
+      externalDeploymentId?: string | undefined;
     };
+    parkedOnExternalDeploymentId?: string;
     planType?: string;
     taskId: string;
     payloadPacket: { data?: string; dataType: string };
@@ -974,6 +1013,7 @@ export class RunEngineTriggerTaskService {
       streamBasinName: args.environment.organization.streamBasinName,
       debounce: removeNullBytesFromKey(args.body.options?.debounce),
       annotations: args.annotations,
+      parkedOnExternalDeploymentId: args.parkedOnExternalDeploymentId,
     };
   }
 

apps/webapp/app/services/externalDeploymentCache.server.ts

@@ -0,0 +1,214 @@
+import type { Callback, Redis, Result } from "ioredis";
+import { logger } from "./logger.server";
+
+export type ExternalDeploymentCacheEntry = {
+  workerId: string;
+  version: string;
+  sdkVersion: string;
+  cliVersion: string;
+};
+
+export type ExternalDeploymentCacheResult =
+  | { outcome: "deployed"; entry: ExternalDeploymentCacheEntry }
+  | { outcome: "missing" };
+
+export interface ExternalDeploymentCache {
+  get(environmentId: string, externalId: string): Promise<ExternalDeploymentCacheResult | null>;
+  setIfNewer(
+    environmentId: string,
+    externalId: string,
+    entry: ExternalDeploymentCacheEntry
+  ): Promise<void>;
+  setMissing(environmentId: string, externalId: string): Promise<void>;
+}
+
+const KEY_PREFIX = "skewid:";
+
+const DEFAULT_TTL_SECONDS = 30 * 24 * 60 * 60;
+
+const DEFAULT_MISSING_TTL_SECONDS = 20;
+
+const MISSING_ENTRY = JSON.stringify({ m: 1 });
+
+function buildKey(environmentId: string, externalId: string): string {
+  return `${KEY_PREFIX}${environmentId}:${externalId}`;
+}
+
+type CachedEntry = {
+  w: string;
+  v: string;
+  s: string;
+  c: string;
+};
+
+function encode(entry: ExternalDeploymentCacheEntry): string {
+  return JSON.stringify({
+    w: entry.workerId,
+    v: entry.version,
+    s: entry.sdkVersion,
+    c: entry.cliVersion,
+  } satisfies CachedEntry);
+}
+
+function decode(raw: string): ExternalDeploymentCacheResult | null {
+  const parsed: unknown = JSON.parse(raw);
+
+  if (typeof parsed !== "object" || parsed === null) {
+    return null;
+  }
+
+  const { w, v, s, c, m } = parsed as Partial<CachedEntry> & { m?: unknown };
+
+  if (m === 1) {
+    return { outcome: "missing" };
+  }
+
+  if (typeof w !== "string" || typeof v !== "string") {
+    return null;
+  }
+
+  return {
+    outcome: "deployed",
+    entry: {
+      workerId: w,
+      version: v,
+      sdkVersion: typeof s === "string" ? s : "",
+      cliVersion: typeof c === "string" ? c : "",
+    },
+  };
+}
+
+const SET_IF_NEWER_LUA = `
+local existing = redis.call("GET", KEYS[1])
+
+if existing then
+  local ok, decoded = pcall(cjson.decode, existing)
+  if ok and type(decoded) == "table" and type(decoded.v) == "string" then
+    local existingDate, existingCounter = string.match(decoded.v, "^([^.]*)%.?(.*)$")
+    local incomingDate, incomingCounter = string.match(ARGV[2], "^([^.]*)%.?(.*)$")
+
+    if existingDate > incomingDate then
+      return 0
+    end
+
+    if existingDate == incomingDate then
+      if (tonumber(existingCounter) or 0) >= (tonumber(incomingCounter) or 0) then
+        return 0
+      end
+    end
+  end
+end
+
+redis.call("SET", KEYS[1], ARGV[1], "EX", tonumber(ARGV[3]))
+return 1
+`;
+
+declare module "ioredis" {
+  interface RedisCommander<Context> {
+    skewIdSetIfNewer(
+      key: string,
+      entry: string,
+      version: string,
+      ttlSeconds: string,
+      callback?: Callback<number>
+    ): Result<number, Context>;
+  }
+}
+
+export type RedisExternalDeploymentCacheOptions = {
+  redis: Redis;
+  ttlSeconds?: number;
+  missingTtlSeconds?: number;
+};
+
+export class RedisExternalDeploymentCache implements ExternalDeploymentCache {
+  private readonly redis: Redis;
+  private readonly ttlSeconds: number;
+  private readonly missingTtlSeconds: number;
+
+  constructor(options: RedisExternalDeploymentCacheOptions) {
+    this.redis = options.redis;
+    this.ttlSeconds = options.ttlSeconds ?? DEFAULT_TTL_SECONDS;
+    this.missingTtlSeconds = options.missingTtlSeconds ?? DEFAULT_MISSING_TTL_SECONDS;
+
+    this.redis.defineCommand("skewIdSetIfNewer", { numberOfKeys: 1, lua: SET_IF_NEWER_LUA });
+  }
+
+  async get(
+    environmentId: string,
+    externalId: string
+  ): Promise<ExternalDeploymentCacheResult | null> {
+    try {
+      const raw = await this.redis.get(buildKey(environmentId, externalId));
+      if (!raw) return null;
+      return decode(raw);
+    } catch (error) {
+      logger.error("Failed to read external deployment resolution from cache", {
+        environmentId,
+        externalId,
+        error,
+      });
+      return null;
+    }
+  }
+
+  async setIfNewer(
+    environmentId: string,
+    externalId: string,
+    entry: ExternalDeploymentCacheEntry
+  ): Promise<void> {
+    try {
+      await this.redis.skewIdSetIfNewer(
+        buildKey(environmentId, externalId),
+        encode(entry),
+        entry.version,
+        String(this.ttlSeconds)
+      );
+    } catch (error) {
+      logger.error("Failed to write external deployment resolution to cache", {
+        environmentId,
+        externalId,
+        version: entry.version,
+        error,
+      });
+
+      try {
+        await this.redis.del(buildKey(environmentId, externalId));
+      } catch (deleteError) {
+        logger.error("Failed to evict stale external deployment resolution after write failure", {
+          environmentId,
+          externalId,
+          error: deleteError,
+        });
+      }
+    }
+  }
+
+  async setMissing(environmentId: string, externalId: string): Promise<void> {
+    try {
+      await this.redis.set(
+        buildKey(environmentId, externalId),
+        MISSING_ENTRY,
+        "EX",
+        this.missingTtlSeconds,
+        "NX"
+      );
+    } catch (error) {
+      logger.error("Failed to write missing external deployment marker to cache", {
+        environmentId,
+        externalId,
+        error,
+      });
+    }
+  }
+}
+
+export class NoopExternalDeploymentCache implements ExternalDeploymentCache {
+  async get(): Promise<ExternalDeploymentCacheResult | null> {
+    return null;
+  }
+
+  async setIfNewer(): Promise<void> {}
+
+  async setMissing(): Promise<void> {}
+}