Wire the Temporal engine's minikube profile so bridged agents actually run, and prove it

apps/agent-orchestrator/src/engine/temporal-engine.ts

@@ -1,6 +1,9 @@
 import { SENDER_ASSERTION_HEADER, mintSenderAssertion } from "../rbac/sender-assertion.js";
+import { CALLER_IDENTITY_HEADER, mintCallerIdentityAssertion } from "../rbac/caller-identity-assertion.js";
 import type { AgentGraphInput, AgentGraphLike } from "../server.js";
 import type { AgentState } from "../agent/graph.js";
+import type { IdentityResolver } from "../rbac/types.js";
+import { CALLER_TOOL_ID_PREFIX } from "../caller-tools/types.js";
 
 /**
  * Runs a turn on the Temporal engine (`engines/temporal`) instead of the
@@ -48,6 +51,21 @@ export interface TemporalEngineOptions {
    * sharing degrades — the same documented weaker mode as the gateway hop.
    */
   senderAssertionSecret?: string;
+  /**
+   * Resolves Open WebUI's per-request signed user JWT into a real per-user
+   * identity (the SAME resolver the LangGraph engine's `resolveIdentity` node
+   * uses for `state.forwardedUserToken`) -- needed because the gateway
+   * otherwise only ever sees ONE shared service identity for every internal
+   * hop, which would collapse every Open WebUI user onto that one subject.
+   * Deliberately NOT a general authToken-based resolver: every OTHER caller of
+   * this process's own /invoke (a webhook relay, a static-token programmatic
+   * caller) shares one token that says nothing about who is actually asking,
+   * and forwarding an identity resolved from it would override the subject a
+   * webhook turn's own senderLogin/sender-assertion channel already handles.
+   * Absent -> every chat turn on this engine resolves the gateway's own
+   * default/bearer identity, same as before this existed.
+   */
+  forwardedUserIdentityResolver?: IdentityResolver;
   /** How long to keep polling one turn before giving up. */
   timeoutMs?: number;
   /** Injectable for tests; defaults to the global fetch. */
@@ -137,6 +155,34 @@ export class TemporalEngine implements AgentGraphLike {
       headers[SENDER_ASSERTION_HEADER] = mintSenderAssertion(this.options.senderAssertionSecret, input.senderLogin);
     }
 
+    // This process's OWN per-request identity resolution (OIDC/static/
+    // forwarded-user-JWT), signed across the hop -- without this, every
+    // caller of /invoke resolves to the SAME gateway-default/bearer subject
+    // regardless of which human triggered the turn, which is exactly the
+    // collapsed-identity bug ADR 0030 fixed for webhooks. Mirrors graph.ts's
+    // resolveIdentity node: prefer the forwarded-user JWT over the shared
+    // static authToken when both are available.
+    // Scoped to the forwarded-user-JWT path ONLY, deliberately -- not a
+    // general "resolve identity somehow" fallback. `input.authToken` alone is
+    // ONE value every caller of this process's own /invoke shares (Open WebUI's
+    // shared bearer token, integration-gateway's static service token for a
+    // relayed webhook turn, etc.), so resolving it says nothing about who is
+    // actually asking and would override the correctly-defaulted gateway
+    // subject that a webhook turn's OWN senderLogin/sender-assertion channel
+    // already handles -- regressing every webhook-driven turn's credential
+    // resolution the moment this identity resolves to anything at all.
+    if (this.options.senderAssertionSecret && input.forwardedUserToken && this.options.forwardedUserIdentityResolver) {
+      const identity = await this.options.forwardedUserIdentityResolver.resolve(input.forwardedUserToken);
+      if (identity) {
+        headers[CALLER_IDENTITY_HEADER] = mintCallerIdentityAssertion(
+          this.options.senderAssertionSecret,
+          identity.subject,
+          identity.roles,
+          true,
+        );
+      }
+    }
+
     const res = await this.fetchImpl(`${this.baseUrl}/invoke`, {
       method: "POST",
       headers,
@@ -148,6 +194,31 @@ export class TemporalEngine implements AgentGraphLike {
         // still re-resolves it under the caller's own roles.
         ...(input.forcedSkillId ? { forcedSkillId: input.forcedSkillId } : {}),
         ...(input.forcedAgentId ? { forcedAgentId: input.forcedAgentId } : {}),
+        // Already resolved, validated and top-K-ranked by this process's own
+        // handleChat pipeline (ADR 0035) before invoke() is ever called --
+        // the engine's /invoke takes the resolved Descriptor shape (each
+        // ToolDescriptor's nested `callerTool`), not a raw OpenAI tools array,
+        // and re-ranks nothing. Omitting this silently drops every caller
+        // tool for any turn routed through this engine.
+        ...(input.callerTools?.length
+          ? { callerTools: input.callerTools.map((t) => t.callerTool).filter(Boolean) }
+          : {}),
+        ...(input.callerToolChoiceRequired ? { callerToolRequired: true } : {}),
+        // Prior client-executed calls (ADR 0035 resume), the same
+        // strip-the-namespace transform as everywhere else a caller-tool id
+        // crosses back out of the `caller:` namespace -- the engine's own
+        // callertools.ID re-adds it, so forwarding it prefixed would double it.
+        ...(input.actionHistory?.length
+          ? {
+              priorCallerToolCalls: input.actionHistory.map((call) => ({
+                name: call.toolId.startsWith(CALLER_TOOL_ID_PREFIX)
+                  ? call.toolId.slice(CALLER_TOOL_ID_PREFIX.length)
+                  : call.toolId,
+                arguments: call.toolArgs,
+                result: call.result,
+              })),
+            }
+          : {}),
       }),
     });
     if (!res.ok) {

apps/agent-orchestrator/src/index.ts

@@ -608,6 +608,12 @@ async function main(): Promise<void> {
       // Reuses the assertion contract rather than trusting a login on an
       // internal hop -- see TemporalEngine's own note.
       ...(config.senderAssertionSecret ? { senderAssertionSecret: config.senderAssertionSecret } : {}),
+      // Same forwarded-user resolver `graph`'s resolveIdentity node uses --
+      // without it, every CHAT turn on this engine collapses onto the
+      // gateway's shared default/bearer identity regardless of which human is
+      // chatting. Scoped to this one resolver deliberately -- see
+      // TemporalEngineOptions.forwardedUserIdentityResolver's doc comment.
+      ...(forwardedUserIdentityResolver ? { forwardedUserIdentityResolver } : {}),
       timeoutMs: config.agentRunTimeoutSeconds * 1_000,
     });
     console.log(`agent engine: temporal (${config.temporalEngineUrl})`);

apps/agent-orchestrator/src/rbac/caller-identity-assertion.ts

@@ -0,0 +1,63 @@
+import { createHmac } from "node:crypto";
+
+/**
+ * Header carrying this process's signed claim about which per-request
+ * identity it already resolved for a chat/invoke turn -- its OWN
+ * OIDC/static/forwarded-user-JWT resolution, not the Temporal engine
+ * gateway's bearer-token map.
+ *
+ * Every internal hop to that gateway otherwise authenticates with ONE shared
+ * service token (or none), so every caller resolves to the same subject
+ * regardless of which human is actually chatting -- collapsing every Open
+ * WebUI user onto one identity, the same bug class ADR 0030 fixed for
+ * webhooks via `SENDER_ASSERTION_HEADER`. This is that fix generalized: a
+ * resolved SUBJECT (and its roles) rather than a GitHub login, signed for the
+ * same reason -- an unsigned field would let anything holding the gateway's
+ * token name an arbitrary subject.
+ */
+export const CALLER_IDENTITY_HEADER = "x-gateway-caller-identity";
+
+const DEFAULT_TTL_SECONDS = 300;
+
+interface CallerIdentityPayload {
+  subject: string;
+  roles: string[];
+  /**
+   * True only when this subject came from a per-request signed identity
+   * (Open WebUI's forwarded-user JWT, or real OIDC) -- never from a shared
+   * static/bearer token, which can resolve successfully yet still be one
+   * value every caller presents. Gates whether the engine will look up a
+   * GitHub-link-based principal upgrade at all (ADR 0030 §6/0031): doing that
+   * for a shared subject would let whoever links first share their
+   * credentials with everyone else who happens to authenticate the same way.
+   */
+  perUser: boolean;
+  /** Unix seconds. */
+  exp: number;
+}
+
+function b64url(buf: Buffer): string {
+  return buf.toString("base64url");
+}
+
+function sign(secret: string, payloadB64: string): string {
+  return b64url(createHmac("sha256", secret).update(payloadB64).digest());
+}
+
+/**
+ * Mints `<payload>.<signature>` for a resolved subject/roles pair. Same HMAC
+ * scheme as `mintSenderAssertion`, deliberately a separate payload/header:
+ * this asserts a resolved caller identity, not a GitHub login.
+ */
+export function mintCallerIdentityAssertion(
+  secret: string,
+  subject: string,
+  roles: string[],
+  perUser: boolean,
+  ttlSeconds = DEFAULT_TTL_SECONDS,
+  now = Date.now(),
+): string {
+  const payload: CallerIdentityPayload = { subject, roles, perUser, exp: Math.floor(now / 1000) + ttlSeconds };
+  const payloadB64 = b64url(Buffer.from(JSON.stringify(payload), "utf8"));
+  return `${payloadB64}.${sign(secret, payloadB64)}`;
+}

charts/agent-controller/Chart.lock

@@ -7,7 +7,7 @@ dependencies:
   version: 0.1.0
 - name: temporal-engine
   repository: file://charts/temporal-engine
-  version: 0.1.1
+  version: 0.2.0
 - name: integration-gateway
   repository: file://charts/integration-gateway
   version: 0.1.0
@@ -17,5 +17,5 @@ dependencies:
 - name: nats
   repository: https://nats-io.github.io/k8s/helm/charts/
   version: 2.14.2
-digest: sha256:571058ff9cac13b57a9e36b07e9a866cca84904a0411bffde4b556f3ea1cea7b
-generated: "2026-08-06T18:08:32.066566-07:00"
+digest: sha256:5dd4996b45156a10e81616ca4293ed907d5404250dc5896c7ade4eb85014e5dd
+generated: "2026-08-14T16:05:33.799016-07:00"

charts/agent-controller/values-e2e.yaml

@@ -17,6 +17,108 @@
 # genuinely real key the stack needs (OPENAI_API_KEY, for the orchestrator's
 # planner) is created out-of-band by a human — see that script.
 
+temporal-engine:
+  # Points the worker/gateway at `e2e/manifests/temporal-dev-server.yaml`
+  # (applied by `e2e/scripts/up.sh` before this release is deployed) instead of
+  # the production default (a real `temporalio/helm-charts` release's frontend
+  # Service, `temporal-frontend.temporal.svc:7233`) -- there is no such cluster
+  # here, and the chart's own default address is unreachable in a hermetic
+  # minikube profile.
+  temporal:
+    address: temporal-dev-server.controller-agent.svc:7233
+    namespace: default
+  llm:
+    # The subchart's own default (`agent-controller-agent-orchestrator`)
+    # assumes the `agent-orchestrator` subchart at its OWN release-name-derived
+    # secret name, but that subchart sets `fullnameOverride: agent-orchestrator`
+    # (`charts/agent-controller/values.yaml`), so its real Secret is
+    # `agent-orchestrator-secrets` -- the default name doesn't exist in ANY
+    # profile, e2e or otherwise, and the worker/gateway Pods CreateContainer-
+    # ConfigError on first boot without this override.
+    secretName: agent-orchestrator-secrets
+  callback:
+    # Same `fullnameOverride` gap as `llm.secretName` above, for the gateway's
+    # HMAC callback secret -- `agent-orchestrator-secrets` already carries
+    # `AGENT_CALLBACK_SECRET` (matching `callback.secretKey`'s default), since
+    # that's the same value agent-orchestrator's own tool-callback path signs
+    # with.
+    secretName: agent-orchestrator-secrets
+  nats:
+    # Empty by default -- `charts/agent-controller/charts/temporal-engine`'s own
+    # comment says this "disables bridged pod agents entirely, leaving the
+    # declarative loop and checkpoint-resume step tools", and the worker logs
+    # it plainly at startup ("AGENT_NATS_URL not set; bridged pod agents
+    # disabled"). This is the exact NATS the orchestrator itself is wired to
+    # (`agent-orchestrator.natsUrl` in values-minikube-demo.yaml derives the
+    # same Service) -- both engines have to speak to the same broker, since a
+    # bridged agent's Job publishes to AgentRun-id-keyed subjects regardless of
+    # which engine is driving the episode.
+    url: nats://agent-controller-nats.controller-agent.svc.cluster.local:4222
+  qdrant:
+    # Empty by default -- and every catalog-dependent activity (IntegrationRoute
+    # matching, Skill/Agent/Tool retrieval, delegation) needs it: without a
+    # Qdrant host, the worker logs "QDRANT_HOST not set; retrieval activities
+    # disabled" and every turn falls through the capability-need gate into a
+    # bare conversational answer, NEVER reaching delegateToAgent -- so no
+    # Agent, bridged or otherwise, is ever run. Reuses agent-orchestrator's own
+    # Qdrant instance (already deployed by this same release) rather than
+    # standing up a second one; `collectionPrefix` keeps its collections
+    # separate from the orchestrator's own (their payload schemas differ).
+    host: agent-controller-qdrant
+    port: 6334
+    collectionPrefix: te-
+  gateway:
+    senderAssertion:
+      # Without this, `GATEWAY_SENDER_ASSERTION_SECRET` is unset on the
+      # Temporal engine gateway, so it can never verify the signed assertion
+      # header agent-orchestrator's TemporalEngine sends (temporal-engine.ts:
+      # "the sender login travels as a SIGNED assertion, not a body field" --
+      # it never falls back to an unsigned body field for this hop). The
+      # gateway silently resolves an empty sender login instead, which is why
+      # `AGENT_ACTOR_LOGIN` never appears in a launched AgentRun's secretEnv
+      # for any webhook-driven turn. Same secret/value integration-gateway and
+      # agent-orchestrator already share for this exact purpose.
+      secretName: e2e-integration-gateway-secrets
+    identity:
+      # The Go gateway resolves ITS OWN caller identity from a bearer token --
+      # entirely separate from agent-orchestrator's own `config.staticIdentities`
+      # below, despite the identical JSON shape. But agent-orchestrator's
+      # internal hop to this gateway (`TemporalEngine`, apps/agent-orchestrator/
+      # src/engine/temporal-engine.ts) sends NO Authorization header at all
+      # unless `AGENT_TEMPORAL_ENGINE_TOKEN` is set, which nothing in this
+      # chart ever populates (its own secretKeyRef is `optional: true` against
+      # a key that doesn't exist). So EVERY call arrives tokenless, resolving
+      # to an EMPTY Caller.Subject -- and `runAgentTurn`
+      # (engines/temporal/internal/temporal/workflows/agentloop.go) treats an
+      # empty subject as "answer bare, skip the catalog entirely",
+      # unconditionally, regardless of the capability gate, retrieval, or a
+      # route's `forcedAgentId`. Without this, NOTHING ever delegates to any
+      # Agent under the Temporal engine -- not a bridged-agent-specific gap,
+      # a total one. `defaultSubject`/`defaultRoles` are exactly the
+      # documented lever for a tokenless caller (main.go: "give tokenless
+      # callers an identity — leave unset to fail closed to no
+      # capabilities"), matching the role agent-orchestrator's own shared
+      # `client-integration-gateway` subject plays for every webhook-driven
+      # turn today.
+      defaultSubject: client-integration-gateway
+      defaultRoles: reader,writer
+  identityLink:
+    # Without this the worker logs "IDENTITY_LINK_GATEWAY_URL not set; using
+    # the in-memory identity-link fake (dev only)" -- a completely separate,
+    # unseedable identity-link store from the one this e2e suite actually
+    # seeds into (e2e/support/credential-store.ts writes real Kubernetes
+    # Secrets that integration-gateway's identity-link API reads). Every
+    # Agent requiring `identityProviders` therefore parks on "link-required"
+    # forever under the fake, regardless of anything seeded. Mirrors
+    # agent-orchestrator's own `identityLink.gatewayUrl` above at the
+    # identical Service address, and reuses the same shared secret VALUE
+    # bootstrap-secrets.sh already wrote for the reverse direction (its own
+    # key name differs -- IDENTITY_LINK_GATEWAY_TOKEN vs
+    # GATEWAY_IDENTITY_LINK_TOKEN -- but both hold the one value both sides
+    # of this handshake must agree on).
+    gatewayUrl: "http://agent-controller-integration-gateway:8090"
+    tokenSecretName: e2e-integration-gateway-secrets
+
 integration-gateway:
   # The whole point of this overlay. Off in every other profile.
   enabled: true

controllers/core-controller/internal/controller/agentrun_controller.go

@@ -150,7 +150,16 @@ func (r *AgentRunReconciler) createJob(ctx context.Context, run *toolv1alpha1.Ag
 	}
 
 	job, err := buildRunJob(runJobParams{
-		jobName:     fmt.Sprintf("agentrun-%s", run.Name),
+		// run.Name IS the Job name directly, not "agentrun-"+run.Name: the
+		// Temporal engine (engines/temporal's BridgedAgentWorkflow) names
+		// AgentRun CRs "agentrun-<agentId>-<uuid>" itself, so prepending this
+		// prefix again produced a Job name (and therefore the API server's
+		// auto-added spec.template.labels["job-name"], which reuses the Job's
+		// own name verbatim with no truncation) exceeding the 63-byte label
+		// limit -- the exact failure mode this fixes. Job lookup is by owner
+		// reference (`.Owns(&batchv1.Job{})` below), never by reconstructing
+		// this name, so dropping the prefix changes nothing else.
+		jobName:     run.Name,
 		namespace:   run.Namespace,
 		annotations: sessionIDAnnotations(run.Annotations),
 		labels: map[string]string{