Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions scripts/test-layout/layout.json
Original file line number Diff line number Diff line change
Expand Up @@ -1342,6 +1342,7 @@
"self-launch-argv.test.ts": "lib",
"server-403-permission-e2e.test.ts": "server",
"server-agent-task-recovery-replay.test.ts": "server",
"server-auth-scoped-quota.test.ts": "server",
"server-auth.test.ts": "server",
"server-background-lifecycle.test.ts": "server",
"server-clickjacking-headers.test.ts": "server",
Expand Down
14 changes: 14 additions & 0 deletions src/codex/auth-context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -598,6 +598,20 @@ function selectedCodexToken(headers: Headers): { accessToken: string; chatgptAcc
};
}

/**
* The workspace account id a request-owned `main` credential materializes under, or
* `undefined` when the caller's headers carry none. This is the `chatgpt-account-id`
* `materializeCodexUpstreamAuth` would set for a caller-owned `{ kind: "main" }` context,
* read here without touching a credential store so a rotation gate can compare workspace
* scope before a send is ever built.
*/
export function callerCodexWorkspaceAccountId(headers: Headers): string | undefined {
const explicit = headers.get("chatgpt-account-id");
if (explicit) return explicit;
const bearer = headers.get("authorization")?.replace(/^Bearer\s+/i, "").trim();
return bearer ? extractAccountId(undefined, bearer) : undefined;
}

function assertMaterializedReserve(headers: Headers, ctx: CodexAuthContext, options: CodexAuthMaterializationOptions): void {
if (!requiresReserveAuthorization(options.config, options.modelId, options.admission)) return;
assertReserveAdmission(options.config!);
Expand Down
12 changes: 8 additions & 4 deletions src/codex/quota-rejection.ts
Original file line number Diff line number Diff line change
Expand Up @@ -349,9 +349,10 @@ export async function codexScopedExhaustionCode(
* Status alone and message text are intentionally insufficient. The broad
* alternate-account retry remains eligible for 429/402 to preserve #584.
*
* The one carve-out from that breadth is an organization- or project-scoped exhaustion
* ({@link SCOPED_EXHAUSTION_CODE_VALUES}), which reports `alternateRetryEligible: false`
* because every credential inside the refusing limit would be refused by the same counter.
* Organization- or project-scoped exhaustion ({@link SCOPED_EXHAUSTION_CODE_VALUES}) remains
* alternate-retry eligible here because the response does not identify the refusing scope. The
* account-rotation path may suppress the send later when the resolved alternate carries binding
* evidence that it shares an organization-level counter.
*/
export async function classifyCodexPreStreamRejection(
response: Response,
Expand All @@ -377,7 +378,10 @@ export async function classifyCodexPreStreamRejection(
});
}
if (scoped) {
return rejection(status, "scoped-quota-exhaustion", { scopedExhaustionCode: scoped });
return rejection(status, "scoped-quota-exhaustion", {
alternateRetryEligible: true,
scopedExhaustionCode: scoped,
});
}
return rejection(
status,
Expand Down
31 changes: 30 additions & 1 deletion src/server/responses/compact.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ import { describeImagesInPlace, planVisionSidecar, shouldResolveOpenAiVisionSide
import { createAdapterEventQueue, preflightAdapterEvents } from "../../adapters/run-turn-queue";
import {
applyCodexAuthContextToProvider,
callerCodexWorkspaceAccountId,
createCodexReserveDispatchGuard,
unwrapUpstreamRetryEvidenceError,
CodexMainProfileDrainingError,
Expand Down Expand Up @@ -186,6 +187,7 @@ import {
handleResponses,
preAuthUpstreamHostCircuitKey,
poolCredentialRefreshIncompleteResponse,
shouldRetryCodexScopedQuotaOnAlternate,
upstreamHostCircuitOpenResponse,
usesCodexForwardPoolAuth,
} from "./core";
Expand Down Expand Up @@ -1211,9 +1213,36 @@ export async function handleResponsesCompact(
if (alternate && req.signal.aborted) {
releaseCodexAuthContextProbeLease(alternate.authCtx);
recordCompactPoolOutcome(outcomeCtx, 499);
void upstream.body?.cancel(req.signal.reason).catch(() => undefined);
return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request");
}
if (alternate) {
// The same scope binding the regular path applies: an organization-scoped
// exhaustion refuses every credential in that workspace, so a proven
// same-workspace alternate pays a cold prompt prefix for no new capacity.
// Suppression is not silence — the buffered recorder below still attributes
// the 429/402 to the account that produced it.
const sharedWorkspaceScope = alternate != null
&& !await shouldRetryCodexScopedQuotaOnAlternate(
upstream,
authCtx.chatgptAccountId,
alternate.authCtx.kind === "pool" || alternate.authCtx.kind === "main-pool"
? alternate.authCtx.chatgptAccountId
: callerCodexWorkspaceAccountId(req.headers),
req.signal,
);
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
// The scope check reads the rejection body asynchronously — the same window the
// comment above covers. Re-check before the branch below records A, cancels its
// body, and sends B for a caller that is gone.
if (alternate && req.signal.aborted) {
releaseCodexAuthContextProbeLease(alternate.authCtx);
recordCompactPoolOutcome(outcomeCtx, 499);
void upstream.body?.cancel(req.signal.reason).catch(() => undefined);
return formatErrorResponse(499, "client_cancelled", "Client cancelled compact request");
}
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
if (alternate && sharedWorkspaceScope) {
releaseCodexAuthContextProbeLease(alternate.authCtx);
}
if (alternate && !sharedWorkspaceScope) {
// Same order the regular path uses (core.ts:349-357): a 429/402 carries the
// quota snapshot that produced it, so refresh A's cache before recording its
// rejection. Skipping this leaves quota-strategy routing and the dashboard
Expand Down
92 changes: 69 additions & 23 deletions src/server/responses/core-codex-account.ts
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ import { isNativeMainTrafficBlocked } from "../../codex/native-profile-startup";
import { MAIN_CODEX_ACCOUNT_ID } from "../../codex/main-account";
import { slugsEquivalent } from "../../providers/slug-codec";
import {
callerCodexWorkspaceAccountId,
codexProbeLeaseId,
codexTransientProbeGrant,
codexProbeQuotaScope,
Expand Down Expand Up @@ -276,15 +277,11 @@ export async function shouldRetryCodexPoolAccountQuota(
// body carries no quota evidence either, but the marker is the contract, not the prose.
if (isNonReplayableResponse(response)) return false;
if (response.status === 402 || response.status === 429) {
// Status alone used to authorize the move, which is right for a limit the ACCOUNT owns and
// wrong for one it merely belongs to. An organization- or project-scoped exhaustion refuses
// every credential inside that organization, so the second account meets the same counter
// and the only thing the rotation buys is a second cold prompt prefix (#4546). Positive
// evidence is required to withhold it: the helper fails closed, so an unreadable or
// ambiguous body keeps the broad #584 behaviour unchanged, and `rate_limit_exceeded`,
// `slow_down` and plan-level exhaustion still rotate exactly as before.
const { codexScopedExhaustionCode } = await import("../../codex/quota-rejection");
return await codexScopedExhaustionCode(response, { signal }) === undefined;
// The response does not identify the organization or project whose quota was exhausted.
// Resolve the alternate before deciding whether its known workspace identity proves that an
// organization-scoped retry would be futile. Until then, preserve the broad #584 behaviour.
void signal;
return true;
}
if (response.status < 500 || response.status >= 600) return false;
try {
Expand All @@ -301,6 +298,20 @@ export async function shouldRetryCodexPoolAccountQuota(
}


export async function shouldRetryCodexScopedQuotaOnAlternate(
response: Response,
firstWorkspaceAccountId: string,
alternateWorkspaceAccountId: string | undefined,
signal?: AbortSignal,
): Promise<boolean> {
if (!firstWorkspaceAccountId || firstWorkspaceAccountId !== alternateWorkspaceAccountId) return true;
const { codexScopedExhaustionCode } = await import("../../codex/quota-rejection");
const code = await codexScopedExhaustionCode(response, { signal });
// Workspace identity binds organization-level limits, but the response supplies no project id.
return code === undefined || code === "project_spend_limit_exceeded";
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
}


/**
* A pre-stream upstream 5xx another Codex account may still be able to serve.
*
Expand Down Expand Up @@ -528,6 +539,22 @@ export async function retryCodexPoolOnAlternateAccount(
writerGeneration: firstAuthCtx.writerGeneration,
});
};
// A body-confirmed quota response may arrive under HTTP 5xx. A path that returns the
// first response without a move must still record the NORMALIZED outcome: the ordinary
// terminal recorder sees only that wire status and would misclassify it as transient,
// leaving the exhausted account immediately selectable next turn.
const recordWrappedQuotaOutcome = (): void => {
if (outcomeStatus === firstResponse.status || (outcomeStatus !== 429 && outcomeStatus !== 402)) return;
recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, {
...codexQuotaOutcomeMeta(firstResponse),
threadId: firstAuthCtx.affinityKey,
modelId: route.modelId,
probeLeaseId: codexProbeLeaseId(firstAuthCtx),
probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
transientProbe: codexTransientProbeGrant(firstAuthCtx),
writerGeneration: firstAuthCtx.writerGeneration,
});
};
if (outcomeStatus === 400 && ACCOUNT_GATED_NATIVE_OPENAI_MODELS.has(route.modelId)) {
invalidateCodexModelEntitlementsForAccount(firstAuthCtx.accountId);
let refreshed;
Expand Down Expand Up @@ -624,26 +651,45 @@ export async function retryCodexPoolOnAlternateAccount(
&& retryAuthCtx?.kind !== "main-pool"
&& retryAuthCtx?.kind !== "main"
) {
// A body-confirmed quota response may arrive under HTTP 5xx. Without an alternate,
// the ordinary terminal recorder sees only that wire status and would misclassify it
// as transient, leaving the exhausted account immediately selectable next turn.
if (outcomeStatus !== firstResponse.status && (outcomeStatus === 429 || outcomeStatus === 402)) {
recordCodexUpstreamOutcome(config, firstAuthCtx.accountId, outcomeStatus, {
...codexQuotaOutcomeMeta(firstResponse),
threadId: firstAuthCtx.affinityKey,
modelId: route.modelId,
probeLeaseId: codexProbeLeaseId(firstAuthCtx),
probeQuotaScope: codexProbeQuotaScope(firstAuthCtx),
transientProbe: codexTransientProbeGrant(firstAuthCtx),
writerGeneration: firstAuthCtx.writerGeneration,
});
}
recordWrappedQuotaOutcome();
// No usable alternate was resolved, so the reserved move never becomes a send.
accountMovePermit?.release();
recordUnmovedTransientOutcome();
return { kind: "no-alternate" };
}

if (
(outcomeStatus === 429 || outcomeStatus === 402)
&& !await shouldRetryCodexScopedQuotaOnAlternate(
firstResponse,
firstAuthCtx.chatgptAccountId,
retryAuthCtx.kind === "pool" || retryAuthCtx.kind === "main-pool"
? retryAuthCtx.chatgptAccountId
// A request-owned `main` alternate has no stored account id; its workspace
// identity is what the caller's own credential materializes upstream.
: callerCodexWorkspaceAccountId(callerAuthHeaders),
options.abortSignal,
)
) {
// Suppressing the move is not suppressing the evidence: a same-workspace refusal
// still records its normalized quota outcome on the account that produced it.
recordWrappedQuotaOutcome();
accountMovePermit?.release();
releaseCodexAuthContextProbeLease(retryAuthCtx);
return { kind: "no-alternate" };
}

// The scope classification above reads the rejection body asynchronously, so the
// request may have been cancelled while it ran. Re-check before the send below
// mutates routing state or spends the alternate on a caller that is gone.
if (options.abortSignal?.aborted) {
recordWrappedQuotaOutcome();
recordUnmovedTransientOutcome();
accountMovePermit?.release();
releaseCodexAuthContextProbeLease(retryAuthCtx);
return { kind: "no-alternate" };
Comment thread
devin-ai-integration[bot] marked this conversation as resolved.
}

const quotaMeta = { ...codexQuotaOutcomeMeta(firstResponse), ...(await codexDenialOutcomeMeta(firstResponse)) };
if (outcomeStatus === 429 || outcomeStatus === 402) {
const { applyAccountQuotaFromUpstreamHeaders } = await import("../../codex/auth-api");
Expand Down
2 changes: 1 addition & 1 deletion src/server/responses/core.ts
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ export { readDisplaySafeErrorText } from "./core-errors";
export { usesCodexForwardPoolAuth } from "./core-codex-account";
export { preAuthUpstreamHostCircuitKey } from "./core-codex-account";
export { upstreamHostCircuitOpenResponse } from "./core-codex-account";
export { shouldRetryCodexPoolAccountQuota } from "./core-codex-account";
export { shouldRetryCodexPoolAccountQuota, shouldRetryCodexScopedQuotaOnAlternate } from "./core-codex-account";
export { shouldRetryCodexPoolAccountTransient } from "./core-codex-account";
export { codexAccountGatedCanonicalWireModel } from "./core-codex-account";
export { codexForwardTerminalOutcomeRecorder } from "./core-codex-account";
Expand Down
5 changes: 3 additions & 2 deletions structure/providers/openai-tiers.md
Original file line number Diff line number Diff line change
Expand Up @@ -149,11 +149,12 @@ and credential/transport failures retain their ordinary handling.
`credit_balance_exhausted`, `organization_spend_limit_exceeded`, `project_spend_limit_exceeded` and
`organization_usage_limit_exceeded` name a balance or cap held by the organization or project, so
`classifyCodexPreStreamRejection` reports `scoped-quota-exhaustion` with `alternateRetryEligible`
false and `scopedExhaustionCode` set, and never `resetCreditEligible` — a reset credit reconciles a
true and `scopedExhaustionCode` set, and never `resetCreditEligible` — a reset credit reconciles a
ChatGPT plan window and cannot pay an organization's bill. The two sets are disjoint and share one
parser, so a `code`/`type` pair that disagrees, a duplicate key at any depth, or a case or
whitespace near-miss yields no code at all. `codexScopedExhaustionCode` exposes the scoped answer
alone for the rotation gate and fails closed, so only positive evidence changes a routing decision.
alone for the post-resolution rotation gate and fails closed. A code by itself cannot bind the
refusal to every credential in a heterogeneous pool.

`pausedCodexAccountIds` is a persisted Pool eligibility boundary. A paused added account or the
stable `__main__` alias remains visible for maintenance and quota reads, but is excluded from new
Expand Down
18 changes: 13 additions & 5 deletions structure/transports/responses.md
Original file line number Diff line number Diff line change
Expand Up @@ -418,11 +418,19 @@ bridge. It uses the existing account quorum, cooldown and three-rotation request
the complete credential/transport/replay identity, and attributes usage to the serving account.
Single-account installs do not retry; a missing alternate credential preserves the original error.

`shouldRetryCodexPoolAccountQuota` withholds that rotation when the 429 or 402 body names an
organization- or project-scoped exhaustion (`codexScopedExhaustionCode` in
`src/codex/quota-rejection.ts`). Every credential inside the refusing organization meets the same
counter, so the move would pay a second cold prompt prefix for no new capacity. Withholding the
move does not withhold the accounting: `src/server/responses/passthrough-delivery.ts` applies the
`shouldRetryCodexPoolAccountQuota` admits that rotation when the 429 or 402 body names an
organization- or project-scoped exhaustion because the response does not identify the refusing
scope. After resolving an alternate — on `/v1/responses` and on the single bounded send the
native `/responses/compact` path resolves — the rotation path uses `codexScopedExhaustionCode`
from `src/codex/quota-rejection.ts` to withhold organization-level retries only when both
credentials have the same known workspace account id. A stored Pool or main-pool alternate
supplies that id directly; a request-owned `main` alternate is bound by the caller credential's
own `chatgpt-account-id` via `callerCodexWorkspaceAccountId`. Project exhaustion remains
retryable because no project identity is available. Credentials in distinct or unknown
workspaces therefore retain failover, while a proven same-workspace move cannot pay a second
cold prompt prefix for no new capacity. A suppressed move still records the normalized 429/402
on the refused account, so a 5xx-wrapped quota body cools it rather than letting its wire
status record as transient. `src/server/responses/passthrough-delivery.ts` applies the
response's quota headers to the serving account and records the 429 outcome on the ordinary
delivery path, so the account still earns its cooldown and leaves the selection pool. The gate
fails closed — an empty, truncated, unparseable, duplicate-keyed or aborted body keeps the broad
Expand Down
31 changes: 24 additions & 7 deletions tests/codex-integration/codex-quota-rejection.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import { BOUNDED_BODY_MAX_BYTES } from "../../src/lib/bounded-body";
import {
consumeComboFailure,
shouldRetryCodexPoolAccountQuota,
shouldRetryCodexScopedQuotaOnAlternate,
shouldRetryCodexPoolAccountTransient,
} from "../../src/server/responses/core";
import { markResponseNonReplayable } from "../../src/lib/upstream-retry";
Expand Down Expand Up @@ -488,7 +489,8 @@ describe("Codex pre-stream quota rejection classification", () => {
});

/**
* Rotating inside the limit that refused is the send amplification #4546 exists to stop.
* Rotating inside a proven-shared limit is the send amplification #4546 exists to stop. A code
* alone cannot prove that a prospective alternate belongs to the same organization or project.
*
* openai/codex #44492 and #45602 reclassified exactly these HTTP 429 codes as terminal quota
* exhaustion while deliberately keeping `rate_limit_exceeded` and `slow_down` retryable, and
Expand All @@ -499,7 +501,7 @@ describe("Codex pre-stream quota rejection classification", () => {
* user-level rate limit stops failing over, and that regression would be invisible until a pool
* stopped rotating in production.
*/
describe("organization-scoped quota exhaustion withholds the account rotation (#4546)", () => {
describe("scoped quota exhaustion preserves unbound account rotation (#4546)", () => {
const SCOPED_CODES = [
"credit_balance_exhausted",
"organization_spend_limit_exceeded",
Expand All @@ -512,26 +514,41 @@ describe("organization-scoped quota exhaustion withholds the account rotation (#
expect(result).toEqual({
kind: "scoped-quota-exhaustion",
status: 429,
alternateRetryEligible: false,
alternateRetryEligible: true,
resetCreditEligible: false,
scopedExhaustionCode: code,
});
// A reset credit reconciles a ChatGPT plan window; it cannot pay an organization's bill.
expect(result).not.toHaveProperty("semanticCode");
});

test.each(SCOPED_CODES)("%s withholds the alternate-account send", async code => {
test.each(SCOPED_CODES)("%s keeps an unresolved alternate-account send eligible", async code => {
await expect(shouldRetryCodexPoolAccountQuota(jsonRejection(429, { code })))
.resolves.toBe(false);
.resolves.toBe(true);
});

test("a root-level code and a 402 are read the same way", async () => {
await expect(shouldRetryCodexPoolAccountQuota(
jsonPayload(429, { code: "organization_spend_limit_exceeded" }),
)).resolves.toBe(false);
)).resolves.toBe(true);
await expect(shouldRetryCodexPoolAccountQuota(
jsonRejection(402, { code: "credit_balance_exhausted" }),
)).resolves.toBe(false);
)).resolves.toBe(true);
});

test("only proven shared organization scope withholds the resolved alternate", async () => {
const rejection = () => jsonRejection(429, { code: "organization_spend_limit_exceeded" });
await expect(shouldRetryCodexScopedQuotaOnAlternate(rejection(), "workspace-a", "workspace-b"))
.resolves.toBe(true);
await expect(shouldRetryCodexScopedQuotaOnAlternate(rejection(), "workspace-a", undefined))
.resolves.toBe(true);
await expect(shouldRetryCodexScopedQuotaOnAlternate(rejection(), "workspace-a", "workspace-a"))
.resolves.toBe(false);
await expect(shouldRetryCodexScopedQuotaOnAlternate(
jsonRejection(429, { code: "project_spend_limit_exceeded" }),
"workspace-a",
"workspace-a",
)).resolves.toBe(true);
});

test.each([
Expand Down
Loading
Loading