diff --git a/packages/opencode/src/tests/ws-pool.test.ts b/packages/opencode/src/tests/ws-pool.test.ts index cc80d1d..8a81fa8 100644 --- a/packages/opencode/src/tests/ws-pool.test.ts +++ b/packages/opencode/src/tests/ws-pool.test.ts @@ -277,6 +277,41 @@ describe('createWebSocketFetch', () => { ) }) + test('keeps a rate-limit response.failed retryable after a lifecycle frame before output', async () => { + const rateLimitCalls: string[] = [] + await withFakeWebSocket( + ({ message }) => ({ + send() { + message(JSON.stringify({ type: 'response.created' })) + message( + JSON.stringify({ + type: 'response.failed', + response: { + failed: { rate_limit_reached_type: 'secondary' }, + }, + }), + ) + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + onRateLimitReached: (window) => rateLimitCalls.push(window), + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + + await expect(response.text()).rejects.toBeInstanceOf( + ResponseStreamError, + ) + expect(rateLimitCalls).toEqual(['secondary']) + websocketFetch.close() + }, + ) + }) + test('marks an admission-time usage limit with the provider reset before rejecting the stream', async () => { const resetAtSeconds = 1_784_958_366 const rateLimitCalls: Array<{ @@ -432,6 +467,43 @@ describe('createWebSocketFetch', () => { ) }) + test('marks a 429 after a lifecycle frame before output', async () => { + const rateLimitCalls: string[] = [] + await withFakeWebSocket( + ({ message }) => ({ + send() { + message(JSON.stringify({ type: 'response.created' })) + message( + JSON.stringify({ + type: 'error', + status: 429, + error: { + type: 'rate_limit_exceeded', + message: 'Rate limit exceeded', + }, + }), + ) + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + onRateLimitReached: (window) => rateLimitCalls.push(window), + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + + await expect(response.text()).rejects.toMatchObject({ + isRetryable: true, + }) + expect(rateLimitCalls).toEqual(['rate_limit_exceeded']) + websocketFetch.close() + }, + ) + }) + test('keeps websocket connection-limit admission errors on HTTP fallback without a rate-limit mark', async () => { const rateLimitCalls: string[] = [] let httpRequests = 0 @@ -1964,6 +2036,312 @@ describe('createWebSocketFetch', () => { ) }) + test('keeps a socket error before output retryable', async () => { + await withFakeWebSocket( + ({ error }) => ({ + send() { + error('connection dropped') + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + firstEventGraceMs: 0, + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const error = await response.text().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(ResponseStreamError) + expect(error).toMatchObject({ isRetryable: true }) + websocketFetch.close() + }, + ) + }) + + test('keeps a socket error after a lifecycle frame but before output retryable', async () => { + let failSocket: () => void = () => { + throw new Error('socket failure was not initialized') + } + await withFakeWebSocket( + ({ error, message }) => ({ + send() { + message(JSON.stringify({ type: 'response.created' })) + failSocket = () => error('connection dropped') + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const reader = response.body!.getReader() + + expect(new TextDecoder().decode((await reader.read()).value)).toContain( + 'response.created', + ) + failSocket() + const error = await reader.read().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(ResponseStreamError) + expect(error).toMatchObject({ isRetryable: true }) + websocketFetch.close() + }, + ) + }) + + test('surfaces a socket error after output without a retryable stream marker', async () => { + let failSocket: () => void = () => { + throw new Error('socket failure was not initialized') + } + await withFakeWebSocket( + ({ error, message }) => ({ + send() { + message(JSON.stringify({ type: 'response.output_text.delta' })) + failSocket = () => error('connection dropped') + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const reader = response.body!.getReader() + + expect(new TextDecoder().decode((await reader.read()).value)).toContain( + 'response.output_text.delta', + ) + failSocket() + const error = await reader.read().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(Error) + expect(error).not.toBeInstanceOf(ResponseStreamError) + expect(APICallError.isInstance(error)).toBe(false) + websocketFetch.close() + }, + ) + }) + + test('surfaces an early close after output without a retryable stream marker', async () => { + let failSocket: () => void = () => { + throw new Error('socket failure was not initialized') + } + await withFakeWebSocket( + ({ close, message }) => ({ + send() { + message(JSON.stringify({ type: 'response.output_text.delta' })) + failSocket = () => close(1006, 'connection dropped') + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const reader = response.body!.getReader() + + expect(new TextDecoder().decode((await reader.read()).value)).toContain( + 'response.output_text.delta', + ) + failSocket() + const error = await reader.read().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(Error) + expect(error).not.toBeInstanceOf(ResponseStreamError) + expect(APICallError.isInstance(error)).toBe(false) + websocketFetch.close() + }, + ) + }) + + test('surfaces an idle timeout after output without a retryable stream marker', async () => { + await withFakeWebSocket( + ({ message }) => ({ + send() { + message(JSON.stringify({ type: 'response.output_text.delta' })) + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + idleTimeout: 1, + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const reader = response.body!.getReader() + + expect(new TextDecoder().decode((await reader.read()).value)).toContain( + 'response.output_text.delta', + ) + await new Promise((resolve) => setTimeout(resolve, 20)) + const error = await reader.read().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(Error) + expect(error).not.toBeInstanceOf(ResponseStreamError) + expect(APICallError.isInstance(error)).toBe(false) + websocketFetch.close() + }, + ) + }) + + test('surfaces a binary frame after output without a retryable stream marker', async () => { + let failSocket: () => void = () => { + throw new Error('socket failure was not initialized') + } + await withFakeWebSocket( + ({ binary, message }) => ({ + send() { + message(JSON.stringify({ type: 'response.output_text.delta' })) + failSocket = () => binary(new Uint8Array([1])) + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const reader = response.body!.getReader() + + expect(new TextDecoder().decode((await reader.read()).value)).toContain( + 'response.output_text.delta', + ) + failSocket() + const error = await reader.read().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(Error) + expect(error).not.toBeInstanceOf(ResponseStreamError) + expect(APICallError.isInstance(error)).toBe(false) + websocketFetch.close() + }, + ) + }) + + test('surfaces a connection-limit callback failure after output without a retryable stream marker', async () => { + let failSocket: () => void = () => { + throw new Error('socket failure was not initialized') + } + await withFakeWebSocket( + ({ message }) => ({ + send() { + message(JSON.stringify({ type: 'response.output_text.delta' })) + failSocket = () => + message( + JSON.stringify({ + type: 'error', + error: { + code: 'websocket_connection_limit_reached', + message: 'Responses websocket connection limit reached', + }, + }), + ) + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const reader = response.body!.getReader() + + expect(new TextDecoder().decode((await reader.read()).value)).toContain( + 'response.output_text.delta', + ) + failSocket() + const error = await reader.read().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(Error) + expect(error).not.toBeInstanceOf(ResponseStreamError) + expect(APICallError.isInstance(error)).toBe(false) + websocketFetch.close() + }, + ) + }) + + test('surfaces a wrapped 503 after output without a retryable stream marker', async () => { + let failSocket: () => void = () => { + throw new Error('socket failure was not initialized') + } + await withFakeWebSocket( + ({ message }) => ({ + send() { + message(JSON.stringify({ type: 'response.output_text.delta' })) + failSocket = () => + message( + JSON.stringify({ + type: 'error', + status: 503, + error: { message: 'upstream unavailable' }, + }), + ) + }, + }), + async () => { + const websocketFetch = createWebSocketFetch({ + url: 'https://example.test/backend-api/codex/responses', + }) + const response = await websocketFetch( + 'https://example.test/backend-api/codex/responses', + streamRequest({ input: [] }), + ) + const reader = response.body!.getReader() + + expect(new TextDecoder().decode((await reader.read()).value)).toContain( + 'response.output_text.delta', + ) + failSocket() + const error = await reader.read().then( + () => undefined, + (caught) => caught, + ) + + expect(error).toBeInstanceOf(Error) + expect(error).not.toBeInstanceOf(ResponseStreamError) + expect(APICallError.isInstance(error)).toBe(false) + websocketFetch.close() + }, + ) + }) + test('preserves provider header timeout abort reason so OpenCode can retry it', async () => { class HeaderTimeoutError extends Error { override readonly name = 'ProviderHeaderTimeoutError' @@ -2006,6 +2384,8 @@ function streamRequest(body: Record): RequestInit { type FakeWebSocketContext = { message(data: string): void + binary(data: Uint8Array): void + error(message?: string): void close(code?: number, reason?: string): void } @@ -2037,6 +2417,8 @@ async function withFakeWebSocket( this.url = url this.behavior = behavior({ message: (data) => this.emit('message', { data }), + binary: (data) => this.emit('message', { data }), + error: (message = '') => this.emit('error', { message }), close: (code = 1000, reason = '') => { this.readyState = FakeWebSocket.CLOSED this.emit('close', { code, reason }) diff --git a/packages/opencode/src/ws.ts b/packages/opencode/src/ws.ts index 27c6232..87c6f51 100644 --- a/packages/opencode/src/ws.ts +++ b/packages/opencode/src/ws.ts @@ -279,6 +279,7 @@ export function streamResponsesWebSocket( let cleanupSocket = () => {} let completed = false let emitted = false + let emittedOutput = false let idleTimer: ReturnType | undefined // Call ids the response finalizes (one response.output_item.done per item). // Only these are guaranteed present in the response previous_response_id will @@ -302,10 +303,36 @@ export function streamResponsesWebSocket( } function invalidate(error: ResponseStreamError) { + fail(error, error) + } + + // Gated on generated output rather than user-visible text: a + // `response.output_item.done` carrying a function_call produces no text at + // all, yet it is exactly the point after which a replay would re-run a + // side-effecting tool and bill for it twice. Duplicate text is the cheap + // half of the hazard; a re-dispatched tool call is the expensive one. + // + // What that conservatism costs: a transport failure in the window after the + // first output item but before anything the user would notice now ends the + // turn instead of rerouting. We give up a reroute rather than risk a double + // charge. Narrowing this further needs a dispatch-based discriminator (has + // OpenCode acted on the frame yet?), not a visibility-based one. + function invalidateTransport(error: ResponseStreamError) { + if (emittedOutput) { + fail( + new Error(error.message, { cause: error }), + new ResponseStreamError(error.message, { cause: error }), + ) + return + } + invalidate(error) + } + + function fail(error: Error, connectionError: ResponseStreamError) { if (completed) return completed = true cleanup() - options.onConnectionInvalid?.(error) + options.onConnectionInvalid?.(connectionError) controller?.error(error) } @@ -314,7 +341,7 @@ export function streamResponsesWebSocket( if (!options.idleTimeout) return if (idleTimer) clearTimeout(idleTimer) idleTimer = setTimeout( - () => invalidate(new ResponseStreamError(message)), + () => invalidateTransport(new ResponseStreamError(message)), options.idleTimeout, ) } @@ -322,7 +349,9 @@ export function streamResponsesWebSocket( async function onMessage(message: MessageEvent) { if (completed) return if (typeof message.data !== 'string') { - invalidate(new ResponseStreamError('Unexpected binary WebSocket frame')) + invalidateTransport( + new ResponseStreamError('Unexpected binary WebSocket frame'), + ) return } @@ -350,7 +379,7 @@ export function streamResponsesWebSocket( return } - const admissionRateLimit = !emitted + const admissionRateLimit = !emittedOutput ? parseRateLimitSignal(event) : undefined if (admissionRateLimit && event) { @@ -385,7 +414,7 @@ export function streamResponsesWebSocket( return } } catch (error) { - invalidate( + invalidateTransport( new ResponseStreamError( error instanceof Error ? error.message : String(error), { @@ -403,15 +432,16 @@ export function streamResponsesWebSocket( completed = true cleanup() options.onTerminal?.(event) + const error = new APICallError({ + message: wrappedError.message, + url: socket.url, + requestBodyValues: options.body, + statusCode: wrappedError.status, + responseHeaders: wrappedError.headers, + responseBody: wrappedError.body, + }) controller?.error( - new APICallError({ - message: wrappedError.message, - url: socket.url, - requestBodyValues: options.body, - statusCode: wrappedError.status, - responseHeaders: wrappedError.headers, - responseBody: wrappedError.body, - }), + emittedOutput ? new Error(error.message, { cause: error }) : error, ) return } @@ -454,7 +484,7 @@ export function streamResponsesWebSocket( // to THIS connection via the captured callback. options.onRateLimitReached?.(label) options.onTerminal?.(event) - if (!emitted) { + if (!emittedOutput) { // Nothing was streamed yet (rate limit at admission, the common // case): force a retryable stream error so OpenCode re-issues and the // fetch override reroutes to a healthy account THIS turn. @@ -504,6 +534,12 @@ export function streamResponsesWebSocket( ), ) emitted = true + if ( + translatedEvent.type !== 'response.created' && + translatedEvent.type !== 'response.in_progress' + ) { + emittedOutput = true + } resetIdleTimeout('idle timeout waiting for websocket') if (!translatedEvent) return @@ -538,12 +574,14 @@ export function streamResponsesWebSocket( } function onError(error: Event) { - invalidate(new ResponseStreamError(errorMessage(error), { cause: error })) + invalidateTransport( + new ResponseStreamError(errorMessage(error), { cause: error }), + ) } function onClose(event: CloseEvent) { if (completed) return - invalidate( + invalidateTransport( new ResponseStreamError( closeMessage( 'WebSocket closed before response.completed',