diff --git a/packages/ai/src/protocols/google-speech.ts b/packages/ai/src/protocols/google-speech.ts index 96e2e59a3075..f168f416b56d 100644 --- a/packages/ai/src/protocols/google-speech.ts +++ b/packages/ai/src/protocols/google-speech.ts @@ -56,10 +56,15 @@ interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata { // --------------------------------------------------------------------------- const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed) { + if (request.format === "pcm" && request.mode === "generate" && /^gemini-3\.8-.*-tts(?:-|$)/.test(request.model.id)) + return yield* route.unsupported( + "media.format", + `${route.name} returns WAV by default for Gemini 3.8 TTS unary requests; omit the format to accept it`, + ) if (request.format !== undefined && request.format !== "pcm") return yield* route.unsupported( "media.format", - `${route.name} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`, + `${route.name} only accepts raw PCM as an explicit format; omit it to accept the provider's default output`, ) const voiceName = SpeechStream.voiceID(request.voice) return MediaProtocol.json( @@ -97,10 +102,18 @@ const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: stri return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const }) -const finish = (state: State) => { +const finish = (state: State, context: MediaProtocol.ResponseContext) => { const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE + const output = + state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav" + ? SpeechStream.container("wav", sampleRate) + : SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`) + if (context.request.format === "pcm" && output.info.format !== "pcm") + return Effect.fail( + route.frameError(`Google Speech returned ${output.info.format} instead of the requested raw PCM`), + ) return SpeechStream.finish(route, state, { - ...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`), + ...output, usage: GeminiGenerateContent.usage(state.usage), providerMetadata: GeminiGenerateContent.providerMetadata(state), detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`, diff --git a/packages/ai/test/speech.test.ts b/packages/ai/test/speech.test.ts index 56de05240201..a4ae078ed328 100644 --- a/packages/ai/test/speech.test.ts +++ b/packages/ai/test/speech.test.ts @@ -28,10 +28,49 @@ const cartesia = Cartesia.configure({ apiKey: "test", baseURL: "https://cartesia const google = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech( "gemini-2.5-flash-preview-tts", ) +const google38 = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech( + "gemini-3.8-flash-tts", +) +const google38Lite = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech( + "gemini-3.8-flash-lite-tts", +) const deepgram = Deepgram.configure({ apiKey: "test", baseURL: "https://deepgram.test" }).speech("aura-2-thalia-en") const voice = "JBFqnCBsd6RMkjVDRZzb" describe("Speech", () => { + it.effect("preserves Google's WAV output instead of describing it as raw PCM", () => + Effect.gen(function* () { + const bytes = new TextEncoder().encode("RIFF....WAVEfmt ") + const response = yield* Speech.generate({ model: google38, text: "Hi" }).pipe( + Effect.provide( + respond( + JSON.stringify({ + candidates: [ + { content: { parts: [{ inlineData: { mimeType: "audio/wav", data: Encoding.encodeBase64(bytes) } }] } }, + ], + }), + "application/json", + ), + ), + ) + expect(response.audio.mediaType).toBe("audio/wav") + expect(response.audio.info?.format).toBe("wav") + expect(response.audio.info?.encoding).toBeUndefined() + expect(yield* response.audio.bytes()).toEqual(bytes) + }), + ) + + it.effect("rejects raw PCM for Gemini 3.8 unary requests before sending", () => + Effect.gen(function* () { + const errors = yield* Effect.all( + [google38, google38Lite].map((model) => + Speech.generate({ model, text: "Hi", format: "pcm" }).pipe(Effect.flip), + ), + ).pipe(Effect.provide(layer(() => Effect.die("An unsupported request reached the network")))) + expect(errors.map((error) => error.reason._tag)).toEqual(["UnsupportedOperation", "UnsupportedOperation"]) + }), + ) + it.effect("rejects what a provider cannot produce before sending anything", () => Effect.gen(function* () { const errors = yield* Effect.all(