Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 16 additions & 3 deletions packages/ai/src/protocols/google-speech.ts
Original file line number Diff line number Diff line change
Expand Up @@ -56,10 +56,15 @@ interface State extends SpeechStream.Audio, GeminiGenerateContent.Metadata {
// ---------------------------------------------------------------------------

const fromRequest = Effect.fn("GoogleSpeech.fromRequest")(function* (request: MediaProtocol.Addressed<Request>) {
if (request.format === "pcm" && request.mode === "generate" && /^gemini-3\.8-.*-tts(?:-|$)/.test(request.model.id))
return yield* route.unsupported(
"media.format",
`${route.name} returns WAV by default for Gemini 3.8 TTS unary requests; omit the format to accept it`,
)
if (request.format !== undefined && request.format !== "pcm")
return yield* route.unsupported(
"media.format",
`${route.name} only returns raw PCM; request format "pcm" or omit it, then wrap the samples yourself`,
`${route.name} only accepts raw PCM as an explicit format; omit it to accept the provider's default output`,
)
const voiceName = SpeechStream.voiceID(request.voice)
return MediaProtocol.json(
Expand Down Expand Up @@ -97,10 +102,18 @@ const step = Effect.fn("GoogleSpeech.step")(function* (state: State, frame: stri
return [next, audio.flatMap((part) => SpeechStream.delta(next, part.data)[1])] as const
})

const finish = (state: State) => {
const finish = (state: State, context: MediaProtocol.ResponseContext<Request>) => {
const sampleRate = SpeechStream.sampleRate(state.mimeType) ?? DEFAULT_SAMPLE_RATE
const output =
state.mimeType?.split(";")[0]?.toLowerCase() === "audio/wav"
? SpeechStream.container("wav", sampleRate)
: SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`)
if (context.request.format === "pcm" && output.info.format !== "pcm")
return Effect.fail(
route.frameError(`Google Speech returned ${output.info.format} instead of the requested raw PCM`),
)
return SpeechStream.finish(route, state, {
...SpeechStream.pcm("pcm_s16le", sampleRate, state.mimeType ?? `audio/L16;codec=pcm;rate=${sampleRate}`),
...output,
usage: GeminiGenerateContent.usage(state.usage),
providerMetadata: GeminiGenerateContent.providerMetadata(state),
detail: state.finishReason === undefined ? undefined : `finish reason: ${state.finishReason}`,
Expand Down
39 changes: 39 additions & 0 deletions packages/ai/test/speech.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,49 @@ const cartesia = Cartesia.configure({ apiKey: "test", baseURL: "https://cartesia
const google = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech(
"gemini-2.5-flash-preview-tts",
)
const google38 = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech(
"gemini-3.8-flash-tts",
)
const google38Lite = Google.configure({ apiKey: "test", baseURL: "https://google.test/v1beta" }).speech(
"gemini-3.8-flash-lite-tts",
)
const deepgram = Deepgram.configure({ apiKey: "test", baseURL: "https://deepgram.test" }).speech("aura-2-thalia-en")
const voice = "JBFqnCBsd6RMkjVDRZzb"

describe("Speech", () => {
it.effect("preserves Google's WAV output instead of describing it as raw PCM", () =>
Effect.gen(function* () {
const bytes = new TextEncoder().encode("RIFF....WAVEfmt ")
const response = yield* Speech.generate({ model: google38, text: "Hi" }).pipe(
Effect.provide(
respond(
JSON.stringify({
candidates: [
{ content: { parts: [{ inlineData: { mimeType: "audio/wav", data: Encoding.encodeBase64(bytes) } }] } },
],
}),
"application/json",
),
),
)
expect(response.audio.mediaType).toBe("audio/wav")
expect(response.audio.info?.format).toBe("wav")
expect(response.audio.info?.encoding).toBeUndefined()
expect(yield* response.audio.bytes()).toEqual(bytes)
}),
)

it.effect("rejects raw PCM for Gemini 3.8 unary requests before sending", () =>
Effect.gen(function* () {
const errors = yield* Effect.all(
[google38, google38Lite].map((model) =>
Speech.generate({ model, text: "Hi", format: "pcm" }).pipe(Effect.flip),
),
).pipe(Effect.provide(layer(() => Effect.die("An unsupported request reached the network"))))
expect(errors.map((error) => error.reason._tag)).toEqual(["UnsupportedOperation", "UnsupportedOperation"])
}),
)

it.effect("rejects what a provider cannot produce before sending anything", () =>
Effect.gen(function* () {
const errors = yield* Effect.all(
Expand Down
Loading