Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ const BITFUN_VOICE_INSTRUCTIONS: &str = r#"You are BitFun's client-level realtim

Use get_bitfun_client_context whenever the user asks about the current client, open workspaces/projects, visible sessions, running tasks, or names a workspace whose exact id is not already known from a fresh context result. Never guess a workspace id. Use switch_bitfun_workspace for navigation-only requests. When the user asks you to inspect, create, change, run, debug, research, or otherwise complete work, call run_bitfun_task with a complete standalone task description and the intended workspace_id. Omit workspace_id only when the user clearly means the active workspace. Set activate_workspace to true when the user asks to enter, switch to, or visibly work in that workspace; use false only for an explicit background request.

If the user asks to stop, cancel, abort, or interrupt the BitFun task currently running through this client voice assistant, call stop_bitfun_task immediately. A stop request is a control operation, not a new task: never pass it to run_bitfun_task and never claim the task stopped before the stop_bitfun_task result confirms it. Do not claim that work is complete before the tool result arrives. BitFun will speak brief public progress summaries while the Agent task is running; do not expose private reasoning, raw logs, or tool payloads. After the tool result arrives, summarize the outcome clearly and mention any user action still required. Never invent client state or task results."#;
If the user asks to stop, cancel, abort, or interrupt the BitFun task currently running through this client voice assistant, call stop_bitfun_task immediately. A stop request is a control operation, not a new task: never pass it to run_bitfun_task and never claim the task stopped before the stop_bitfun_task result confirms it. Do not claim that work is complete before the tool result arrives. BitFun will speak brief public progress summaries while the Agent task is running; do not expose private reasoning, raw logs, or tool payloads. BitFun also speaks a concise final outcome itself. When a task tool result contains outcome_spoken=true, do not repeat that outcome; wait for the user's next request. If outcome_spoken is false or absent, summarize the outcome clearly and mention any user action still required. Never invent client state or task results."#;

#[derive(Debug, Clone)]
pub struct VolcengineRealtimeSpeechConfig {
Expand Down Expand Up @@ -916,6 +916,7 @@ mod tests {
.and_then(Value::as_str)
.unwrap();
assert!(instructions.contains("never claim the task stopped"));
assert!(instructions.contains("outcome_spoken=true"));
assert!(instructions.contains("workspace-1"));
assert_eq!(
payload.pointer("/extension/extra/enable_proactive_speak"),
Expand Down
12 changes: 12 additions & 0 deletions src/web-ui/src/app/App.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ import {
hideStartupOverlay,
isStartupOverlayPresent,
} from './startup/startupOverlay';
import {
clearStartupModuleReloadAttempt,
retryStartupAfterModuleLoadFailure,
} from './startup/startupModuleRecovery';
import { ToolbarModeProvider } from '../flow_chat/components/toolbar-mode/ToolbarModeProvider';
import { RealtimeVoiceCallProvider } from '../flow_chat/components/voice/RealtimeVoiceCallContext';
import type { AgentCompanionPetCommand } from './services/agentCompanionPetCommands';
Expand All @@ -49,6 +53,7 @@ const LazyAppLayout = lazy(async () => {
startupTrace.markPhase('app_layout_import_start');
try {
const module = await import('./layout/AppLayout');
clearStartupModuleReloadAttempt();
startupTrace.markPhase('app_layout_import_end');
return {
default: function AppLayoutStartupGate({ onReady }: AppLayoutStartupGateProps) {
Expand All @@ -62,6 +67,13 @@ const LazyAppLayout = lazy(async () => {
};
} catch (error) {
startupTrace.markPhase('app_layout_import_failed');
if (retryStartupAfterModuleLoadFailure(error)) {
startupTrace.markPhase('app_layout_import_reload_requested');
return await new Promise<never>(() => undefined);
}
// The static overlay otherwise hides AppErrorBoundary and makes a real
// startup failure look like an endless loading state.
void hideStartupOverlay();
throw error;
}
});
Expand Down
76 changes: 76 additions & 0 deletions src/web-ui/src/app/startup/startupModuleRecovery.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
import { describe, expect, it, vi } from 'vitest';

import {
clearStartupModuleReloadAttempt,
isRecoverableStartupModuleLoadError,
retryStartupAfterModuleLoadFailure,
} from './startupModuleRecovery';

function createStorage() {
const values = new Map<string, string>();
return {
getItem: (key: string) => values.get(key) ?? null,
setItem: (key: string, value: string) => values.set(key, value),
removeItem: (key: string) => values.delete(key),
};
}

describe('startup module recovery', () => {
it('recognizes WebKit and Chromium dynamic module load failures', () => {
expect(isRecoverableStartupModuleLoadError(
new TypeError('Importing a module script failed.'),
)).toBe(true);
expect(isRecoverableStartupModuleLoadError(
new TypeError('Failed to fetch dynamically imported module'),
)).toBe(true);
expect(isRecoverableStartupModuleLoadError(new Error('render failed'))).toBe(false);
});

it('reloads at most once until a module import succeeds', () => {
const storage = createStorage();
const reload = vi.fn();
const runtime = { isDevelopment: true, storage, reload };
const error = new TypeError('Importing a module script failed.');

expect(retryStartupAfterModuleLoadFailure(error, runtime)).toBe(true);
expect(reload).toHaveBeenCalledOnce();
expect(retryStartupAfterModuleLoadFailure(error, runtime)).toBe(false);
expect(reload).toHaveBeenCalledOnce();

clearStartupModuleReloadAttempt(storage);
expect(retryStartupAfterModuleLoadFailure(error, runtime)).toBe(true);
expect(reload).toHaveBeenCalledTimes(2);
});

it('does not reload production or non-module failures', () => {
const storage = createStorage();
const reload = vi.fn();

expect(retryStartupAfterModuleLoadFailure(
new TypeError('Importing a module script failed.'),
{ isDevelopment: false, storage, reload },
)).toBe(false);
expect(retryStartupAfterModuleLoadFailure(
new Error('application initialization failed'),
{ isDevelopment: true, storage, reload },
)).toBe(false);
expect(reload).not.toHaveBeenCalled();
});

it('falls through to the error boundary if reload itself fails', () => {
const storage = createStorage();
const reload = vi.fn(() => {
throw new Error('reload unavailable');
});

expect(retryStartupAfterModuleLoadFailure(
new TypeError('Importing a module script failed.'),
{ isDevelopment: true, storage, reload },
)).toBe(false);

expect(retryStartupAfterModuleLoadFailure(
new TypeError('Importing a module script failed.'),
{ isDevelopment: true, storage, reload: vi.fn() },
)).toBe(true);
});
});
72 changes: 72 additions & 0 deletions src/web-ui/src/app/startup/startupModuleRecovery.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
const STARTUP_MODULE_RELOAD_KEY = 'bitfun:startup-module-reload-attempted';

const DYNAMIC_MODULE_LOAD_ERROR =
/(?:importing a module script failed|failed to fetch dynamically imported module|error loading dynamically imported module|load failed)/i;

type StartupModuleRecoveryStorage = Pick<Storage, 'getItem' | 'setItem' | 'removeItem'>;

interface StartupModuleRecoveryRuntime {
isDevelopment: boolean;
storage: StartupModuleRecoveryStorage;
reload: () => void;
}

function browserRuntime(): StartupModuleRecoveryRuntime | null {
try {
return {
isDevelopment: import.meta.env.DEV,
storage: window.sessionStorage,
reload: () => window.location.reload(),
};
} catch {
return null;
}
}

export function isRecoverableStartupModuleLoadError(error: unknown): boolean {
if (!(error instanceof Error) || error.name !== 'TypeError') {
return false;
}
return DYNAMIC_MODULE_LOAD_ERROR.test(error.message);
}

/**
* Reload once when a development WebView rejects a lazy module request.
* A single retry recovers stale/transient Vite module responses without
* turning a persistent code or server error into a reload loop.
*/
export function retryStartupAfterModuleLoadFailure(
error: unknown,
runtime: StartupModuleRecoveryRuntime | null = browserRuntime(),
): boolean {
if (!runtime?.isDevelopment || !isRecoverableStartupModuleLoadError(error)) {
return false;
}

try {
if (runtime.storage.getItem(STARTUP_MODULE_RELOAD_KEY) === '1') {
return false;
}
runtime.storage.setItem(STARTUP_MODULE_RELOAD_KEY, '1');
runtime.reload();
return true;
} catch {
try {
runtime.storage.removeItem(STARTUP_MODULE_RELOAD_KEY);
} catch {
// Storage may be unavailable in a restricted WebView. The caller will
// reveal the normal application error boundary instead.
}
return false;
}
}

export function clearStartupModuleReloadAttempt(
storage?: StartupModuleRecoveryStorage,
): void {
try {
(storage ?? window.sessionStorage).removeItem(STARTUP_MODULE_RELOAD_KEY);
} catch {
// A successful module import must not fail because storage is unavailable.
}
}
8 changes: 8 additions & 0 deletions src/web-ui/src/app/startup/startupPerformanceContract.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -292,6 +292,14 @@ describe('startup performance contract', () => {
expect(source).toContain('!appLayoutReady');
});

it('recovers development module load failures without trapping the error boundary behind the startup overlay', () => {
const source = readSource('../App.tsx');

expect(source).toContain('retryStartupAfterModuleLoadFailure(error)');
expect(source).toContain("startupTrace.markPhase('app_layout_import_reload_requested')");
expect(source).toMatch(/void hideStartupOverlay\(\);\s+throw error;/);
});

it('keeps non-default shell surfaces out of the startup import path', () => {
const appSource = readSource('../App.tsx');
const appLayoutSource = readSource('../layout/AppLayout.tsx');
Expand Down
90 changes: 65 additions & 25 deletions src/web-ui/src/flow_chat/components/voice/useRealtimeVoiceCall.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { RealtimePcmPlayer } from './realtimeVoiceAudio';
import { applyRealtimeAsrSnapshot } from './realtimeVoiceTranscript';
import {
runBitFunVoiceTask,
summarizeVoiceTaskConclusion,
VoiceTaskCancelledError,
type VoiceTaskProgress,
type VoiceTaskProgressPhase,
Expand Down Expand Up @@ -245,7 +246,7 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
return t(`voiceCall.call.taskPhases.${progress.phase}`);
}, [t]);

const enqueueSpokenProgress = useCallback((sessionId: string, text: string): Promise<void> => {
const enqueueSpokenTaskText = useCallback((sessionId: string, text: string): Promise<void> => {
const spokenText = text.trim();
if (!spokenText) return Promise.resolve();
const epoch = spokenProgressEpochRef.current;
Expand All @@ -259,7 +260,7 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
try {
await speechAPI.speakRealtimeText(sessionId, spokenText);
} catch (firstError) {
log.warn('Failed to enqueue BitFun progress speech; retrying once', {
log.warn('Failed to enqueue BitFun task speech; retrying once', {
sessionId,
firstError,
});
Expand All @@ -279,7 +280,7 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
.catch(() => undefined)
.then(send);
spokenProgressQueueRef.current = queued.catch(error => {
log.warn('Failed to speak BitFun task progress after retry', { sessionId, error });
log.warn('Failed to speak BitFun task update after retry', { sessionId, error });
setStatus(t('voiceCall.call.status.audioPlaybackFailed'));
});
return queued;
Expand All @@ -290,15 +291,33 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
setTaskPhase(progress.phase);
setTaskProgressText(text);
setStatus(text);
void enqueueSpokenProgress(sessionId, text);
}, [enqueueSpokenProgress, progressText]);
void enqueueSpokenTaskText(sessionId, text);
}, [enqueueSpokenTaskText, progressText]);

const speakTextProgress = useCallback((sessionId: string, text: string) => {
setTaskPhase(previous => previous === 'stopping' ? previous : 'working');
setTaskProgressText(text);
setStatus(text);
void enqueueSpokenProgress(sessionId, text);
}, [enqueueSpokenProgress]);
void enqueueSpokenTaskText(sessionId, text);
}, [enqueueSpokenTaskText]);

const speakTaskOutcome = useCallback(async (
sessionId: string,
text: string,
): Promise<boolean> => {
setTaskPhase(null);
setTaskProgressText('');
setAssistantTranscript(text);
setStatus(text);
try {
// Share the progress queue so the closing brief cannot overtake an
// already accepted in-flight update.
await enqueueSpokenTaskText(sessionId, text);
return true;
} catch {
return false;
}
}, [enqueueSpokenTaskText]);

const clearAssistantSpeechFallbackTimer = useCallback(() => {
if (assistantSpeechFallbackTimerRef.current === null) return;
Expand Down Expand Up @@ -516,14 +535,19 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
});
settleActiveTask(activeTask, { status: 'completed', result });
setTaskSessionId(result.sessionId);
setTaskPhase(null);
setTaskProgressText('');
setStatus(t('voiceCall.call.taskComplete'));
await spokenProgressQueueRef.current.catch(() => undefined);
const outcomeText = result.conclusion
? t('voiceCall.call.taskOutcome.completed', { conclusion: result.conclusion })
: t('voiceCall.call.taskOutcome.completedWithoutConclusion');
const outcomeSpoken = await speakTaskOutcome(callSessionId, outcomeText);
await speechAPI.sendRealtimeToolResult(
callSessionId,
call.callId,
JSON.stringify({ ok: true, session_id: result.sessionId, summary: result.summary }),
JSON.stringify({
ok: true,
session_id: result.sessionId,
summary: result.summary,
outcome_spoken: outcomeSpoken,
}),
);
} catch (error) {
const message = error instanceof Error ? error.message : String(error);
Expand All @@ -533,13 +557,19 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
if (activeTask && error instanceof VoiceTaskCancelledError) {
settleActiveTask(activeTask, { status: 'cancelled', sessionId: error.sessionId });
setTaskSessionId(error.sessionId);
setTaskPhase(null);
setTaskProgressText('');
setStatus(t('voiceCall.call.taskStopped'));
const outcomeSpoken = await speakTaskOutcome(
callSessionId,
t('voiceCall.call.taskOutcome.cancelled'),
);
await speechAPI.sendRealtimeToolResult(
callSessionId,
call.callId,
JSON.stringify({ ok: true, cancelled: true, session_id: error.sessionId }),
JSON.stringify({
ok: true,
cancelled: true,
session_id: error.sessionId,
outcome_spoken: outcomeSpoken,
}),
).catch(sendError => {
log.warn('Failed to return BitFun task cancellation to realtime voice session', {
sendError,
Expand All @@ -550,18 +580,28 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
if (activeTask) {
settleActiveTask(activeTask, { status: 'failed', error: message });
}
setTaskPhase(null);
setTaskProgressText('');
setStatus(
command?.kind === 'run_task' || command?.kind === 'stop_task'
? t('voiceCall.call.taskFailed')
: t('voiceCall.call.status.error'),
);
const isTaskCommand = command?.kind === 'run_task' || command?.kind === 'stop_task';
let outcomeSpoken: boolean | undefined;
if (isTaskCommand) {
const reason = summarizeVoiceTaskConclusion(message);
const outcomeText = reason
? t('voiceCall.call.taskOutcome.failed', { reason })
: t('voiceCall.call.taskOutcome.failedWithoutReason');
outcomeSpoken = await speakTaskOutcome(callSessionId, outcomeText);
} else {
setTaskPhase(null);
setTaskProgressText('');
setStatus(t('voiceCall.call.status.error'));
}
log.error('BitFun client voice tool failed', { callId: call.callId, tool: call.name, error });
await speechAPI.sendRealtimeToolResult(
callSessionId,
call.callId,
JSON.stringify({ ok: false, error: message }),
JSON.stringify({
ok: false,
error: message,
...(outcomeSpoken === undefined ? {} : { outcome_spoken: outcomeSpoken }),
}),
).catch(sendError => {
log.warn('Failed to return BitFun task error to realtime voice session', { sendError });
});
Expand All @@ -570,7 +610,7 @@ export function useRealtimeVoiceCallController(disabled = false): RealtimeVoiceC
activeTaskRef.current = null;
}
}
}, [speakProgress, speakTextProgress, t]);
}, [speakProgress, speakTaskOutcome, speakTextProgress, t]);

const handleRealtimeEvent = useCallback((event: SpeechRealtimeEvent) => {
const session = sessionRef.current;
Expand Down
Loading
Loading