diff --git a/crates/compositor/src/audio.rs b/crates/compositor/src/audio.rs index 1b51a7a99..fb6e3da21 100644 --- a/crates/compositor/src/audio.rs +++ b/crates/compositor/src/audio.rs @@ -4,6 +4,7 @@ use crate::ffi::*; use crate::regions::SpeedSegment; +use crate::scene::SceneAudio; use anyhow::{bail, Result}; use std::f32::consts::PI; use std::ffi::CString; @@ -26,6 +27,60 @@ const PASSTHROUGH_EPSILON: f64 = 1e-3; pub type PlanarPcm = Vec>; +/// Apply the editor's output trim and signed sync offset to the assembled timeline. +/// +/// Both stages are stateless on purpose. The editor preview plays the untouched source file +/// (seeked), while this runs on the assembled timeline — trimmed, speed-adjusted, concatenated +/// — so the two see different signals. A linear gain and a whole-sample delay are the only +/// operations that land identically on both, which is what lets the editor claim that what you +/// hear is what you export. +/// +/// That rules out the obvious next feature. A filter or a compressor carries state across +/// cuts here and not in the preview; a loudness normaliser is worse still, because its makeup +/// is a single scalar measured over the whole assembled programme, which the preview never +/// holds and which changes with every trim. Adding either back means either an offline preview +/// render or an accepted, documented divergence — not a quiet extra stage in this function. +/// +/// Bounds mirror `AUDIO_GAIN_DB_LIMIT` / `AUDIO_OFFSET_MS_LIMIT` in editorSettings.ts. The +/// result stays the same length so video and following clips cannot drift. +pub fn finish_audio(mut pcm: PlanarPcm, settings: SceneAudio) -> PlanarPcm { + let samples = pcm.first().map(Vec::len).unwrap_or(0); + if samples == 0 { + return pcm; + } + for channel in pcm.iter_mut() { + channel.resize(samples, 0.0); + } + + let trim = 10.0f32.powf(settings.gain_db.clamp(-12.0, 12.0) / 20.0); + for sample in pcm.iter_mut().flatten() { + *sample = (*sample * trim).clamp(-1.0, 1.0); + } + + let shift = ((settings.offset_ms.clamp(-500.0, 500.0) / 1_000.0) + * AUDIO_OUTPUT_SAMPLE_RATE as f64) + .round() as i64; + if shift == 0 { + return pcm; + } + if shift > 0 { + let destination = (shift as usize).min(samples); + let count = samples - destination; + for channel in pcm.iter_mut() { + channel.copy_within(..count, destination); + channel[..destination].fill(0.0); + } + } else { + let source = ((-shift) as usize).min(samples); + let count = samples - source; + for channel in pcm.iter_mut() { + channel.copy_within(source.., 0); + channel[count..].fill(0.0); + } + } + pcm +} + extern "C" { fn sn_fmt_stream(s: *mut AVFormatContext, i: i32) -> *mut AVStream; // bindgen rend `AVFormatContext` opaque (atteinte seulement par pointeur), d'où l'accesseur @@ -1063,4 +1118,71 @@ mod tests { let mixed = mix_aligned_tracks(&[(-0.001, &early)], 0.0, 8); assert_eq!(mixed[0], vec![0.5; 8]); } + + #[test] + fn signed_audio_offset_is_length_preserving() { + let settings = SceneAudio { + offset_ms: 1000.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64, + gain_db: 0.0, + }; + // Values stay inside [-1, 1] so this asserts the shift, not the output clamp. + let delayed = finish_audio(planar(&[0.1, 0.2, 0.3]), settings); + assert_eq!(delayed[0], vec![0.0, 0.1, 0.2]); + + let advanced = finish_audio( + planar(&[0.1, 0.2, 0.3]), + SceneAudio { + offset_ms: -1000.0 / AUDIO_OUTPUT_SAMPLE_RATE as f64, + ..settings + }, + ); + assert_eq!(advanced[0], vec![0.2, 0.3, 0.0]); + } + + /// The gain must be the SAME scalar the editor preview feeds its GainNode + /// (`10 ** (dB / 20)`), because that identity is the whole parity guarantee: nothing + /// else stands between what the editor plays and what this writes. + #[test] + fn output_trim_is_the_same_scalar_the_preview_applies() { + for gain_db in [-12.0f32, -6.0206, 0.0, 6.0206, 12.0] { + let result = finish_audio( + planar(&[0.25, -0.25]), + SceneAudio { + offset_ms: 0.0, + gain_db, + }, + ); + let expected = (0.25 * 10.0f32.powf(gain_db / 20.0)).clamp(-1.0, 1.0); + assert!( + (result[0][0] - expected).abs() < 1e-6, + "gain {gain_db} dB: got {}, want {expected}", + result[0][0] + ); + assert!((result[0][1] + expected).abs() < 1e-6); + } + } + + #[test] + fn out_of_range_settings_are_clamped_to_the_editor_bounds() { + // A hand-edited project (or a future UI change) must not be able to ask for an + // offset or a gain the sliders cannot display. + let result = finish_audio( + planar(&[0.5, 0.5]), + SceneAudio { + offset_ms: 9_999.0, + gain_db: 99.0, + }, + ); + assert_eq!(result[0], vec![0.0, 0.0], "offset clamps to 500 ms, not 10 s"); + + let quiet = finish_audio( + planar(&[0.5]), + SceneAudio { + offset_ms: 0.0, + gain_db: -99.0, + }, + ); + let floor = 0.5 * 10.0f32.powf(-12.0 / 20.0); + assert!((quiet[0][0] - floor).abs() < 1e-6); + } } diff --git a/crates/compositor/src/compositor_linux.rs b/crates/compositor/src/compositor_linux.rs index fc32ecaff..aa6db8de0 100644 --- a/crates/compositor/src/compositor_linux.rs +++ b/crates/compositor/src/compositor_linux.rs @@ -1230,9 +1230,12 @@ impl Compositor { // `cover_crop_uv` est la primitive partagee que macOS et Windows // utilisent ; elle rend le rect inchange quand il a deja le bon // ratio, donc aucun placement correct ne bouge. - let (cu0, cv0, cu1, cv1) = crate::frame_geometry::cover_crop_uv( + let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( [wcw, wch], [wtw as f32, wth as f32], + scene_ref + .as_ref() + .and_then(|scene| scene.layout.webcam_crop), g.w_px[0] / g.w_px[1].max(0.0001), ); // MIROIR : on inverse l'intervalle u. Le VS interpole `src` diff --git a/crates/compositor/src/compositor_macos.rs b/crates/compositor/src/compositor_macos.rs index 4ea09a42c..a9cdaedcb 100644 --- a/crates/compositor/src/compositor_macos.rs +++ b/crates/compositor/src/compositor_macos.rs @@ -1601,9 +1601,10 @@ impl Compositor { // --- caméra : ombre PiP puis vidéo --- let enc = self.begin_pass(cmd_buf, &self.rt, None, &self.pipeline_main)?; if let (true, Some((wy, wuv))) = (lp.has_webcam, webcam_tex.as_ref()) { - let (cu0, cv0, cu1, cv1) = crate::frame_geometry::cover_crop_uv( + let [cu0, cv0, cu1, cv1] = crate::frame_geometry::webcam_source_rect( [wcw, wch], [wtw as f32, wth as f32], + scene_ref.as_ref().and_then(|scene| scene.layout.webcam_crop), g.w_px[0] / g.w_px[1].max(0.0001), ); let (u0, u1) = if lp.webcam_mirror { (cu1, cu0) } else { (cu0, cu1) }; diff --git a/crates/compositor/src/compositor_windows.rs b/crates/compositor/src/compositor_windows.rs index b4f19b9be..f4cf9a5b6 100644 --- a/crates/compositor/src/compositor_windows.rs +++ b/crates/compositor/src/compositor_windows.rs @@ -1517,9 +1517,12 @@ impl Compositor { // // Le center-crop carré de square/circle en est un cas particulier (boîte 1:1) — il n'a // plus besoin d'être traité à part. - let (su0, sv0, su1, sv1) = cover_crop_uv( + let [su0, sv0, su1, sv1] = crate::frame_geometry::webcam_source_rect( [wcw, wch], [wtw as f32, wth as f32], + scene_ref + .as_ref() + .and_then(|scene| scene.layout.webcam_crop), w_px[0] / w_px[1].max(0.0001), ); // miroir = échanger les bornes u du rect source (flip horizontal). diff --git a/crates/compositor/src/frame_geometry.rs b/crates/compositor/src/frame_geometry.rs index 738a14f8e..6cfe290af 100644 --- a/crates/compositor/src/frame_geometry.rs +++ b/crates/compositor/src/frame_geometry.rs @@ -284,6 +284,23 @@ pub(crate) fn cover_crop_uv(visible: [f32; 2], tex: [f32; 2], box_ar: f32) -> (f let [u0, v0, u1, v1] = cover_uv_rect(full, tex, box_ar); (u0, v0, u1, v1) } + +/// Camera equivalent of the screen crop pipeline: apply the user crop first, then a centred +/// cover-crop inside that authored window so arbitrary layout slots never stretch the image. +pub(crate) fn webcam_source_rect( + visible: [f32; 2], + tex: [f32; 2], + crop: Option, + box_ar: f32, +) -> [f32; 4] { + let u_max = visible[0].max(1.0) / tex[0].max(1.0); + let v_max = visible[1].max(1.0) / tex[1].max(1.0); + cover_uv_rect( + screen_source_rect(u_max, v_max, crop, 1.0, [0.5, 0.5]), + tex, + box_ar, + ) +} /// Rétrécit un rect SOURCE déjà exprimé en UV (`[u0, v0, u1, v1]`) autour de son /// centre pour qu'il porte le ratio `box_ar` une fois rapporté aux pixels de la /// texture. C'est la forme générale de `object-fit: cover`, et LA primitive qui @@ -1750,4 +1767,22 @@ mod tests { assert!((su0 - (960.0 - 720.0) * 0.5 / tex[0]).abs() < 1e-6); assert!((su1 - (960.0 + 720.0) * 0.5 / tex[0]).abs() < 1e-6); } + + #[test] + fn webcam_crop_identity_keeps_the_full_visible_frame() { + let uv = webcam_source_rect([1280.0, 720.0], [2048.0, 1024.0], None, 16.0 / 9.0); + assert_rect(uv, [0.0, 0.0, 1280.0 / 2048.0, 720.0 / 1024.0]); + } + + #[test] + fn webcam_crop_applies_authored_zoom_and_pan_before_layout_cover() { + let crop = SceneCrop { + x: 0.25, + y: 0.20, + width: 0.50, + height: 0.60, + }; + let uv = webcam_source_rect([100.0, 100.0], [100.0, 100.0], Some(crop), 0.50 / 0.60); + assert_rect(uv, [0.25, 0.20, 0.75, 0.80]); + } } diff --git a/crates/compositor/src/pipeline_linux.rs b/crates/compositor/src/pipeline_linux.rs index 0d0d9804b..910738fc0 100644 --- a/crates/compositor/src/pipeline_linux.rs +++ b/crates/compositor/src/pipeline_linux.rs @@ -21,7 +21,7 @@ use std::ffi::CString; use std::ptr; use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::config::Cfg; @@ -457,6 +457,7 @@ pub fn run_composited_multi( let mut clip_frame_counts: Vec = vec![0; clips.len()]; let scene = comp.scene_snapshot(); + let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); // Ring de staging a 2 : l'export ne veut que du debit, une frame de latence // ne se voit pas dans un fichier. Voir `Compositor::set_readback_depth` pour // la raison pour laquelle la preview, elle, reste a 1. @@ -533,7 +534,10 @@ pub fn run_composited_multi( // raccourci voit son audio raccourci d'autant), puis un seul encode AAC. let declared_audio: Vec = clips.iter().map(|c| c.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); - audio_encoder.encode(&assemble_concatenated_pcm(&clip_pcm, &plan), octx)?; + audio_encoder.encode( + &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + octx, + )?; crate::ffi::averr(crate::ffi::av_write_trailer(octx), "write_trailer")?; crate::ffi::avio_closep(&mut pb); crate::ffi::avformat_free_context(octx); diff --git a/crates/compositor/src/pipeline_macos.rs b/crates/compositor/src/pipeline_macos.rs index 2ad12b662..10de3fac2 100644 --- a/crates/compositor/src/pipeline_macos.rs +++ b/crates/compositor/src/pipeline_macos.rs @@ -30,7 +30,7 @@ //! décodeurs, symétrique. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::compositor::Compositor; @@ -1076,6 +1076,7 @@ pub fn run_composited_multi( // exactement le bug de troncature en slow-motion que la doc de `walk_composited_timeline` // raconte avoir déjà coûté une fois. let scene = comp.scene_snapshot(); + let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); frames = unsafe { crate::timeline_walk::walk_composited_timeline( clips, @@ -1130,7 +1131,10 @@ pub fn run_composited_multi( // d'autant, sinon la piste dérive pour tous les suivants. let declared_audio: Vec = clips.iter().map(|clip| clip.has_audio).collect(); let plan = build_audio_concat_plan(&clip_frame_counts, &declared_audio, out_fps as f64); - audio_encoder.encode(&assemble_concatenated_pcm(&clip_pcm, &plan), octx)?; + audio_encoder.encode( + &finish_audio(assemble_concatenated_pcm(&clip_pcm, &plan), audio_settings), + octx, + )?; crate::ffi::averr( crate::ffi::av_write_trailer(octx), @@ -1188,4 +1192,4 @@ pub fn probe_frame_count(_path: &str) -> Result { // cfg-re-export `crate::compositor::Compositor`, et cette fonction helper garantit // que le type reste référencé. #[allow(dead_code)] -fn _typecheck_compositor(_c: &Compositor, _g: &Gpu) {} \ No newline at end of file +fn _typecheck_compositor(_c: &Compositor, _g: &Gpu) {} diff --git a/crates/compositor/src/pipeline_windows.rs b/crates/compositor/src/pipeline_windows.rs index f5b02ac27..11738bcd4 100644 --- a/crates/compositor/src/pipeline_windows.rs +++ b/crates/compositor/src/pipeline_windows.rs @@ -3,7 +3,7 @@ //! tout le run, deux lectures seulement. Rien dans la boucle ne peut fausser le fps. use crate::audio::{ - assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, + assemble_concatenated_pcm, build_audio_concat_plan, decode_clip_audio, finish_audio, stretch_clip_pcm_by_speed, AacEncoder, PlanarPcm, }; use crate::compositor::{Compositor, OUT_H, OUT_W}; @@ -1342,6 +1342,7 @@ unsafe fn run_multi_inner( // La scène (déjà posée par l'appelant via comp.set_scene) pilote le curseur et le // fenêtrage par clip ; `walk_composited_timeline` s'en charge. let scene = comp.scene_snapshot(); + let audio_settings = scene.as_ref().map(|scene| scene.audio).unwrap_or_default(); // ---- encodeur (choisi à l'exécution, cf. ExportCodec::candidates) + mux ---- // Backend CPU : pas de pool D3D11 du tout. `av_hwdevice_ctx_init(D3D11VA)` échoue sur @@ -1465,7 +1466,10 @@ unsafe fn run_multi_inner( &declared_audio, out_fps as f64, ); - let assembled_audio = assemble_concatenated_pcm(&clip_pcm, &audio_plan); + let assembled_audio = finish_audio( + assemble_concatenated_pcm(&clip_pcm, &audio_plan), + audio_settings, + ); audio_encoder.encode(&assembled_audio, octx)?; averr(av_write_trailer(octx), "write_trailer")?; diff --git a/crates/compositor/src/scene.rs b/crates/compositor/src/scene.rs index 2de569510..eec857fbc 100644 --- a/crates/compositor/src/scene.rs +++ b/crates/compositor/src/scene.rs @@ -45,6 +45,9 @@ pub struct SceneLayout { pub webcam_position: Option, /// la webcam rétrécit pendant un zoom actif. pub webcam_reactive_zoom: bool, + /// User-authored source crop for the camera. Absent keeps the full frame. + #[serde(default)] + pub webcam_crop: Option, /// Rect webcam résolu côté app (0..1 fractions du cadre de sortie), en PARITÉ EXACTE avec /// `computeCompositeLayout` (TS). Permet à TS et Rust de partager la même source de vérité : /// le natif ne dérive PLUS ses propres placements pour PiP/dual-frame/vertical-stack — il @@ -361,6 +364,21 @@ pub struct SceneCrop { pub height: f32, } +/// Audio finishing. Deliberately limited to a linear gain and a signed delay — the two +/// operations the editor preview can apply identically to the source file it plays. See +/// `audio::finish_audio` before adding a field here. +/// +/// Every field carries `#[serde(default)]`: a payload from a build that predates one of them +/// must degrade to "that stage is neutral", not fail the whole scene. +#[derive(Debug, Clone, Copy, Default, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SceneAudio { + #[serde(default)] + pub offset_ms: f64, + #[serde(default)] + pub gain_db: f32, +} + #[derive(Debug, Clone, Copy, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SceneOutput { @@ -389,6 +407,9 @@ pub struct Scene { #[serde(default)] pub camera_fullscreen_regions: Vec, pub cursor: SceneCursor, + /// Global audio finishing. Default keeps old scene payloads bit-for-bit compatible. + #[serde(default)] + pub audio: SceneAudio, /// Crop écran par clip, dans le même ordre que `clips` (`cropByClip` côté TS). #[serde(default)] pub crop_by_clip: Vec>, diff --git a/src/components/ai-edition/NewEditorShell.module.css b/src/components/ai-edition/NewEditorShell.module.css index bb96cc6ce..4dbef964e 100644 --- a/src/components/ai-edition/NewEditorShell.module.css +++ b/src/components/ai-edition/NewEditorShell.module.css @@ -2026,6 +2026,20 @@ color: var(--danger); } +.secondaryBtn { + margin: 4px var(--sp-4) 12px; + min-height: 32px; + padding: 0 12px; + border: 1px solid var(--border); + border-radius: var(--r-sm); + background: var(--surface-2); + color: var(--fg-2); + font: 500 12px/1 var(--font-body); + cursor: pointer; +} +.secondaryBtn:hover { background: var(--surface-3); color: var(--fg); } +.secondaryBtn:disabled { opacity: 0.45; cursor: not-allowed; } + .authPanel { padding: 14px 16px; border: 1px solid var(--brand); diff --git a/src/components/ai-edition/NewEditorShell.tsx b/src/components/ai-edition/NewEditorShell.tsx index 68931e66c..3ca3a1100 100644 --- a/src/components/ai-edition/NewEditorShell.tsx +++ b/src/components/ai-edition/NewEditorShell.tsx @@ -324,6 +324,7 @@ export function NewEditorShell() { if (!document) return []; return document.assets.map((asset) => ({ id: asset.id, + filePath: /^(https?|blob|data):/.test(asset.originalPath) ? undefined : asset.originalPath, // Real Electron assets are filesystem paths and go through toFileUrl. // In the browser preview an asset can already point at an http(s)/ // blob/data URL served by Vite; toFileUrl would mangle those into a @@ -399,10 +400,14 @@ export function NewEditorShell() { ); const handleDropAsset = useCallback( - (assetId: string) => { - void tl.insertClipAt(assetId, clips.length); - }, - [tl, clips.length], + (assetId: string) => + tl.insertClipAt(assetId, clips.length).catch((error) => { + toast.error(te("mediaStage.couldNotAddAsset"), { + description: error instanceof Error ? error.message : String(error), + }); + throw error; + }), + [tl, clips.length, te], ); // Ref so the 'ended' listener below always sees the latest clips without tearing @@ -1226,7 +1231,7 @@ export function NewEditorShell() { /> ) : mode === "media" ? ( - + ) : ( void handleNewRecording()} diff --git a/src/components/ai-edition/RightPanes.tsx b/src/components/ai-edition/RightPanes.tsx index 5ac70471a..42a35179a 100644 --- a/src/components/ai-edition/RightPanes.tsx +++ b/src/components/ai-edition/RightPanes.tsx @@ -6,6 +6,7 @@ // self-sufficient). import { + AudioLines, FileText, HelpCircle, Layout as LayoutIcon, @@ -41,6 +42,7 @@ import type { AxcutTrimRange, AxcutWord, } from "@/lib/ai-edition/schema"; +import { AUDIO_GAIN_DB_LIMIT, AUDIO_OFFSET_MS_LIMIT } from "@/lib/ai-edition/store/editorSettings"; import { useProjectStore } from "@/lib/ai-edition/store/projectStore"; import { useEditorSettings } from "@/lib/ai-edition/store/useEditorSettings"; import { @@ -1520,6 +1522,23 @@ export function LayoutPane() { ? hasAnyClipWithCamera(document.assets, document.timeline.clips) : false; const layoutControlsDisabled = !hasDocument || !hasAnyCamera; + const webcamCrop = settings.webcamCropRegion; + const cropZoomPct = Math.round(100 / webcamCrop.width); + const cropPanX = webcamCrop.width >= 0.999 ? 50 : (webcamCrop.x / (1 - webcamCrop.width)) * 100; + const cropPanY = webcamCrop.height >= 0.999 ? 50 : (webcamCrop.y / (1 - webcamCrop.height)) * 100; + const setCropZoom = (zoomPct: number) => { + const size = 100 / Math.max(100, zoomPct); + const centerX = webcamCrop.x + webcamCrop.width / 2; + const centerY = webcamCrop.y + webcamCrop.height / 2; + setLive({ + webcamCropRegion: { + x: Math.min(1 - size, Math.max(0, centerX - size / 2)), + y: Math.min(1 - size, Math.max(0, centerY - size / 2)), + width: size, + height: size, + }, + }); + }; return ( } helpText={ts("layout.help")}>
{ts("layout.preset")}
@@ -1652,6 +1671,91 @@ export function LayoutPane() { ) : null} +
{ts("layout.webcamFraming")}
+
+ void commit()} + /> + = 0.999} + onChange={(value) => + setLive({ + webcamCropRegion: { ...webcamCrop, x: (value / 100) * (1 - webcamCrop.width) }, + }) + } + onCommit={() => void commit()} + /> + = 0.999} + onChange={(value) => + setLive({ + webcamCropRegion: { ...webcamCrop, y: (value / 100) * (1 - webcamCrop.height) }, + }) + } + onCommit={() => void commit()} + /> +
+ + ); +} + +// ─── Audio ──────────────────────────────────────────────────────── + +export function AudioPane() { + const ts = useScopedT("settings"); + const { settings, set, setLive, commit, hasDocument } = useEditorSettings(); + return ( + } helpText={ts("audio.help")}> +
+ setLive({ audioOffsetMs: value })} + onCommit={() => void commit()} + /> + setLive({ audioGainDb: value })} + onCommit={() => void commit()} + /> +
+ ); } diff --git a/src/components/ai-edition/VirtualPreview.audio.test.ts b/src/components/ai-edition/VirtualPreview.audio.test.ts new file mode 100644 index 000000000..a565aebc0 --- /dev/null +++ b/src/components/ai-edition/VirtualPreview.audio.test.ts @@ -0,0 +1,27 @@ +import { describe, expect, it } from "vitest"; +import { resolveAudioPreviewTime } from "./VirtualPreview"; + +describe("resolveAudioPreviewTime", () => { + it("delays audio for a positive offset", () => { + expect(resolveAudioPreviewTime(0.1, 160, 10)).toEqual({ targetTimeSec: 0, shouldPlay: false }); + expect(resolveAudioPreviewTime(1, 160, 10)).toEqual({ targetTimeSec: 0.84, shouldPlay: true }); + }); + + it("advances audio for a negative offset", () => { + expect(resolveAudioPreviewTime(1, -160, 10)).toEqual({ targetTimeSec: 1.16, shouldPlay: true }); + }); + + it("stops instead of seeking past the track", () => { + expect(resolveAudioPreviewTime(9.9, -160, 10)).toEqual({ + targetTimeSec: 10, + shouldPlay: false, + }); + }); + + it("plays while the duration is still unknown", () => { + expect(resolveAudioPreviewTime(1, 0, Number.NaN)).toEqual({ + targetTimeSec: 1, + shouldPlay: true, + }); + }); +}); diff --git a/src/components/ai-edition/VirtualPreview.tsx b/src/components/ai-edition/VirtualPreview.tsx index c3eb6cbbd..923d426c0 100644 --- a/src/components/ai-edition/VirtualPreview.tsx +++ b/src/components/ai-edition/VirtualPreview.tsx @@ -28,9 +28,55 @@ import styles from "./VirtualPreview.module.css"; export interface VideoSource { id: string; src: string; + /** Original filesystem path, used by the main process to expose the second audio track. */ + filePath?: string; label: string; } +export function resolveAudioPreviewTime( + videoTimeSec: number, + offsetMs: number, + durationSec = Number.POSITIVE_INFINITY, +) { + const raw = videoTimeSec - offsetMs / 1000; + const finiteDuration = Number.isFinite(durationSec) && durationSec > 0 ? durationSec : Infinity; + return { + targetTimeSec: Math.min(Math.max(0, raw), finiteDuration), + shouldPlay: raw >= 0 && raw < finiteDuration, + }; +} + +interface PreviewAudioGraph { + context: AudioContext; + gain: GainNode; +} + +/** + * The preview's ONLY audio processing is the output trim, and that is deliberate: it is + * the same `10 ** (dB / 20)` scalar `finish_audio` applies natively, so what the editor + * plays is what the export writes. + * + * Nothing with state belongs here. The export runs on the assembled timeline (trimmed, + * speed-adjusted, concatenated); the preview runs on the untouched source file, seeked. + * A filter or a compressor would see a different signal on each side and drift — and an + * offline stage measured over the whole programme (a loudness normaliser) cannot exist + * here at all, because the preview never holds that programme. + */ +function applyPreviewAudioSettings( + graph: PreviewAudioGraph | null, + elements: Array, + gainDb: number, +): void { + const outputGain = 10 ** (gainDb / 20); + if (!graph) { + for (const element of elements) { + if (element) element.volume = Math.min(1, outputGain); + } + return; + } + graph.gain.gain.value = outputGain; +} + /** First clip (by timeline order) starting strictly after `afterTimelineStartSec` — * independent of the `clips` array's own order, which is never guaranteed to match * timeline order (a clip can be inserted/reordered at any array index; only @@ -93,6 +139,10 @@ export function VirtualPreview({ clockRef, }: VirtualPreviewProps) { const { settings } = useEditorSettings(); + const audioOffsetMsRef = useRef(settings.audioOffsetMs); + useEffect(() => { + audioOffsetMsRef.current = settings.audioOffsetMs; + }, [settings.audioOffsetMs]); // ponytail: an oversized, offset video inside .videoFrame's overflow:hidden // box — the same "scale + negative-position the full frame, let the // container clip the rest" technique the export renderer uses via a Pixi @@ -114,6 +164,20 @@ export function VirtualPreview({ top: `${(-region.y * 100) / region.height}%`, }; const videoRef = useRef(null); + const primaryAudioRef = useRef(null); + const supplementalAudioRef = useRef(null); + const [primaryAudioEl, setPrimaryAudioEl] = useState(null); + const [supplementalAudioEl, setSupplementalAudioEl] = useState(null); + const [supplementalAudioSrc, setSupplementalAudioSrc] = useState(null); + const [audioProbeComplete, setAudioProbeComplete] = useState(false); + const audioGainDbRef = useRef(settings.audioGainDb); + useEffect(() => { + audioGainDbRef.current = settings.audioGainDb; + }, [settings.audioGainDb]); + const audioContextRef = useRef(null); + const audioContextCloseTimerRef = useRef | null>(null); + const audioSourceNodesRef = useRef(new WeakMap()); + const audioGraphRef = useRef(null); const videoFrameRef = useRef(null); const isProgrammaticSeekRef = useRef(false); @@ -133,6 +197,129 @@ export function VirtualPreview({ const virtualDurationSec = useMemo(() => totalVirtualDuration(clips), [clips]); const activeSource = videoSources[sourceIndex] ?? null; + useEffect(() => { + let cancelled = false; + setSupplementalAudioSrc(null); + setAudioProbeComplete(false); + if (!activeSource?.filePath || !window.electronAPI?.preparePreviewAudioTrack) { + setAudioProbeComplete(true); + return () => { + cancelled = true; + }; + } + void window.electronAPI.preparePreviewAudioTrack(activeSource.filePath).then( + (result) => { + if (cancelled) return; + setSupplementalAudioSrc(result.success ? (result.path ?? null) : null); + setAudioProbeComplete(true); + }, + () => { + if (cancelled) return; + setSupplementalAudioSrc(null); + setAudioProbeComplete(true); + }, + ); + return () => { + cancelled = true; + }; + }, [activeSource?.filePath]); + + // Sum the audio elements into one gain node so the output trim can boost past 0 dB, + // which `element.volume` cannot do. The primary media element carries track 1; on macOS + // the existing IPC helper extracts track 2 (normally the microphone) so both are audible + // instead of Chromium silently choosing one. + useEffect(() => { + if (!primaryAudioEl || !audioProbeComplete) return; + if (supplementalAudioSrc && !supplementalAudioEl) return; + const elements = [primaryAudioEl, supplementalAudioEl].filter( + (value): value is HTMLAudioElement => Boolean(value), + ); + const graph = ((): PreviewAudioGraph | null => { + try { + let context = audioContextRef.current; + if (!context || context.state === "closed") { + context = new AudioContext(); + audioContextRef.current = context; + audioSourceNodesRef.current = new WeakMap(); + } + const gain = context.createGain(); + gain.connect(context.destination); + return { context, gain }; + } catch { + return null; + } + })(); + if (!graph) { + // WebAudio can be unavailable in unit tests or under a denied audio policy. No source + // node was created, so `volume` still reaches the output — capped at 0 dB. + applyPreviewAudioSettings(null, elements, audioGainDbRef.current); + return; + } + + const connectedSources: MediaElementAudioSourceNode[] = []; + for (const element of elements) { + try { + let source = audioSourceNodesRef.current.get(element); + if (!source) { + source = graph.context.createMediaElementSource(element); + audioSourceNodesRef.current.set(element, source); + } + source.disconnect(); + source.connect(graph.gain); + connectedSources.push(source); + } catch { + // Routing THIS element failed; leave the others alone. Once + // createMediaElementSource has run for an element its audio no longer reaches + // the default output, so tearing the whole graph down here would mute the + // preview outright rather than degrade it. + } + } + audioGraphRef.current = graph; + applyPreviewAudioSettings(graph, elements, audioGainDbRef.current); + return () => { + audioGraphRef.current = null; + for (const source of connectedSources) source.disconnect(); + graph.gain.disconnect(); + }; + }, [primaryAudioEl, supplementalAudioEl, supplementalAudioSrc, audioProbeComplete]); + + // Keep one AudioContext for the component. Closing and recreating it on an effect rerun + // permanently silences an HTMLAudioElement because createMediaElementSource may only be + // called once for that element. Delay final cleanup by one task so React StrictMode's + // intentional setup → cleanup → setup cycle can cancel the close and reuse the context. + useEffect(() => { + if (audioContextCloseTimerRef.current) { + clearTimeout(audioContextCloseTimerRef.current); + audioContextCloseTimerRef.current = null; + } + return () => { + audioContextCloseTimerRef.current = setTimeout(() => { + audioContextCloseTimerRef.current = null; + const context = audioContextRef.current; + audioContextRef.current = null; + audioSourceNodesRef.current = new WeakMap(); + if (context) void context.close(); + }, 0); + }; + }, []); + + useEffect(() => { + applyPreviewAudioSettings( + audioGraphRef.current, + [primaryAudioRef.current, supplementalAudioRef.current], + settings.audioGainDb, + ); + }, [settings.audioGainDb]); + + const setPrimaryAudioElement = useCallback((element: HTMLAudioElement | null) => { + primaryAudioRef.current = element; + setPrimaryAudioEl(element); + }, []); + const setSupplementalAudioElement = useCallback((element: HTMLAudioElement | null) => { + supplementalAudioRef.current = element; + setSupplementalAudioEl(element); + }, []); + // ponytail: the cursor overlay wants source-media time (the recorded // cursor samples live on the original mp4 timeline, not the edited // virtual timeline). `setSourceTimeSec` is called from the 60 Hz rAF @@ -209,6 +396,31 @@ export function VirtualPreview({ if (!v || !Number.isFinite(v.currentTime)) { return; } + for (const audio of [primaryAudioRef.current, supplementalAudioRef.current]) { + if (!audio) continue; + const target = resolveAudioPreviewTime( + v.currentTime, + audioOffsetMsRef.current, + audio.duration, + ); + if (audio.playbackRate !== v.playbackRate) audio.playbackRate = v.playbackRate; + if (Math.abs(audio.currentTime - target.targetTimeSec) > 0.025) { + try { + audio.currentTime = target.targetTimeSec; + } catch { + // media metadata not ready yet + } + } + if (!v.paused && target.shouldPlay && audio.paused) { + if (audioGraphRef.current?.context.state === "suspended") { + void audioGraphRef.current.context.resume(); + } + const playback = audio.play(); + if (playback) void playback.catch(() => undefined); + } else if ((v.paused || !target.shouldPlay) && !audio.paused) { + audio.pause(); + } + } // Publish this frame's live position/rate for other media elements // (webcam) to read directly — see playback-clock.ts for why this // bypasses React state entirely. @@ -539,6 +751,7 @@ export function VirtualPreview({ cursor: settings.cursorShow ? "none" : undefined, }} preload="metadata" + muted playsInline onLoadedMetadata={(e) => { setLoadState("ready"); @@ -626,6 +839,24 @@ export function VirtualPreview({ // handles clip-end advancement, so dropping the event // handler here is safe. /> +