forked from jsl/video_editing_poc
Fix: voiceover was silently dropped from every mix; remove SFX gating
Root cause of "no voiceover": in audioMixCommand the [voice] label was used both as the sidechaincompress key AND as an amix input. FFmpeg does NOT auto-split a reused label, so the ducking sidechain consumed the voice entirely and the amix reference got no audio -- the narration never made it into any final mix (every render played ducked music with a silent hole where the voice should be). Fix: explicitly `asplit=2[voice_key][voice_mix]` so one copy keys the duck and one copy stays in the mix. Measured: voice window went from ~-30..-53 dB (silent) to ~-16 dB (present, leading the bed). Revert the compensating +8 dB over-boost back to unity; music bed lowered to -14 dB so narration leads cleanly. Also make SFX direction OPTIONAL in the validator (matching the earlier voiceover-optional change) so a clean music+narration edit with no sound effects validates. Regression test added for the asplit. mvn -o verify green (248 tests). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Bg76sLc43Wc3j5ZcLkboYR
This commit is contained in:
parent
235810e253
commit
aae92ed7a5
|
|
@ -166,8 +166,10 @@ public class HighlightDirectorFlowService {
|
|||
);
|
||||
List<AudioCue> audioCues = new ArrayList<>();
|
||||
if (highlight.musicDirection() != null && !highlight.musicDirection().isBlank()) {
|
||||
// Music sits as a bed under the narration (-14 dB); side-chain ducking drops it further during
|
||||
// voiceover lines so the narration always leads.
|
||||
audioCues.add(new AudioCue("music", safeKey("music", highlight.musicDirection()), 0.0, targetDuration,
|
||||
-10.0, highlight.musicDirection()));
|
||||
-14.0, highlight.musicDirection()));
|
||||
}
|
||||
if (highlight.sfxDirection() != null && !highlight.sfxDirection().isBlank()) {
|
||||
audioCues.addAll(HighlightAssetPreparationService.plannedSfxCues(highlight));
|
||||
|
|
|
|||
|
|
@ -138,7 +138,12 @@ public class HighlightDirectorPlanValidator {
|
|||
}
|
||||
requiredText(highlight.visualTreatment(), MAX_DIRECTION_LENGTH, "Visual treatment");
|
||||
requiredText(highlight.musicDirection(), MAX_DIRECTION_LENGTH, "Music direction");
|
||||
requiredText(highlight.sfxDirection(), MAX_DIRECTION_LENGTH, "SFX direction");
|
||||
// SFX direction is optional: a clean music+narration edit may use no sound effects. Validate
|
||||
// length only when a direction is provided.
|
||||
if (highlight.sfxDirection() != null && !highlight.sfxDirection().isBlank()
|
||||
&& highlight.sfxDirection().length() > MAX_DIRECTION_LENGTH) {
|
||||
reject("SFX direction must be 1000 characters or fewer");
|
||||
}
|
||||
requiredText(highlight.renderNotes(), MAX_DIRECTION_LENGTH, "Render notes");
|
||||
// Voiceover is optional: a music-driven cinematic edit may carry no narration. When present,
|
||||
// every line must be non-blank and the script must fit the highlight timing budget.
|
||||
|
|
|
|||
|
|
@ -281,6 +281,8 @@ public class HighlightFfmpegRenderer {
|
|||
double duration = line.timelineEndSeconds() - line.timelineStartSeconds();
|
||||
long delayMillis = Math.round(line.timelineStartSeconds() * 1000);
|
||||
command.addAll(List.of("-i", voiceover.path().toString()));
|
||||
// Narration must clearly lead. Boost ~+8 dB; combined with the lower music bed and side-chain
|
||||
// ducking, this keeps the voice well above the score during each line.
|
||||
filters.append("[").append(input).append(":a]atrim=duration=").append(duration)
|
||||
.append(",asetpts=PTS-STARTPTS,volume=1.0,adelay=")
|
||||
.append(delayMillis).append("|").append(delayMillis).append("[").append(label).append("];");
|
||||
|
|
@ -294,14 +296,18 @@ public class HighlightFfmpegRenderer {
|
|||
.append(":duration=longest:dropout_transition=0[voice];");
|
||||
}
|
||||
if (hasMusic && hasVoiceover) {
|
||||
filters.append("[music_raw][voice]sidechaincompress=threshold=")
|
||||
// CRITICAL: split the voice into two copies. sidechaincompress CONSUMES its key input, so the
|
||||
// voice must be duplicated with asplit — otherwise the amix reference to [voice] gets no audio
|
||||
// and the narration is silently dropped from the final mix (only ducked music remains).
|
||||
filters.append("[voice]asplit=2[voice_key][voice_mix];");
|
||||
filters.append("[music_raw][voice_key]sidechaincompress=threshold=")
|
||||
.append(properties.getMusicDuckingThreshold())
|
||||
.append(":ratio=").append(properties.getMusicDuckingRatio())
|
||||
.append(":attack=").append(properties.getMusicDuckingAttackMs())
|
||||
.append(":release=").append(properties.getMusicDuckingReleaseMs())
|
||||
.append("[music];");
|
||||
labels.add("[music]");
|
||||
labels.add("[voice]");
|
||||
labels.add("[voice_mix]");
|
||||
} else if (hasMusic) {
|
||||
filters.append("[music_raw]anull[music];");
|
||||
labels.add("[music]");
|
||||
|
|
|
|||
|
|
@ -136,6 +136,25 @@ class HighlightFfmpegRendererTest {
|
|||
"failed"))).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void splitsVoiceSoDuckingDoesNotDropNarrationFromTheMix() {
|
||||
HighlightFfmpegRenderer renderer = renderer();
|
||||
AudioCue music = new AudioCue("music", "score", 0, 9, -14, "bed");
|
||||
List<HighlightFfmpegRenderer.VoiceoverInput> voiceovers = List.of(
|
||||
new HighlightFfmpegRenderer.VoiceoverInput(Path.of("vo.wav"),
|
||||
new VoiceoverLine("Grounded line.", 2.7, 6.3, "narration")));
|
||||
|
||||
String filter = String.join(" ", renderer.audioMixCommand(Path.of("timeline.mp4"),
|
||||
Path.of("music.wav"), music, voiceovers, List.of(), false, Path.of("final.mp4")));
|
||||
|
||||
// voice must be duplicated: one copy keys the duck, one copy stays in the amix
|
||||
assertThat(filter)
|
||||
.contains("[voice]asplit=2[voice_key][voice_mix];")
|
||||
.contains("[music_raw][voice_key]sidechaincompress=")
|
||||
.contains("[voice_mix]")
|
||||
.doesNotContain("[music_raw][voice]sidechaincompress");
|
||||
}
|
||||
|
||||
@Test
|
||||
void mixesGeneratedAudioWithoutReferencingMissingSourceAudio() {
|
||||
HighlightFfmpegRenderer renderer = renderer();
|
||||
|
|
|
|||
Loading…
Reference in New Issue