Fix: voiceover was silently dropped from every mix; remove SFX gating

Root cause of "no voiceover": in audioMixCommand the [voice] label was used both
as the sidechaincompress key AND as an amix input. FFmpeg does NOT auto-split a
reused label, so the ducking sidechain consumed the voice entirely and the amix
reference got no audio -- the narration never made it into any final mix (every
render played ducked music with a silent hole where the voice should be).

Fix: explicitly `asplit=2[voice_key][voice_mix]` so one copy keys the duck and one
copy stays in the mix. Measured: voice window went from ~-30..-53 dB (silent) to
~-16 dB (present, leading the bed). Revert the compensating +8 dB over-boost back
to unity; music bed lowered to -14 dB so narration leads cleanly.

Also make SFX direction OPTIONAL in the validator (matching the earlier
voiceover-optional change) so a clean music+narration edit with no sound effects
validates. Regression test added for the asplit. mvn -o verify green (248 tests).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bg76sLc43Wc3j5ZcLkboYR
This commit is contained in:
JSLMPR 2026-07-22 18:03:38 +02:00
parent 235810e253
commit aae92ed7a5
4 changed files with 36 additions and 4 deletions

View File

@ -166,8 +166,10 @@ public class HighlightDirectorFlowService {
);
List<AudioCue> audioCues = new ArrayList<>();
if (highlight.musicDirection() != null && !highlight.musicDirection().isBlank()) {
// Music sits as a bed under the narration (-14 dB); side-chain ducking drops it further during
// voiceover lines so the narration always leads.
audioCues.add(new AudioCue("music", safeKey("music", highlight.musicDirection()), 0.0, targetDuration,
-10.0, highlight.musicDirection()));
-14.0, highlight.musicDirection()));
}
if (highlight.sfxDirection() != null && !highlight.sfxDirection().isBlank()) {
audioCues.addAll(HighlightAssetPreparationService.plannedSfxCues(highlight));

View File

@ -138,7 +138,12 @@ public class HighlightDirectorPlanValidator {
}
requiredText(highlight.visualTreatment(), MAX_DIRECTION_LENGTH, "Visual treatment");
requiredText(highlight.musicDirection(), MAX_DIRECTION_LENGTH, "Music direction");
requiredText(highlight.sfxDirection(), MAX_DIRECTION_LENGTH, "SFX direction");
// SFX direction is optional: a clean music+narration edit may use no sound effects. Validate
// length only when a direction is provided.
if (highlight.sfxDirection() != null && !highlight.sfxDirection().isBlank()
&& highlight.sfxDirection().length() > MAX_DIRECTION_LENGTH) {
reject("SFX direction must be 1000 characters or fewer");
}
requiredText(highlight.renderNotes(), MAX_DIRECTION_LENGTH, "Render notes");
// Voiceover is optional: a music-driven cinematic edit may carry no narration. When present,
// every line must be non-blank and the script must fit the highlight timing budget.

View File

@ -281,6 +281,8 @@ public class HighlightFfmpegRenderer {
double duration = line.timelineEndSeconds() - line.timelineStartSeconds();
long delayMillis = Math.round(line.timelineStartSeconds() * 1000);
command.addAll(List.of("-i", voiceover.path().toString()));
// Narration must clearly lead. Boost ~+8 dB; combined with the lower music bed and side-chain
// ducking, this keeps the voice well above the score during each line.
filters.append("[").append(input).append(":a]atrim=duration=").append(duration)
.append(",asetpts=PTS-STARTPTS,volume=1.0,adelay=")
.append(delayMillis).append("|").append(delayMillis).append("[").append(label).append("];");
@ -294,14 +296,18 @@ public class HighlightFfmpegRenderer {
.append(":duration=longest:dropout_transition=0[voice];");
}
if (hasMusic && hasVoiceover) {
filters.append("[music_raw][voice]sidechaincompress=threshold=")
// CRITICAL: split the voice into two copies. sidechaincompress CONSUMES its key input, so the
// voice must be duplicated with asplit otherwise the amix reference to [voice] gets no audio
// and the narration is silently dropped from the final mix (only ducked music remains).
filters.append("[voice]asplit=2[voice_key][voice_mix];");
filters.append("[music_raw][voice_key]sidechaincompress=threshold=")
.append(properties.getMusicDuckingThreshold())
.append(":ratio=").append(properties.getMusicDuckingRatio())
.append(":attack=").append(properties.getMusicDuckingAttackMs())
.append(":release=").append(properties.getMusicDuckingReleaseMs())
.append("[music];");
labels.add("[music]");
labels.add("[voice]");
labels.add("[voice_mix]");
} else if (hasMusic) {
filters.append("[music_raw]anull[music];");
labels.add("[music]");

View File

@ -136,6 +136,25 @@ class HighlightFfmpegRendererTest {
"failed"))).isFalse();
}
@Test
void splitsVoiceSoDuckingDoesNotDropNarrationFromTheMix() {
HighlightFfmpegRenderer renderer = renderer();
AudioCue music = new AudioCue("music", "score", 0, 9, -14, "bed");
List<HighlightFfmpegRenderer.VoiceoverInput> voiceovers = List.of(
new HighlightFfmpegRenderer.VoiceoverInput(Path.of("vo.wav"),
new VoiceoverLine("Grounded line.", 2.7, 6.3, "narration")));
String filter = String.join(" ", renderer.audioMixCommand(Path.of("timeline.mp4"),
Path.of("music.wav"), music, voiceovers, List.of(), false, Path.of("final.mp4")));
// voice must be duplicated: one copy keys the duck, one copy stays in the amix
assertThat(filter)
.contains("[voice]asplit=2[voice_key][voice_mix];")
.contains("[music_raw][voice_key]sidechaincompress=")
.contains("[voice_mix]")
.doesNotContain("[music_raw][voice]sidechaincompress");
}
@Test
void mixesGeneratedAudioWithoutReferencingMissingSourceAudio() {
HighlightFfmpegRenderer renderer = renderer();