diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index bc93498..5ac7341 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -308,7 +308,7 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 11: Implement asset provider interfaces for voiceover, music, SFX, fonts, and LUTs. - [x] Milestone 12: Add a local licensed asset library with category tags and deterministic asset selection. - [ ] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests. -- [ ] Milestone 13 progress: Renderer now applies timed text overlays and records resolved/planned edit assets in the render manifest. Dynamic crop keyframes, audio ducking, and loudness normalization remain. +- [ ] Milestone 13 progress: Renderer now applies timed text overlays, records resolved/planned edit assets in the render manifest, ducks music under voiceover, and normalizes final mix loudness with configurable mastering values. Dynamic crop keyframes remain. - [ ] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters. - [ ] Milestone 15: Add an operator runbook for placing a video, running the service locally, choosing a model, reviewing the director plan, and finding final clips. - [ ] Milestone 16: Add integration tests with small fixture videos for family, food, car, and generic content. diff --git a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java index c7eee84..8118068 100644 --- a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java +++ b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java @@ -522,6 +522,22 @@ public class VideoClippingProperties { private String voiceoverProvider = "noop"; + private double loudnessTargetI = -16.0; + + private double loudnessTruePeak = -1.5; + + private double loudnessRange = 11.0; + + private double musicDuckingThreshold = 0.045; + + private double musicDuckingRatio = 8.0; + + @Min(1) + private int musicDuckingAttackMs = 20; + + @Min(1) + private int musicDuckingReleaseMs = 250; + private final Assets assets = new Assets(); private final LocalDirector localDirector = new LocalDirector(); @@ -654,6 +670,62 @@ public class VideoClippingProperties { this.voiceoverProvider = voiceoverProvider; } + public double getLoudnessTargetI() { + return loudnessTargetI; + } + + public void setLoudnessTargetI(double loudnessTargetI) { + this.loudnessTargetI = loudnessTargetI; + } + + public double getLoudnessTruePeak() { + return loudnessTruePeak; + } + + public void setLoudnessTruePeak(double loudnessTruePeak) { + this.loudnessTruePeak = loudnessTruePeak; + } + + public double getLoudnessRange() { + return loudnessRange; + } + + public void setLoudnessRange(double loudnessRange) { + this.loudnessRange = loudnessRange; + } + + public double getMusicDuckingThreshold() { + return musicDuckingThreshold; + } + + public void setMusicDuckingThreshold(double musicDuckingThreshold) { + this.musicDuckingThreshold = musicDuckingThreshold; + } + + public double getMusicDuckingRatio() { + return musicDuckingRatio; + } + + public void setMusicDuckingRatio(double musicDuckingRatio) { + this.musicDuckingRatio = musicDuckingRatio; + } + + public int getMusicDuckingAttackMs() { + return musicDuckingAttackMs; + } + + public void setMusicDuckingAttackMs(int musicDuckingAttackMs) { + this.musicDuckingAttackMs = musicDuckingAttackMs; + } + + public int getMusicDuckingReleaseMs() { + return musicDuckingReleaseMs; + } + + public void setMusicDuckingReleaseMs(int musicDuckingReleaseMs) { + this.musicDuckingReleaseMs = musicDuckingReleaseMs; + } + public Assets getAssets() { return assets; } diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index 9b6f096..a399904 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -265,18 +265,33 @@ public class FfmpegEditRenderer implements EditRenderer { List labels = new ArrayList<>(List.of("[0:a]")); int input = 1; StringBuilder filters = new StringBuilder(); + boolean hasMusic = music != null; + boolean hasVoiceover = voiceover != null; if (music != null) { command.addAll(List.of("-i", music.toString())); - filters.append("[").append(input).append(":a]volume=0.25[music];"); - labels.add("[music]"); + filters.append("[").append(input).append(":a]volume=0.25[music_raw];"); input++; } if (voiceover != null) { command.addAll(List.of("-i", voiceover.toString())); filters.append("[").append(input).append(":a]volume=1.0[voice];"); - labels.add("[voice]"); input++; } + if (hasMusic && hasVoiceover) { + filters.append("[music_raw][voice]sidechaincompress=threshold=") + .append(properties.getMusicDuckingThreshold()) + .append(":ratio=").append(properties.getMusicDuckingRatio()) + .append(":attack=").append(properties.getMusicDuckingAttackMs()) + .append(":release=").append(properties.getMusicDuckingReleaseMs()) + .append("[music];"); + labels.add("[music]"); + labels.add("[voice]"); + } else if (hasMusic) { + filters.append("[music_raw]anull[music];"); + labels.add("[music]"); + } else if (hasVoiceover) { + labels.add("[voice]"); + } for (int index = 0; index < soundEffects.size(); index++) { SfxInput soundEffect = soundEffects.get(index); AudioCue cue = soundEffect.cue(); @@ -291,7 +306,11 @@ public class FfmpegEditRenderer implements EditRenderer { input++; } filters.append(String.join("", labels)).append("amix=inputs=").append(labels.size()) - .append(":duration=first:dropout_transition=2[a]"); + .append(":duration=first:dropout_transition=2,loudnorm=I=") + .append(properties.getLoudnessTargetI()) + .append(":TP=").append(properties.getLoudnessTruePeak()) + .append(":LRA=").append(properties.getLoudnessRange()) + .append("[a]"); command.addAll(List.of("-filter_complex", filters.toString(), "-map", "0:v:0", "-map", "[a]", "-c:v", "copy", "-c:a", "aac", "-b:a", properties.getAudioBitrate(), "-ar", Integer.toString(properties.getAudioSampleRate()), output.toString())); diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index dbc958c..446478d 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -30,6 +30,13 @@ video-clipping: video-bitrate: ${VIDEO_EDITING_VIDEO_BITRATE:12000k} audio-bitrate: ${VIDEO_EDITING_AUDIO_BITRATE:192k} voiceover-provider: ${VIDEO_EDITING_VOICEOVER_PROVIDER:noop} + loudness-target-i: ${VIDEO_EDITING_LOUDNESS_TARGET_I:-16.0} + loudness-true-peak: ${VIDEO_EDITING_LOUDNESS_TRUE_PEAK:-1.5} + loudness-range: ${VIDEO_EDITING_LOUDNESS_RANGE:11.0} + music-ducking-threshold: ${VIDEO_EDITING_MUSIC_DUCKING_THRESHOLD:0.045} + music-ducking-ratio: ${VIDEO_EDITING_MUSIC_DUCKING_RATIO:8.0} + music-ducking-attack-ms: ${VIDEO_EDITING_MUSIC_DUCKING_ATTACK_MS:20} + music-ducking-release-ms: ${VIDEO_EDITING_MUSIC_DUCKING_RELEASE_MS:250} assets: music-folder: ${VIDEO_EDITING_ASSETS_MUSIC_FOLDER:./input/highlights/assets/music} sfx-folder: ${VIDEO_EDITING_ASSETS_SFX_FOLDER:./input/highlights/assets/sfx} diff --git a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java index 2ecce85..4bc0eec 100644 --- a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java +++ b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java @@ -34,6 +34,13 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().getVideoBitrate()).isEqualTo("12000k"); assertThat(properties.getEditing().getAudioBitrate()).isEqualTo("192k"); assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("noop"); + assertThat(properties.getEditing().getLoudnessTargetI()).isEqualTo(-16.0); + assertThat(properties.getEditing().getLoudnessTruePeak()).isEqualTo(-1.5); + assertThat(properties.getEditing().getLoudnessRange()).isEqualTo(11.0); + assertThat(properties.getEditing().getMusicDuckingThreshold()).isEqualTo(0.045); + assertThat(properties.getEditing().getMusicDuckingRatio()).isEqualTo(8.0); + assertThat(properties.getEditing().getMusicDuckingAttackMs()).isEqualTo(20); + assertThat(properties.getEditing().getMusicDuckingReleaseMs()).isEqualTo(250); assertThat(properties.getEditing().getAssets().getMusicFolder()) .isEqualTo("./input/highlights/assets/music"); assertThat(properties.getEditing().getAssets().getSfxFolder()).isEqualTo("./input/highlights/assets/sfx"); @@ -57,6 +64,13 @@ class VideoClippingPropertiesTest { "video-clipping.editing.project-directory=/tmp/edit-projects", "video-clipping.editing.thumbnail-count-per-clip=7", "video-clipping.editing.voiceover-provider=remote", + "video-clipping.editing.loudness-target-i=-14", + "video-clipping.editing.loudness-true-peak=-2", + "video-clipping.editing.loudness-range=9", + "video-clipping.editing.music-ducking-threshold=0.08", + "video-clipping.editing.music-ducking-ratio=10", + "video-clipping.editing.music-ducking-attack-ms=15", + "video-clipping.editing.music-ducking-release-ms=400", "video-clipping.editing.assets.music-folder=/tmp/music", "video-clipping.editing.assets.sfx-folder=/tmp/sfx", "video-clipping.editing.assets.fonts-folder=/tmp/fonts", @@ -74,6 +88,13 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().getProjectDirectory()).isEqualTo("/tmp/edit-projects"); assertThat(properties.getEditing().getThumbnailCountPerClip()).isEqualTo(7); assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("remote"); + assertThat(properties.getEditing().getLoudnessTargetI()).isEqualTo(-14.0); + assertThat(properties.getEditing().getLoudnessTruePeak()).isEqualTo(-2.0); + assertThat(properties.getEditing().getLoudnessRange()).isEqualTo(9.0); + assertThat(properties.getEditing().getMusicDuckingThreshold()).isEqualTo(0.08); + assertThat(properties.getEditing().getMusicDuckingRatio()).isEqualTo(10.0); + assertThat(properties.getEditing().getMusicDuckingAttackMs()).isEqualTo(15); + assertThat(properties.getEditing().getMusicDuckingReleaseMs()).isEqualTo(400); assertThat(properties.getEditing().getAssets().getMusicFolder()).isEqualTo("/tmp/music"); assertThat(properties.getEditing().getAssets().getSfxFolder()).isEqualTo("/tmp/sfx"); assertThat(properties.getEditing().getAssets().getFontsFolder()).isEqualTo("/tmp/fonts"); diff --git a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java index 3663328..c8c3e6b 100644 --- a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java +++ b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java @@ -51,11 +51,50 @@ class FfmpegEditRendererTest { Path.of("voiceover.wav"), Path.of("final.mp4")); assertThat(command).containsSubsequence("-i", "timeline.mp4", "-i", "music.wav", "-i", "voiceover.wav"); - assertThat(command).contains("[1:a]volume=0.25[music];[2:a]volume=1.0[voice];" - + "[0:a][music][voice]amix=inputs=3:duration=first:dropout_transition=2[a]"); + assertThat(command).contains("[1:a]volume=0.25[music_raw];[2:a]volume=1.0[voice];" + + "[music_raw][voice]sidechaincompress=threshold=0.045:ratio=8.0:attack=20:release=250[music];" + + "[0:a][music][voice]amix=inputs=3:duration=first:dropout_transition=2," + + "loudnorm=I=-16.0:TP=-1.5:LRA=11.0[a]"); assertThat(command).containsSubsequence("-c:v", "copy", "-c:a", "aac"); } + @Test + void buildsMusicOnlyMixWithoutDucking() { + VideoClippingProperties properties = new VideoClippingProperties(); + FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class), + mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null); + + var command = renderer.audioMixCommand(Path.of("timeline.mp4"), Path.of("music.wav"), null, + Path.of("final.mp4")); + + assertThat(command).contains("[1:a]volume=0.25[music_raw];[music_raw]anull[music];" + + "[0:a][music]amix=inputs=2:duration=first:dropout_transition=2," + + "loudnorm=I=-16.0:TP=-1.5:LRA=11.0[a]"); + assertThat(command).noneMatch(argument -> argument.contains("sidechaincompress")); + } + + @Test + void usesConfiguredAudioMasteringValues() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setLoudnessTargetI(-14); + properties.getEditing().setLoudnessTruePeak(-2); + properties.getEditing().setLoudnessRange(9); + properties.getEditing().setMusicDuckingThreshold(0.08); + properties.getEditing().setMusicDuckingRatio(10); + properties.getEditing().setMusicDuckingAttackMs(15); + properties.getEditing().setMusicDuckingReleaseMs(400); + FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class), + mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null); + + var command = renderer.audioMixCommand(Path.of("timeline.mp4"), Path.of("music.wav"), + Path.of("voiceover.wav"), Path.of("final.mp4")); + + assertThat(command).anySatisfy(argument -> assertThat(argument) + .contains("[music_raw][voice]sidechaincompress=threshold=0.08:ratio=10.0:" + + "attack=15:release=400[music];") + .contains("loudnorm=I=-14.0:TP=-2.0:LRA=9.0[a]")); + } + @Test void appliesFadeAndPlaybackTreatmentsToSegments() { VideoClippingProperties properties = new VideoClippingProperties(); @@ -100,7 +139,8 @@ class FfmpegEditRendererTest { assertThat(command).containsSubsequence("-i", "timeline.mp4", "-i", "engine-rev.wav"); assertThat(command).contains("[1:a]atrim=duration=1.5,volume=-3.0dB,adelay=1500|1500[sfx0];" - + "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2[a]"); + + "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2," + + "loudnorm=I=-16.0:TP=-1.5:LRA=11.0[a]"); } @Test