jsl
/
video_editing_poc
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add cinematic audio mastering filters

This commit is contained in:
JSLMPR 2026-07-11 01:16:14 +02:00
parent 43fccb7dc6
commit bb23ac80e1
6 changed files with 167 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/production-cinematic-highlight-editing-plan.md
View File

@ -308,7 +308,7 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can
- [x] Milestone 11: Implement asset provider interfaces for voiceover, music, SFX, fonts, and LUTs.
- [x] Milestone 12: Add a local licensed asset library with category tags and deterministic asset selection.
- [ ] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests.
- [ ] Milestone 13 progress: Renderer now applies timed text overlays and records resolved/planned edit assets in the render manifest. Dynamic crop keyframes, audio ducking, and loudness normalization remain.
- [ ] Milestone 13 progress: Renderer now applies timed text overlays, records resolved/planned edit assets in the render manifest, ducks music under voiceover, and normalizes final mix loudness with configurable mastering values. Dynamic crop keyframes remain.
- [ ] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters.
- [ ] Milestone 15: Add an operator runbook for placing a video, running the service locally, choosing a model, reviewing the director plan, and finding final clips.
- [ ] Milestone 16: Add integration tests with small fixture videos for family, food, car, and generic content.

72
src/main/java/org/example/videoclips/config/VideoClippingProperties.java
View File

@ -522,6 +522,22 @@ public class VideoClippingProperties {
private String voiceoverProvider = "noop";
private double loudnessTargetI = -16.0;
private double loudnessTruePeak = -1.5;
private double loudnessRange = 11.0;
private double musicDuckingThreshold = 0.045;
private double musicDuckingRatio = 8.0;
@Min(1)
private int musicDuckingAttackMs = 20;
@Min(1)
private int musicDuckingReleaseMs = 250;
private final Assets assets = new Assets();
private final LocalDirector localDirector = new LocalDirector();
@ -654,6 +670,62 @@ public class VideoClippingProperties {
this.voiceoverProvider = voiceoverProvider;
}
public double getLoudnessTargetI() {
return loudnessTargetI;
}
public void setLoudnessTargetI(double loudnessTargetI) {
this.loudnessTargetI = loudnessTargetI;
}
public double getLoudnessTruePeak() {
return loudnessTruePeak;
}
public void setLoudnessTruePeak(double loudnessTruePeak) {
this.loudnessTruePeak = loudnessTruePeak;
}
public double getLoudnessRange() {
return loudnessRange;
}
public void setLoudnessRange(double loudnessRange) {
this.loudnessRange = loudnessRange;
}
public double getMusicDuckingThreshold() {
return musicDuckingThreshold;
}
public void setMusicDuckingThreshold(double musicDuckingThreshold) {
this.musicDuckingThreshold = musicDuckingThreshold;
}
public double getMusicDuckingRatio() {
return musicDuckingRatio;
}
public void setMusicDuckingRatio(double musicDuckingRatio) {
this.musicDuckingRatio = musicDuckingRatio;
}
public int getMusicDuckingAttackMs() {
return musicDuckingAttackMs;
}
public void setMusicDuckingAttackMs(int musicDuckingAttackMs) {
this.musicDuckingAttackMs = musicDuckingAttackMs;
}
public int getMusicDuckingReleaseMs() {
return musicDuckingReleaseMs;
}
public void setMusicDuckingReleaseMs(int musicDuckingReleaseMs) {
this.musicDuckingReleaseMs = musicDuckingReleaseMs;
}
public Assets getAssets() {
return assets;
}

27
src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java
View File

@ -265,18 +265,33 @@ public class FfmpegEditRenderer implements EditRenderer {
List<String> labels = new ArrayList<>(List.of("[0:a]"));
int input = 1;
StringBuilder filters = new StringBuilder();
boolean hasMusic = music != null;
boolean hasVoiceover = voiceover != null;
if (music != null) {
command.addAll(List.of("-i", music.toString()));
filters.append("[").append(input).append(":a]volume=0.25[music];");
labels.add("[music]");
filters.append("[").append(input).append(":a]volume=0.25[music_raw];");
input++;
}
if (voiceover != null) {
command.addAll(List.of("-i", voiceover.toString()));
filters.append("[").append(input).append(":a]volume=1.0[voice];");
labels.add("[voice]");
input++;
}
if (hasMusic && hasVoiceover) {
filters.append("[music_raw][voice]sidechaincompress=threshold=")
.append(properties.getMusicDuckingThreshold())
.append(":ratio=").append(properties.getMusicDuckingRatio())
.append(":attack=").append(properties.getMusicDuckingAttackMs())
.append(":release=").append(properties.getMusicDuckingReleaseMs())
.append("[music];");
labels.add("[music]");
labels.add("[voice]");
} else if (hasMusic) {
filters.append("[music_raw]anull[music];");
labels.add("[music]");
} else if (hasVoiceover) {
labels.add("[voice]");
}
for (int index = 0; index < soundEffects.size(); index++) {
SfxInput soundEffect = soundEffects.get(index);
AudioCue cue = soundEffect.cue();
@ -291,7 +306,11 @@ public class FfmpegEditRenderer implements EditRenderer {
input++;
}
filters.append(String.join("", labels)).append("amix=inputs=").append(labels.size())
.append(":duration=first:dropout_transition=2[a]");
.append(":duration=first:dropout_transition=2,loudnorm=I=")
.append(properties.getLoudnessTargetI())
.append(":TP=").append(properties.getLoudnessTruePeak())
.append(":LRA=").append(properties.getLoudnessRange())
.append("[a]");
command.addAll(List.of("-filter_complex", filters.toString(), "-map", "0:v:0", "-map", "[a]",
"-c:v", "copy", "-c:a", "aac", "-b:a", properties.getAudioBitrate(),
"-ar", Integer.toString(properties.getAudioSampleRate()), output.toString()));

7
src/main/resources/application.yml
View File

@ -30,6 +30,13 @@ video-clipping:
video-bitrate: ${VIDEO_EDITING_VIDEO_BITRATE:12000k}
audio-bitrate: ${VIDEO_EDITING_AUDIO_BITRATE:192k}
voiceover-provider: ${VIDEO_EDITING_VOICEOVER_PROVIDER:noop}
loudness-target-i: ${VIDEO_EDITING_LOUDNESS_TARGET_I:-16.0}
loudness-true-peak: ${VIDEO_EDITING_LOUDNESS_TRUE_PEAK:-1.5}
loudness-range: ${VIDEO_EDITING_LOUDNESS_RANGE:11.0}
music-ducking-threshold: ${VIDEO_EDITING_MUSIC_DUCKING_THRESHOLD:0.045}
music-ducking-ratio: ${VIDEO_EDITING_MUSIC_DUCKING_RATIO:8.0}
music-ducking-attack-ms: ${VIDEO_EDITING_MUSIC_DUCKING_ATTACK_MS:20}
music-ducking-release-ms: ${VIDEO_EDITING_MUSIC_DUCKING_RELEASE_MS:250}
assets:
music-folder: ${VIDEO_EDITING_ASSETS_MUSIC_FOLDER:./input/highlights/assets/music}
sfx-folder: ${VIDEO_EDITING_ASSETS_SFX_FOLDER:./input/highlights/assets/sfx}

21
src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java
View File

@ -34,6 +34,13 @@ class VideoClippingPropertiesTest {
assertThat(properties.getEditing().getVideoBitrate()).isEqualTo("12000k");
assertThat(properties.getEditing().getAudioBitrate()).isEqualTo("192k");
assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("noop");
assertThat(properties.getEditing().getLoudnessTargetI()).isEqualTo(-16.0);
assertThat(properties.getEditing().getLoudnessTruePeak()).isEqualTo(-1.5);
assertThat(properties.getEditing().getLoudnessRange()).isEqualTo(11.0);
assertThat(properties.getEditing().getMusicDuckingThreshold()).isEqualTo(0.045);
assertThat(properties.getEditing().getMusicDuckingRatio()).isEqualTo(8.0);
assertThat(properties.getEditing().getMusicDuckingAttackMs()).isEqualTo(20);
assertThat(properties.getEditing().getMusicDuckingReleaseMs()).isEqualTo(250);
assertThat(properties.getEditing().getAssets().getMusicFolder())
.isEqualTo("./input/highlights/assets/music");
assertThat(properties.getEditing().getAssets().getSfxFolder()).isEqualTo("./input/highlights/assets/sfx");
@ -57,6 +64,13 @@ class VideoClippingPropertiesTest {
"video-clipping.editing.project-directory=/tmp/edit-projects",
"video-clipping.editing.thumbnail-count-per-clip=7",
"video-clipping.editing.voiceover-provider=remote",
"video-clipping.editing.loudness-target-i=-14",
"video-clipping.editing.loudness-true-peak=-2",
"video-clipping.editing.loudness-range=9",
"video-clipping.editing.music-ducking-threshold=0.08",
"video-clipping.editing.music-ducking-ratio=10",
"video-clipping.editing.music-ducking-attack-ms=15",
"video-clipping.editing.music-ducking-release-ms=400",
"video-clipping.editing.assets.music-folder=/tmp/music",
"video-clipping.editing.assets.sfx-folder=/tmp/sfx",
"video-clipping.editing.assets.fonts-folder=/tmp/fonts",
@ -74,6 +88,13 @@ class VideoClippingPropertiesTest {
assertThat(properties.getEditing().getProjectDirectory()).isEqualTo("/tmp/edit-projects");
assertThat(properties.getEditing().getThumbnailCountPerClip()).isEqualTo(7);
assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("remote");
assertThat(properties.getEditing().getLoudnessTargetI()).isEqualTo(-14.0);
assertThat(properties.getEditing().getLoudnessTruePeak()).isEqualTo(-2.0);
assertThat(properties.getEditing().getLoudnessRange()).isEqualTo(9.0);
assertThat(properties.getEditing().getMusicDuckingThreshold()).isEqualTo(0.08);
assertThat(properties.getEditing().getMusicDuckingRatio()).isEqualTo(10.0);
assertThat(properties.getEditing().getMusicDuckingAttackMs()).isEqualTo(15);
assertThat(properties.getEditing().getMusicDuckingReleaseMs()).isEqualTo(400);
assertThat(properties.getEditing().getAssets().getMusicFolder()).isEqualTo("/tmp/music");
assertThat(properties.getEditing().getAssets().getSfxFolder()).isEqualTo("/tmp/sfx");
assertThat(properties.getEditing().getAssets().getFontsFolder()).isEqualTo("/tmp/fonts");

46
src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java
View File

@ -51,11 +51,50 @@ class FfmpegEditRendererTest {
Path.of("voiceover.wav"), Path.of("final.mp4"));
assertThat(command).containsSubsequence("-i", "timeline.mp4", "-i", "music.wav", "-i", "voiceover.wav");
assertThat(command).contains("[1:a]volume=0.25[music];[2:a]volume=1.0[voice];"
+ "[0:a][music][voice]amix=inputs=3:duration=first:dropout_transition=2[a]");
assertThat(command).contains("[1:a]volume=0.25[music_raw];[2:a]volume=1.0[voice];"
+ "[music_raw][voice]sidechaincompress=threshold=0.045:ratio=8.0:attack=20:release=250[music];"
+ "[0:a][music][voice]amix=inputs=3:duration=first:dropout_transition=2,"
+ "loudnorm=I=-16.0:TP=-1.5:LRA=11.0[a]");
assertThat(command).containsSubsequence("-c:v", "copy", "-c:a", "aac");
}
@Test
void buildsMusicOnlyMixWithoutDucking() {
VideoClippingProperties properties = new VideoClippingProperties();
FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class),
mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null);
var command = renderer.audioMixCommand(Path.of("timeline.mp4"), Path.of("music.wav"), null,
Path.of("final.mp4"));
assertThat(command).contains("[1:a]volume=0.25[music_raw];[music_raw]anull[music];"
+ "[0:a][music]amix=inputs=2:duration=first:dropout_transition=2,"
+ "loudnorm=I=-16.0:TP=-1.5:LRA=11.0[a]");
assertThat(command).noneMatch(argument -> argument.contains("sidechaincompress"));
}
@Test
void usesConfiguredAudioMasteringValues() {
VideoClippingProperties properties = new VideoClippingProperties();
properties.getEditing().setLoudnessTargetI(-14);
properties.getEditing().setLoudnessTruePeak(-2);
properties.getEditing().setLoudnessRange(9);
properties.getEditing().setMusicDuckingThreshold(0.08);
properties.getEditing().setMusicDuckingRatio(10);
properties.getEditing().setMusicDuckingAttackMs(15);
properties.getEditing().setMusicDuckingReleaseMs(400);
FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class),
mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null);
var command = renderer.audioMixCommand(Path.of("timeline.mp4"), Path.of("music.wav"),
Path.of("voiceover.wav"), Path.of("final.mp4"));
assertThat(command).anySatisfy(argument -> assertThat(argument)
.contains("[music_raw][voice]sidechaincompress=threshold=0.08:ratio=10.0:"
+ "attack=15:release=400[music];")
.contains("loudnorm=I=-14.0:TP=-2.0:LRA=9.0[a]"));
}
@Test
void appliesFadeAndPlaybackTreatmentsToSegments() {
VideoClippingProperties properties = new VideoClippingProperties();
@ -100,7 +139,8 @@ class FfmpegEditRendererTest {
assertThat(command).containsSubsequence("-i", "timeline.mp4", "-i", "engine-rev.wav");
assertThat(command).contains("[1:a]atrim=duration=1.5,volume=-3.0dB,adelay=1500|1500[sfx0];"
+ "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2[a]");
+ "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2,"
+ "loudnorm=I=-16.0:TP=-1.5:LRA=11.0[a]");
}
@Test