From a2a7d7d7360a789c97dee4cf41224e946a10520b Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Fri, 24 Jul 2026 09:37:43 +0200 Subject: [PATCH] R8: swell the music into the payoff MusicGen's internal structure is uncontrolled, so the mix applies a deterministic swell envelope to the score (volume='min(1,0.5+0.5*t/peak)':eval=frame): the music amplitude rises from 0.5x to full over the run-up to the payoff, then holds. The peak is the payoff (slow-motion) beat's timeline midpoint on the crossfade- compressed timeline -- generic, driven only by the plan, no content assumptions. No-ops when there is no slow-mo beat. Filter string unit-tested; verified in a real render (swell peaks at 6.83s, output -16.1 LUFS). Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01MPuJXQyAeWpFcTtcnxo1UN --- docs/cinematic-quality-rules.md | 10 +++++-- .../editing/HighlightFfmpegRenderer.java | 29 +++++++++++++++++-- .../editing/HighlightFfmpegRendererTest.java | 14 +++++++++ 3 files changed, 47 insertions(+), 6 deletions(-) diff --git a/docs/cinematic-quality-rules.md b/docs/cinematic-quality-rules.md index 522031c..664e83b 100644 --- a/docs/cinematic-quality-rules.md +++ b/docs/cinematic-quality-rules.md @@ -57,9 +57,13 @@ Legend: ✅ implemented · ⏳ planned (see `cinematic-highlight-poc-plan.md` P5 - **Ties to Tier 2 (R9):** the vision director produces the caption *text* ("STRIKE"); R7 makes it *land*. The overlay is placed on the payoff beat, so it's timed to the musical/edit accent. Verified on bowling. -## R8 — Music sync: climax on the payoff ⏳ (P5.13) -- **Measure:** the payoff beat's timeline position. -- **Adapt:** prompt/trim the score so its peak lands on the payoff, not wherever the generator happened to put it. +## R8 — Music dynamics: build to the payoff ✅ +- **Rule:** because MusicGen's internal structure is uncontrolled, the mix applies a deterministic swell + envelope to the score (`volume='min(1,0.5+0.5*t/peak)':eval=frame`) so the music amplitude rises from 0.5x to + full over the run-up to the payoff, then holds. Driven only by the payoff beat's (crossfade-compressed) + timeline position — generic for any source. In `audioMixCommand`; verified on bowling (peak at 6.83s). +- **Still open:** aligning MusicGen's *own* melodic climax (vs a volume swell) needs a controllable music + model or a produced track (P5.3). ## R9 — Show the action, not just the reaction — Tier 1 ✅ / Tier 2 ⏳ - **Tier 1 (measurement director, DONE):** `HighlightMontageDirector` measures a per-window motion curve diff --git a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java index 2d33f29..b44d252 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java +++ b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java @@ -147,9 +147,18 @@ public class HighlightFfmpegRenderer { .filter(cue -> "music".equals(cue.type())) .findFirst() .orElse(null); + // Where the score should peak: the payoff (slow-motion) beat's timeline midpoint, on the + // crossfade-compressed timeline. Falls back to 0 (no swell) when there is no slow-mo beat. + double payoffPeakSeconds = plan.decisions().stream() + .filter(d -> d.playbackSpeed() < 0.85) + .mapToDouble(d -> (d.timelineStartSeconds() + d.timelineEndSeconds()) / 2.0) + .findFirst().orElse(0.0); + if (payoffPeakSeconds > 0) { + payoffPeakSeconds = Math.max(0.3, payoffPeakSeconds - crossfadeShrink); + } if (musicCue != null || !plan.voiceover().isEmpty() || !sfx.isEmpty()) { run(audioMixCommand(postTimeline, musicCue == null ? null : music, musicCue, - voiceovers, sfx, sourceAudioPresent, output), commands); + voiceovers, sfx, sourceAudioPresent, payoffPeakSeconds, output), commands); } else { copy(postTimeline, output); } @@ -559,6 +568,12 @@ public class HighlightFfmpegRenderer { List audioMixCommand(Path timeline, Path music, AudioCue musicCue, List voiceovers, List soundEffects, boolean sourceAudioPresent, Path output) { + return audioMixCommand(timeline, music, musicCue, voiceovers, soundEffects, sourceAudioPresent, 0.0, output); + } + + List audioMixCommand(Path timeline, Path music, AudioCue musicCue, List voiceovers, + List soundEffects, boolean sourceAudioPresent, double musicPeakSeconds, + Path output) { List command = new ArrayList<>(List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", timeline.toString())); List labels = new ArrayList<>(); @@ -584,8 +599,16 @@ public class HighlightFfmpegRenderer { double duration = musicCue.timelineEndSeconds() - musicCue.timelineStartSeconds(); long delayMillis = Math.round(musicCue.timelineStartSeconds() * 1000); filters.append("atrim=duration=").append(duration) - .append(",asetpts=PTS-STARTPTS,volume=").append(musicCue.gainDb()).append("dB,adelay=") - .append(delayMillis).append("|").append(delayMillis).append("[music_raw];"); + .append(",asetpts=PTS-STARTPTS,volume=").append(musicCue.gainDb()).append("dB"); + if (musicPeakSeconds > 0.3) { + // R8: swell the score so it builds into the payoff, regardless of MusicGen's own (random) + // internal structure. Amplitude rises from 0.5x to 1.0x over the run-up to the payoff, + // then holds. Generic — driven only by the payoff's timeline position. + double peak = Math.max(0.3, musicPeakSeconds - musicCue.timelineStartSeconds()); + filters.append(",volume='min(1\\,0.5+0.5*t/").append(fixed(peak)).append(")':eval=frame"); + } + filters.append(",adelay=").append(delayMillis).append("|").append(delayMillis) + .append("[music_raw];"); } else { filters.append("volume=-12dB[music_raw];"); } diff --git a/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java b/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java index 2d1e7cb..037e62f 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java @@ -205,6 +205,20 @@ class HighlightFfmpegRendererTest { assertThat(portrait).contains("scale=1080:1920").doesNotContain("crop=1080:"); // no letterbox } + @Test + void swellsTheMusicTowardThePayoffWhenAPeakIsGiven() { + HighlightFfmpegRenderer renderer = renderer(); + AudioCue music = new AudioCue("music", "score", 0, 10, -9, "bed"); + + String withSwell = String.join(" ", renderer.audioMixCommand(Path.of("t.mp4"), Path.of("m.wav"), music, + List.of(), List.of(), true, 6.0, Path.of("f.mp4"))); + String noSwell = String.join(" ", renderer.audioMixCommand(Path.of("t.mp4"), Path.of("m.wav"), music, + List.of(), List.of(), true, 0.0, Path.of("f.mp4"))); + + assertThat(withSwell).contains("volume='min(1\\,0.5+0.5*t/6.000)':eval=frame"); + assertThat(noSwell).doesNotContain("eval=frame"); + } + @Test void crossfadeChainOverlapsSegmentsAndShiftsOverlaysOntoTheCompressedTimeline() { HighlightFfmpegRenderer renderer = renderer();