From 30c832e9d5bd5fc5d96817dfbfe0f17994119bda Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Thu, 23 Jul 2026 23:09:06 +0200 Subject: [PATCH] Accurate loudness mastering: measure the finished file and correct to target Single-pass loudnorm in the mix is only ~+/-2 LUFS accurate, so the auto-rendered output could land quiet (e.g. -18.7 LUFS vs the -16 target). After the mix, probeIntegratedLoudness measures the file, loudnessGainDb computes the corrective gain, and masterLoudness applies it with a brickwall limiter for true peak. No-ops when already on target or when the measurement is implausible; handles MusicGen loudness variance. loudnessGainDb unit-tested. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01MPuJXQyAeWpFcTtcnxo1UN --- .claude/memory/ai-handover.md | 15 ++++- docs/cinematic-quality-rules.md | 4 ++ .../editing/HighlightFfmpegRenderer.java | 64 +++++++++++++++++++ .../editing/HighlightFfmpegRendererTest.java | 11 ++++ 4 files changed, 91 insertions(+), 3 deletions(-) diff --git a/.claude/memory/ai-handover.md b/.claude/memory/ai-handover.md index 7a5d89d..8e8f3cd 100644 --- a/.claude/memory/ai-handover.md +++ b/.claude/memory/ai-handover.md @@ -13,8 +13,8 @@ git log -8 --oneline --decorate git status --short # uncommitted work may be the real current state ``` -- **Recorded HEAD when this was written: `8b68116`**, plus uncommitted working-tree changes (portrait + - audio-mix + unbounded-duration fixes, see below). +- **Recorded HEAD when this was written: `1da0661`** ("Automatic two-tier highlight director + + source-adaptive rendering") — the session's work below is COMMITTED as of 2026-07-23. - If HEAD is newer than `8b68116`, or there are unfamiliar commits/uncommitted files, **this file is behind the repo**. The authoritative live state is: the git log, `git status`, and `docs/cinematic-highlight-poc-plan.md` (milestone log at its tail). Trust those over this file. @@ -31,7 +31,16 @@ earlier no-download stance; that reversal is real and current). To run source → final render, follow [[highlight-e2e-render-runbook]] — do not re-derive it. -## Fixes applied 2026-07-23 (whole service, not just PoC) — uncommitted in working tree +## Delivered 2026-07-23 (whole service, not just PoC) — committed at `1da0661` + +**Automatic two-tier director** (plans were hand-authored before; now auto-generated): +- Tier 1 `HighlightMontageDirector`: composes `director/montage.json` from measured motion (YDIF) + audio (RMS) + — setup, continuous action, slow-mo payoff on the audio climax, resolution button, camera-whip tail trimmed. +- Tier 2 `HighlightVisionDirector` + `tools/vision_caption.py`: local moondream2 (offline) captions the payoff + → semantic overlay ("STRIKE") + scene-informed music; fails soft. See [[local-model-runtime-intel-mac]]. +- Wired into the scheduler behind `auto-director-enabled` / `vision-director-enabled` (localpoc on). + +**Source-adaptive rendering fixes:** 1. **Portrait/orientation**: `FfmpegClipInspector` now reads rotation side-data/`rotate` tag and stores the EFFECTIVE (display) width/height. `HighlightFfmpegRenderer.outputGeometry()` renders portrait sources to a diff --git a/docs/cinematic-quality-rules.md b/docs/cinematic-quality-rules.md index 1e6f74e..3043609 100644 --- a/docs/cinematic-quality-rules.md +++ b/docs/cinematic-quality-rules.md @@ -23,6 +23,10 @@ Legend: ✅ implemented · ⏳ planned (see `cinematic-highlight-poc-plan.md` P5 ## R3 — Audio balance: the story audio must lead ✅ - **Rule:** the generated score is the bed and leads; source audio is ducked under it (−20 dB, −24 with narration); with narration, voice leads via side-chain ducking. No element is silently buried. +- **Accurate mastering:** single-pass `loudnorm` is only ~±2 LUFS accurate, so the finished file is measured + (`probeIntegratedLoudness`) and a corrective gain (`loudnessGainDb` → `masterLoudness`) is applied to hit the + target, with a brickwall limiter for true peak. No-ops when already on target (e.g. a render that lands at + −15.5 needs no fix; one at −18.7 is boosted). Handles MusicGen loudness variance. - **Verify:** integrated loudness ≈ −16 LUFS, true peak ≤ −1.5 dBTP; confirm the score is audible, not just present. ## R4 — Duration: length follows the story ✅ diff --git a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java index d88eebf..9a15f77 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java +++ b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java @@ -142,6 +142,7 @@ public class HighlightFfmpegRenderer { } else { copy(postTimeline, output); } + masterLoudness(output, commands); run(previewCommand(output, preview), commands); double duration = plan.decisions().isEmpty() ? 0 : plan.decisions().get(plan.decisions().size() - 1) .timelineEndSeconds(); @@ -382,6 +383,67 @@ public class HighlightFfmpegRenderer { } } + /** + * Accurate loudness mastering. Single-pass {@code loudnorm} in the mix is only ~+/-2 LUFS accurate, so the + * finished file is measured (integrated LUFS) and a corrective gain is applied to hit the configured + * target, with a brickwall limiter to keep true peak safe. No-ops when already on target or when the + * measurement is missing/implausible. + */ + private void masterLoudness(Path output, List> commands) { + if (!Files.isRegularFile(output)) { + return; + } + double gain = loudnessGainDb(probeIntegratedLoudness(output.toString()), properties.getLoudnessTargetI()); + if (gain == 0.0) { + return; + } + Path corrected = output.resolveSibling(output.getFileName().toString() + ".master.mp4"); + List command = List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", output.toString(), + "-c:v", "copy", + "-af", "volume=" + String.format(Locale.ROOT, "%.2f", gain) + "dB,alimiter=level=disabled:limit=0.72", + "-c:a", "aac", "-b:a", properties.getAudioBitrate(), + "-ar", Integer.toString(properties.getAudioSampleRate()), corrected.toString()); + run(command, commands); + try { + Files.move(corrected, output, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ex) { + throw new IllegalStateException("Unable to promote loudness-mastered output: " + output, ex); + } + } + + /** Corrective gain (dB) to move measured integrated loudness onto target; 0 when on-target or implausible. */ + static double loudnessGainDb(double measuredLufs, double targetLufs) { + if (Double.isNaN(measuredLufs)) { + return 0.0; + } + double delta = targetLufs - measuredLufs; + if (Math.abs(delta) < 0.75 || Math.abs(delta) > 12.0) { + return 0.0; // already close enough, or an implausible measurement -> leave it alone + } + return delta; + } + + double probeIntegratedLoudness(String path) { + List command = List.of(properties.getFfmpegBinary(), "-hide_banner", "-nostats", "-i", path, + "-filter_complex", "ebur128", "-f", "null", "-"); + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String out = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + process.waitFor(); + java.util.regex.Matcher matcher = LOUDNESS_TOKEN.matcher(out); + double value = Double.NaN; + while (matcher.find()) { + value = Double.parseDouble(matcher.group(1)); // last match is the summary integrated loudness + } + return value; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return Double.NaN; + } catch (IOException | RuntimeException ex) { + return Double.NaN; + } + } + List overlayCommand(Path input, List overlays, Path output) { return List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", input.toString(), "-vf", overlayFilter(overlays), @@ -783,6 +845,8 @@ public class HighlightFfmpegRenderer { java.util.regex.Pattern.compile("YAVG=([0-9]+(?:\\.[0-9]+)?)"); private static final java.util.regex.Pattern YDIF_TOKEN = java.util.regex.Pattern.compile("YDIF=([0-9]+(?:\\.[0-9]+)?)"); + private static final java.util.regex.Pattern LOUDNESS_TOKEN = + java.util.regex.Pattern.compile("I:\\s*(-?[0-9]+(?:\\.[0-9]+)?)\\s*LUFS"); // 2.39:1 cinematic letterbox: crop the center band, then pad back to frame with black bars. private String letterboxFilter() { diff --git a/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java b/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java index 84a55ed..2c18aeb 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightFfmpegRendererTest.java @@ -11,6 +11,7 @@ import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.Assertions.within; import static org.mockito.Mockito.mock; class HighlightFfmpegRendererTest { @@ -202,6 +203,16 @@ class HighlightFfmpegRendererTest { assertThat(portrait).contains("scale=1080:1920").doesNotContain("crop=1080:"); // no letterbox } + @Test + void loudnessMasteringCorrectsTowardTargetOnlyWhenNeeded() { + // Too quiet -> positive boost; too loud -> negative cut; on-target and implausible -> no change. + assertThat(HighlightFfmpegRenderer.loudnessGainDb(-18.7, -16.0)).isCloseTo(2.7, within(0.001)); + assertThat(HighlightFfmpegRenderer.loudnessGainDb(-14.0, -16.0)).isCloseTo(-2.0, within(0.001)); + assertThat(HighlightFfmpegRenderer.loudnessGainDb(-16.3, -16.0)).isZero(); // within tolerance + assertThat(HighlightFfmpegRenderer.loudnessGainDb(-40.0, -16.0)).isZero(); // implausible -> skip + assertThat(HighlightFfmpegRenderer.loudnessGainDb(Double.NaN, -16.0)).isZero(); + } + @Test void exposureNormalizationAdaptsToMeasuredSourceBrightness() { HighlightFfmpegRenderer renderer = renderer();