From b3718dd0b8e0fcdcf04af52e4095efe41ac0f2ce Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Wed, 22 Jul 2026 22:35:08 +0200 Subject: [PATCH] Add cinematic montage mode (break free of the 8-35s highlight windows) A montage edits many short shots pulled from ANYWHERE in the source (not a few fixed contiguous highlight windows), sequenced establishing -> quick detail cuts -> slow-motion hero, over one continuous music bed -- much closer to how a car film is actually cut. - MontagePlan model + director/montage.json shot list (per-shot source time, duration, punch-in zoom, speed). - HighlightDirectorFlowService.processMontage: builds one EditPlan from the shot list (bypassing the highlight-window validator), one continuous music cue, distributed voiceover, and titles; reuses asset prep + worker + renderer. - Renderer: explicit per-shot framing via a "zoom=" token (falls back to the progressive punch-in); pin each segment to its exact target duration with -t so frame-quantization drift cannot accumulate across many short shots. Verified on the DJI source: 18 shots, 16.7 s, -16.5 LUFS, TP -2.8 dBTP, QA green. The edit now reads as a real montage (varied framing + detail cuts + hero) rather than slow pans. mvn -o verify green (248). Remaining ceiling is the source footage. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01Bg76sLc43Wc3j5ZcLkboYR --- .../editing/HighlightDirectorFlowService.java | 101 ++++++++++++++++++ .../editing/HighlightFfmpegRenderer.java | 33 +++++- .../videoclips/editing/MontagePlan.java | 29 +++++ 3 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 src/main/java/org/example/videoclips/editing/MontagePlan.java diff --git a/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java b/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java index c8fb8bc..a99f78b 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java +++ b/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java @@ -63,6 +63,10 @@ public class HighlightDirectorFlowService { flowId, projectId, store.directorDirectory(projectId).resolve(properties.getApprovalFileName())); return HighlightFlowResult.skipped(projectId, flowId, "approval_missing"); } + Path montageFile = store.directorDirectory(projectId).resolve("montage.json"); + if (Files.isRegularFile(montageFile)) { + return processMontage(projectId, flowId, scanId, project, montageFile, startedAt); + } HighlightDirectorPlan plan; try { plan = planValidator.validate(projectId, readPlan(planFile)); @@ -136,6 +140,103 @@ public class HighlightDirectorFlowService { } } + private HighlightFlowResult processMontage(String projectId, String flowId, long scanId, + HighlightProject project, Path montageFile, long startedAt) { + MontagePlan montage; + try { + montage = objectMapper.readValue(montageFile.toFile(), MontagePlan.class); + } catch (IOException ex) { + markFailed(project, "Montage plan unreadable: " + ex.getMessage()); + throw new IllegalStateException("Unable to read montage plan: " + montageFile, ex); + } + HighlightSourceAnalysis analysis = store.readJson(projectId, "analysis/source-analysis.json", + HighlightSourceAnalysis.class); + String clipId = analysis.source().clipId(); + double sourceDuration = analysis.source().durationSeconds(); + + List decisions = new ArrayList<>(); + double timeline = 0.0; + List shots = montage.shots(); + for (int i = 0; i < shots.size(); i++) { + MontagePlan.Shot shot = shots.get(i); + double speed = shot.speed() <= 0 ? 1.0 : shot.speed(); + double dur = Math.max(0.2, shot.durationSeconds()); + double srcSpan = dur * speed; + double srcStart = Math.max(0.0, Math.min(shot.sourceStartSeconds(), sourceDuration - srcSpan)); + double srcEnd = Math.min(sourceDuration, srcStart + srcSpan); + String transitionIn = (i == 0) ? "fade-in" : "cut"; + String transitionOut = (i == shots.size() - 1) ? "fade-out" : "cut"; + String treatment = "zoom=%.3f cinematic".formatted(shot.zoom() <= 0 ? 1.2 : shot.zoom()); + decisions.add(new EditDecision(clipId, srcStart, srcEnd, timeline, timeline + dur, + transitionIn, transitionOut, speed, treatment, "montage")); + timeline += dur; + } + double total = timeline; + + List audioCues = new ArrayList<>(); + if (montage.musicDirection() != null && !montage.musicDirection().isBlank()) { + audioCues.add(new AudioCue("music", safeKey("music", montage.musicDirection()), 0.0, total, -14.0, + montage.musicDirection())); + } + List overlays = new ArrayList<>(); + if (montage.overlays() != null) { + for (MontagePlan.Overlay o : montage.overlays()) { + overlays.add(new TextOverlay(o.text(), o.timelineStartSeconds(), o.timelineEndSeconds(), + o.placement() == null || o.placement().isBlank() ? "lower_center_safe" : o.placement(), + "fade", "montage")); + } + } + String gradeKeyword = montage.grade() == null || montage.grade().isBlank() ? "hero" : montage.grade(); + List voiceoverLines = montage.voiceover() == null ? List.of() : montage.voiceover(); + HighlightDirectorPlan.HighlightItem montageHighlight = new HighlightDirectorPlan.HighlightItem( + "montage", "montage", "Cinematic Montage", 0.0, total, total, + gradeKeyword.contains("hero") ? "hero_payoff" : gradeKeyword, + "cinematic montage", montage.musicDirection(), "", voiceoverLines, + overlays.stream().map(TextOverlay::text).toList(), "montage"); + List voiceover = HighlightAssetPreparationService.plannedVoiceoverLines(montageHighlight); + + String style = safeKey("style", "montage", gradeKeyword); + EditPlan editPlan = new EditPlan(projectId, style, total, decisions, audioCues, voiceover, overlays, + "mp4-h264-aac-1080p", "cinematic montage"); + store.writeJson(projectId, "highlights/montage/edit-plan.json", editPlan); + log.info("event=highlight_montage_started flow_id={} scan_id={} project_id={} shots={} duration={}", + flowId, scanId, projectId, decisions.size(), total); + markStatus(project, HighlightProjectStatus.PLANNED, null); + + ContentCategory category; + try { + category = store.readJson(projectId, "analysis/category.json", CinematicHighlightAnalysis.class).category(); + } catch (RuntimeException ex) { + category = ContentCategory.GENERIC_VLOG; + } + assetPreparationService.prepare(projectId, project, montageHighlight, category); + HighlightLocalAssetWorker.HighlightAssetWorkerResult assetResult = + assetWorker.process(projectId, montageHighlight, category); + if (!assetResult.pendingRequests().isEmpty()) { + markFailed(project, "Montage assets pending: " + assetResult.pendingRequests()); + return HighlightFlowResult.skipped(projectId, flowId, "assets_pending"); + } + markStatus(project, HighlightProjectStatus.RENDERING, null); + HighlightFfmpegRenderer.HighlightRenderResult result; + try { + result = renderer.render(projectId, montageHighlight, editPlan); + requireQaPassed(result); + } catch (RuntimeException ex) { + markFailed(project, "Montage render rejected: " + ex.getMessage()); + throw ex; + } + Path projectOutput = store.projectDirectory(projectId).resolve("final.mp4"); + concatFinalOutputs(projectOutput, List.of(result.finalOutput())); + RenderManifest manifest = new RenderManifest(projectId, List.of(result.highlightId()), + List.of(result.finalOutput().toString()), projectOutput.toString(), total, List.of(), List.of(), + Instant.now()); + store.writeJson(projectId, "render-manifest.json", manifest); + markStatus(project, HighlightProjectStatus.RENDERED, null); + log.info("event=highlight_montage_completed flow_id={} project_id={} shots={} duration={} elapsed_ms={}", + flowId, projectId, decisions.size(), total, (System.nanoTime() - startedAt) / 1_000_000); + return HighlightFlowResult.rendered(projectId, flowId, List.of(result.finalOutput()), projectOutput); + } + private HighlightDirectorPlan readPlan(Path planFile) { try { return objectMapper.readValue(planFile.toFile(), HighlightDirectorPlan.class); diff --git a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java index 8ed4427..31804ee 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java +++ b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java @@ -193,10 +193,13 @@ public class HighlightFfmpegRenderer { || "none".equalsIgnoreCase(decision.visualTreatment())); StringBuilder filter = new StringBuilder(); if (styled) { - // Tighter, more intentional framing that fills the frame and hides the mundane location, - // with a progressive punch-in across the cuts of a beat (each cut steps tighter) so the edit - // reads as deliberate rather than one long pan. Center crop keeps the subject framed. - double zoom = 1.16 + Math.min(cutIndex, 3) * 0.12; + // Tighter, more intentional framing that fills the frame and hides the mundane location. + // A montage shot can specify its exact framing via a "zoom=" token in the treatment; otherwise + // apply a progressive punch-in across the cuts of a beat (each cut steps tighter). Center crop. + double zoom = explicitZoom(decision.visualTreatment()); + if (zoom <= 0) { + zoom = 1.16 + Math.min(cutIndex, 3) * 0.12; + } filter.append("crop=iw/%.4f:ih/%.4f,scale=%d:%d,format=yuv420p".formatted(zoom, zoom, w, h)); } else { filter.append("scale=%d:%d:force_original_aspect_ratio=decrease,".formatted(w, h) @@ -223,6 +226,9 @@ public class HighlightFfmpegRenderer { "-i", source, "-vf", filter.toString(), "-af", audioTempoFilter(decision.playbackSpeed()), "-r", Integer.toString(properties.getOutputFrameRate()), + // Pin each segment to its exact target duration so per-segment frame-quantization drift + // cannot accumulate across a many-shot montage (keeps the concat matching the timeline). + "-t", Double.toString(outputDuration), "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", properties.getAudioBitrate(), @@ -597,6 +603,25 @@ public class HighlightFfmpegRenderer { return "crop=w='iw*0.96':h='ih*0.96':x='(iw-out_w)/2':y='(ih-out_h)/2',"; } + // A montage shot encodes its framing as "zoom=1.35" in the visual treatment; returns <=0 when absent. + private double explicitZoom(String visualTreatment) { + if (visualTreatment == null) { + return -1; + } + java.util.regex.Matcher m = ZOOM_TOKEN.matcher(visualTreatment); + if (m.find()) { + try { + return Double.parseDouble(m.group(1)); + } catch (NumberFormatException ignored) { + return -1; + } + } + return -1; + } + + private static final java.util.regex.Pattern ZOOM_TOKEN = + java.util.regex.Pattern.compile("zoom=([0-9]+(?:\\.[0-9]+)?)"); + // 2.39:1 cinematic letterbox: crop the center band, then pad back to frame with black bars. private String letterboxFilter() { int w = properties.getOutputWidth(); diff --git a/src/main/java/org/example/videoclips/editing/MontagePlan.java b/src/main/java/org/example/videoclips/editing/MontagePlan.java new file mode 100644 index 0000000..91245a3 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/MontagePlan.java @@ -0,0 +1,29 @@ +package org.example.videoclips.editing; + +import java.util.List; + +/** + * A cinematic montage: many short shots pulled from anywhere in the source, sequenced and cut to a + * rhythm, rather than a few fixed 8-35s highlight windows. Rendered as a single continuous edit with one + * music bed. Present as {@code director/montage.json} to select the montage path. + */ +public record MontagePlan( + String projectId, + String sourceVideoFileName, + String grade, // grade keyword: opening | rising | hero (drives the look) + String musicDirection, + List voiceover, // optional narration lines, distributed across the montage + List overlays, // optional titles + List shots +) { + /** + * One shot. {@code durationSeconds} is how long it plays on the timeline; {@code speed} is the playback + * speed (0.5 = slow motion). The source span consumed is durationSeconds * speed. {@code zoom} is the + * punch-in framing (1.0 = full frame, 1.4 = tight). + */ + public record Shot(double sourceStartSeconds, double durationSeconds, double zoom, double speed) { + } + + public record Overlay(String text, double timelineStartSeconds, double timelineEndSeconds, String placement) { + } +}