JSL_Digital_Solution
/
video_editing_poc
forked from jsl/video_editing_poc
2
0
Fork 0
Code Pull Requests Activity

Add cinematic montage mode (break free of the 8-35s highlight windows) 

A montage edits many short shots pulled from ANYWHERE in the source (not a few
fixed contiguous highlight windows), sequenced establishing -> quick detail cuts ->
slow-motion hero, over one continuous music bed -- much closer to how a car film is
actually cut.

- MontagePlan model + director/montage.json shot list (per-shot source time,
  duration, punch-in zoom, speed).
- HighlightDirectorFlowService.processMontage: builds one EditPlan from the shot
  list (bypassing the highlight-window validator), one continuous music cue,
  distributed voiceover, and titles; reuses asset prep + worker + renderer.
- Renderer: explicit per-shot framing via a "zoom=" token (falls back to the
  progressive punch-in); pin each segment to its exact target duration with -t so
  frame-quantization drift cannot accumulate across many short shots.

Verified on the DJI source: 18 shots, 16.7 s, -16.5 LUFS, TP -2.8 dBTP, QA green.
The edit now reads as a real montage (varied framing + detail cuts + hero) rather
than slow pans. mvn -o verify green (248). Remaining ceiling is the source footage.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01Bg76sLc43Wc3j5ZcLkboYR
This commit is contained in:
JSLMPR 2026-07-22 22:35:08 +02:00
parent daa58a8c8d
commit b3718dd0b8
3 changed files with 159 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

101
src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java
View File

@ -63,6 +63,10 @@ public class HighlightDirectorFlowService {
flowId, projectId, store.directorDirectory(projectId).resolve(properties.getApprovalFileName()));
return HighlightFlowResult.skipped(projectId, flowId, "approval_missing");
}
Path montageFile = store.directorDirectory(projectId).resolve("montage.json");
if (Files.isRegularFile(montageFile)) {
return processMontage(projectId, flowId, scanId, project, montageFile, startedAt);
}
HighlightDirectorPlan plan;
try {
plan = planValidator.validate(projectId, readPlan(planFile));
@ -136,6 +140,103 @@ public class HighlightDirectorFlowService {
}
}
private HighlightFlowResult processMontage(String projectId, String flowId, long scanId,
HighlightProject project, Path montageFile, long startedAt) {
MontagePlan montage;
try {
montage = objectMapper.readValue(montageFile.toFile(), MontagePlan.class);
} catch (IOException ex) {
markFailed(project, "Montage plan unreadable: " + ex.getMessage());
throw new IllegalStateException("Unable to read montage plan: " + montageFile, ex);
}
HighlightSourceAnalysis analysis = store.readJson(projectId, "analysis/source-analysis.json",
HighlightSourceAnalysis.class);
String clipId = analysis.source().clipId();
double sourceDuration = analysis.source().durationSeconds();
List<EditDecision> decisions = new ArrayList<>();
double timeline = 0.0;
List<MontagePlan.Shot> shots = montage.shots();
for (int i = 0; i < shots.size(); i++) {
MontagePlan.Shot shot = shots.get(i);
double speed = shot.speed() <= 0 ? 1.0 : shot.speed();
double dur = Math.max(0.2, shot.durationSeconds());
double srcSpan = dur * speed;
double srcStart = Math.max(0.0, Math.min(shot.sourceStartSeconds(), sourceDuration - srcSpan));
double srcEnd = Math.min(sourceDuration, srcStart + srcSpan);
String transitionIn = (i == 0) ? "fade-in" : "cut";
String transitionOut = (i == shots.size() - 1) ? "fade-out" : "cut";
String treatment = "zoom=%.3f cinematic".formatted(shot.zoom() <= 0 ? 1.2 : shot.zoom());
decisions.add(new EditDecision(clipId, srcStart, srcEnd, timeline, timeline + dur,
transitionIn, transitionOut, speed, treatment, "montage"));
timeline += dur;
}
double total = timeline;
List<AudioCue> audioCues = new ArrayList<>();
if (montage.musicDirection() != null && !montage.musicDirection().isBlank()) {
audioCues.add(new AudioCue("music", safeKey("music", montage.musicDirection()), 0.0, total, -14.0,
montage.musicDirection()));
}
List<TextOverlay> overlays = new ArrayList<>();
if (montage.overlays() != null) {
for (MontagePlan.Overlay o : montage.overlays()) {
overlays.add(new TextOverlay(o.text(), o.timelineStartSeconds(), o.timelineEndSeconds(),
o.placement() == null || o.placement().isBlank() ? "lower_center_safe" : o.placement(),
"fade", "montage"));
}
}
String gradeKeyword = montage.grade() == null || montage.grade().isBlank() ? "hero" : montage.grade();
List<String> voiceoverLines = montage.voiceover() == null ? List.of() : montage.voiceover();
HighlightDirectorPlan.HighlightItem montageHighlight = new HighlightDirectorPlan.HighlightItem(
"montage", "montage", "Cinematic Montage", 0.0, total, total,
gradeKeyword.contains("hero") ? "hero_payoff" : gradeKeyword,
"cinematic montage", montage.musicDirection(), "", voiceoverLines,
overlays.stream().map(TextOverlay::text).toList(), "montage");
List<VoiceoverLine> voiceover = HighlightAssetPreparationService.plannedVoiceoverLines(montageHighlight);
String style = safeKey("style", "montage", gradeKeyword);
EditPlan editPlan = new EditPlan(projectId, style, total, decisions, audioCues, voiceover, overlays,
"mp4-h264-aac-1080p", "cinematic montage");
store.writeJson(projectId, "highlights/montage/edit-plan.json", editPlan);
log.info("event=highlight_montage_started flow_id={} scan_id={} project_id={} shots={} duration={}",
flowId, scanId, projectId, decisions.size(), total);
markStatus(project, HighlightProjectStatus.PLANNED, null);
ContentCategory category;
try {
category = store.readJson(projectId, "analysis/category.json", CinematicHighlightAnalysis.class).category();
} catch (RuntimeException ex) {
category = ContentCategory.GENERIC_VLOG;
}
assetPreparationService.prepare(projectId, project, montageHighlight, category);
HighlightLocalAssetWorker.HighlightAssetWorkerResult assetResult =
assetWorker.process(projectId, montageHighlight, category);
if (!assetResult.pendingRequests().isEmpty()) {
markFailed(project, "Montage assets pending: " + assetResult.pendingRequests());
return HighlightFlowResult.skipped(projectId, flowId, "assets_pending");
}
markStatus(project, HighlightProjectStatus.RENDERING, null);
HighlightFfmpegRenderer.HighlightRenderResult result;
try {
result = renderer.render(projectId, montageHighlight, editPlan);
requireQaPassed(result);
} catch (RuntimeException ex) {
markFailed(project, "Montage render rejected: " + ex.getMessage());
throw ex;
}
Path projectOutput = store.projectDirectory(projectId).resolve("final.mp4");
concatFinalOutputs(projectOutput, List.of(result.finalOutput()));
RenderManifest manifest = new RenderManifest(projectId, List.of(result.highlightId()),
List.of(result.finalOutput().toString()), projectOutput.toString(), total, List.of(), List.of(),
Instant.now());
store.writeJson(projectId, "render-manifest.json", manifest);
markStatus(project, HighlightProjectStatus.RENDERED, null);
log.info("event=highlight_montage_completed flow_id={} project_id={} shots={} duration={} elapsed_ms={}",
flowId, projectId, decisions.size(), total, (System.nanoTime() - startedAt) / 1_000_000);
return HighlightFlowResult.rendered(projectId, flowId, List.of(result.finalOutput()), projectOutput);
}
private HighlightDirectorPlan readPlan(Path planFile) {
try {
return objectMapper.readValue(planFile.toFile(), HighlightDirectorPlan.class);

33
src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java
View File

@ -193,10 +193,13 @@ public class HighlightFfmpegRenderer {
|| "none".equalsIgnoreCase(decision.visualTreatment()));
StringBuilder filter = new StringBuilder();
if (styled) {
// Tighter, more intentional framing that fills the frame and hides the mundane location,
// with a progressive punch-in across the cuts of a beat (each cut steps tighter) so the edit
// reads as deliberate rather than one long pan. Center crop keeps the subject framed.
double zoom = 1.16 + Math.min(cutIndex, 3) * 0.12;
// Tighter, more intentional framing that fills the frame and hides the mundane location.
// A montage shot can specify its exact framing via a "zoom=" token in the treatment; otherwise
// apply a progressive punch-in across the cuts of a beat (each cut steps tighter). Center crop.
double zoom = explicitZoom(decision.visualTreatment());
if (zoom <= 0) {
zoom = 1.16 + Math.min(cutIndex, 3) * 0.12;
}
filter.append("crop=iw/%.4f:ih/%.4f,scale=%d:%d,format=yuv420p".formatted(zoom, zoom, w, h));
} else {
filter.append("scale=%d:%d:force_original_aspect_ratio=decrease,".formatted(w, h)
@ -223,6 +226,9 @@ public class HighlightFfmpegRenderer {
"-i", source, "-vf", filter.toString(),
"-af", audioTempoFilter(decision.playbackSpeed()),
"-r", Integer.toString(properties.getOutputFrameRate()),
// Pin each segment to its exact target duration so per-segment frame-quantization drift
// cannot accumulate across a many-shot montage (keeps the concat matching the timeline).
"-t", Double.toString(outputDuration),
"-map", "0:v:0", "-map", "0:a?",
"-c:v", "libx264", "-preset", "veryfast", "-crf", "18",
"-c:a", "aac", "-b:a", properties.getAudioBitrate(),
@ -597,6 +603,25 @@ public class HighlightFfmpegRenderer {
return "crop=w='iw*0.96':h='ih*0.96':x='(iw-out_w)/2':y='(ih-out_h)/2',";
}
// A montage shot encodes its framing as "zoom=1.35" in the visual treatment; returns <=0 when absent.
private double explicitZoom(String visualTreatment) {
if (visualTreatment == null) {
return -1;
}
java.util.regex.Matcher m = ZOOM_TOKEN.matcher(visualTreatment);
if (m.find()) {
try {
return Double.parseDouble(m.group(1));
} catch (NumberFormatException ignored) {
return -1;
}
}
return -1;
}
private static final java.util.regex.Pattern ZOOM_TOKEN =
java.util.regex.Pattern.compile("zoom=([0-9]+(?:\\.[0-9]+)?)");
// 2.39:1 cinematic letterbox: crop the center band, then pad back to frame with black bars.
private String letterboxFilter() {
int w = properties.getOutputWidth();

29
src/main/java/org/example/videoclips/editing/MontagePlan.java Normal file
View File

@ -0,0 +1,29 @@
package org.example.videoclips.editing;
import java.util.List;
/**
* A cinematic montage: many short shots pulled from anywhere in the source, sequenced and cut to a
* rhythm, rather than a few fixed 8-35s highlight windows. Rendered as a single continuous edit with one
* music bed. Present as {@code director/montage.json} to select the montage path.
*/
public record MontagePlan(
String projectId,
String sourceVideoFileName,
String grade, // grade keyword: opening | rising | hero (drives the look)
String musicDirection,
List<String> voiceover, // optional narration lines, distributed across the montage
List<Overlay> overlays, // optional titles
List<Shot> shots
) {
/**
* One shot. {@code durationSeconds} is how long it plays on the timeline; {@code speed} is the playback
* speed (0.5 = slow motion). The source span consumed is durationSeconds * speed. {@code zoom} is the
* punch-in framing (1.0 = full frame, 1.4 = tight).
*/
public record Shot(double sourceStartSeconds, double durationSeconds, double zoom, double speed) {
}
public record Overlay(String text, double timelineStartSeconds, double timelineEndSeconds, String placement) {
}
}