diff --git a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java index c8f8d9e..54ede1f 100644 --- a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java +++ b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java @@ -554,6 +554,13 @@ public class VideoClippingProperties { /** R14 subject-tracking reframe: follow the detected subject instead of a static centre crop. */ private boolean subjectReframeEnabled = false; + /** + * Highlight-reel selection bar (0..1). Every candidate whose blended score (0.6*judge-worthiness + + * 0.4*action-intensity) clears this becomes its own segment in the final reel; the single best is + * always kept. Lower = more segments. See {@code HighlightMontageDirector.composeReel}. + */ + private double highlightSelectThreshold = 0.5; + /** Offline subject-tracking tool (YOLO). YOLOv8 is AGPL-3.0 → non-commercial, like the CV worker. */ private String subjectTrackScript = "./tools/subject_track.py"; @@ -804,6 +811,14 @@ public class VideoClippingProperties { this.subjectReframeEnabled = subjectReframeEnabled; } + public double getHighlightSelectThreshold() { + return highlightSelectThreshold; + } + + public void setHighlightSelectThreshold(double highlightSelectThreshold) { + this.highlightSelectThreshold = highlightSelectThreshold; + } + public String getSubjectTrackScript() { return subjectTrackScript; } diff --git a/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java b/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java index 2029017..6bddb93 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java +++ b/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java @@ -30,12 +30,13 @@ import java.util.List; public class HighlightMontageDirector { static final double WINDOW_SECONDS = 0.5; - /** How many candidate decisive moments to propose for the vision judge to rank. */ - static final int MAX_CANDIDATES = 5; /** Minimum spacing (seconds) between proposed candidates, so they are distinct moments. Kept short so a * reaction close on the heels of the action (a celebration right after the strike) stays a separate * candidate the vision judge can score, rather than being merged into the louder neighbouring peak. */ static final double MIN_CANDIDATE_SEPARATION = 1.0; + /** Selection-score blend: how the judge's meaning-rating and the measured action intensity combine. */ + static final double SELECT_SEMANTIC_WEIGHT = 0.6; + static final double SELECT_INTENSITY_WEIGHT = 0.4; private final VideoClippingProperties.Editing properties; @@ -43,45 +44,44 @@ public class HighlightMontageDirector { this.properties = properties.getEditing(); } - /** Measure the source and compose a montage, choosing the decisive moment by measured intensity alone. */ + /** Measure the source and compose a reel, choosing decisive moments by measured intensity alone. */ public MontagePlan direct(String projectId, String sourceFileName, Path source, double durationSeconds) { return direct(projectId, sourceFileName, source, durationSeconds, null); } /** - * Measure the source, propose candidate decisive moments (intensity peaks), and let {@code chooser} — the - * Tier-2 vision judge — pick which candidate is the actual highlight. This is the generic rule: measurement - * never decides the moment by position or threshold (a camera whip or a loud aftermath can out-score a - * quiet celebration); it only PROPOSES, and the model JUDGES by meaning. When no chooser is supplied, or it - * declines (NaN), the highest-intensity candidate is used as a measured fallback. + * Compose a multi-segment HIGHLIGHT REEL. Measurement proposes candidate moments across the WHOLE clip + * (intensity peaks); the Tier-2 vision {@code judge} rates each by meaning and supplies an overlay; the + * director keeps EVERY moment worth showing (blending the judge's rating with measured action intensity, + * so a dynamic section is kept even when the model can't name it), builds an action segment + * (entry -> peak -> exit) around each, and concatenates them all into one video. No cap on how many + * segments or how long. When no judge is supplied, selection is by measured intensity alone. */ public MontagePlan direct(String projectId, String sourceFileName, Path source, double durationSeconds, - MomentChooser chooser) { + MomentJudge judge) { double[] motion = probeCurve(source.toString(), true, durationSeconds); double[] audio = probeCurve(source.toString(), false, durationSeconds); List candidates = candidatePeaks(motion, audio, WINDOW_SECONDS, durationSeconds); - double climax = Double.NaN; - if (chooser != null && !candidates.isEmpty()) { - climax = chooser.choose(List.copyOf(candidates)); - } - if (!Double.isFinite(climax)) { - climax = candidates.isEmpty() ? 0.0 : candidates.get(0); // measured fallback: strongest peak - } - return composeMontageAt(projectId, sourceFileName, motion, audio, climax, WINDOW_SECONDS, durationSeconds); + List judgements = (judge != null && !candidates.isEmpty()) + ? judge.judge(List.copyOf(candidates)) : null; + return composeReel(projectId, sourceFileName, motion, audio, candidates, judgements, + WINDOW_SECONDS, durationSeconds); } - /** Chooses which candidate moment (seconds) is the real highlight; returns NaN to defer to measurement. */ + /** Per-candidate verdict from the Tier-2 vision judge: how highlight-worthy (0..1) and a ready overlay. */ + public record Judgement(double worthiness, String overlayText) { + } + + /** Rates each candidate moment; returns one {@link Judgement} per candidate (same order). */ @FunctionalInterface - public interface MomentChooser { - double choose(List candidateTimeSeconds); + public interface MomentJudge { + List judge(List candidateTimeSeconds); } /** * Proposes candidate decisive moments as the local maxima of measured intensity (motion + audio), - * strongest first, min-separated. GENERIC: no positional band, no per-video threshold — every real - * event (a strike, a celebration, a turn-away, a goal, an anticipation) becomes a candidate, and the - * vision judge decides which is the highlight. {@code motion[i]}/{@code audio[i]} cover the window at - * {@code i * window} seconds. + * min-separated, returned in TIME order. GENERIC: no positional band, no per-video threshold and NO cap on + * how many — every real event becomes a candidate, and the judge + intensity decide which make the reel. */ List candidatePeaks(double[] motion, double[] audio, double window, double duration) { int n = Math.min(motion.length, audio.length); @@ -117,52 +117,112 @@ public class HighlightMontageDirector { peaks.add(i); } } - peaks.sort((x, y) -> Double.compare(intensity[y], intensity[x])); // strongest first + peaks.sort((x, y) -> Double.compare(intensity[y], intensity[x])); // strongest first for dedup List chosen = new ArrayList<>(); for (int idx : peaks) { double t = idx * window; boolean tooClose = chosen.stream().anyMatch(c -> Math.abs(c - t) < MIN_CANDIDATE_SEPARATION); if (!tooClose) { - chosen.add(t); - } - if (chosen.size() >= MAX_CANDIDATES) { - break; + chosen.add(t); // no cap on how many candidates } } + chosen.sort(Double::compareTo); // return in time order return chosen; } - MontagePlan composeMontage(String projectId, String sourceFileName, double[] motion, double[] audio, - double window, double duration) { - // Measured path (no vision judge): build around the strongest intensity peak. - List candidates = candidatePeaks(motion, audio, window, duration); - double climax = candidates.isEmpty() ? 0.0 : candidates.get(0); - return composeMontageAt(projectId, sourceFileName, motion, audio, climax, window, duration); - } - /** - * Builds the ACTION SEGMENT (start -> peak -> outcome) around a chosen climax time. Pure and deterministic; - * package-visible for unit testing without ffmpeg. The onset (start) and resolution (end) are MEASURED from - * motion around the given peak — the peak itself is chosen by the caller (the vision judge). + * Builds the multi-segment reel from the candidates. Each candidate's SELECTION SCORE blends the judge's + * highlight-worthiness with the candidate's measured action intensity: {@code 0.6*worthiness + + * 0.4*intensity}. Every candidate whose score clears the threshold becomes a segment (at least the best + * one always does); each segment is an action unit (entry -> slow-mo peak -> exit) that never overlaps the + * previous one. Pure and deterministic; package-visible for unit testing without ffmpeg. */ - MontagePlan composeMontageAt(String projectId, String sourceFileName, double[] motion, double[] audio, - double climaxTime, double window, double duration) { + MontagePlan composeReel(String projectId, String sourceFileName, double[] motion, double[] audio, + List candidates, List judgements, double window, double duration) { int n = Math.min(motion.length, audio.length); - if (n < 4 || duration <= 0) { + if (n < 4 || duration <= 0 || candidates == null || candidates.isEmpty()) { return straightCut(projectId, sourceFileName, duration); } double[] m = smooth(motion, n); + double[] mN = normalize(m, n); + double[] aN = normalize(smooth(audio, n), n); + + // Score every candidate = 0.6 * (judge worthiness) + 0.4 * (measured intensity). When the model can't + // discriminate (all "riding" -> neutral), intensity decides which sections are the real highlights. + int k = candidates.size(); + double[] score = new double[k]; + for (int i = 0; i < k; i++) { + int idx = Math.max(0, Math.min(n - 1, (int) Math.round(candidates.get(i) / window))); + double intensity = (aN[idx] + mN[idx]) / 2.0; // 0..1 + double worthiness = (judgements != null && i < judgements.size()) + ? judgements.get(i).worthiness() : 0.4; // neutral when no judge + score[i] = SELECT_SEMANTIC_WEIGHT * worthiness + SELECT_INTENSITY_WEIGHT * intensity; + } + double bar = properties.getHighlightSelectThreshold(); + int best = 0; + for (int i = 1; i < k; i++) { + if (score[i] > score[best]) { + best = i; + } + } + List selected = new ArrayList<>(); + for (int i = 0; i < k; i++) { + if (score[i] >= bar) { + selected.add(i); // candidates are time-ordered + } + } + if (selected.isEmpty()) { + selected.add(best); // always at least the single best + } + + List allShots = new ArrayList<>(); + List overlays = new ArrayList<>(); + double timeline = 0.0; + double prevExit = Double.NEGATIVE_INFINITY; + for (int i : selected) { + Segment seg = buildSegment(m, candidates.get(i), window, duration, prevExit); + if (seg == null) { + continue; // overlapped or degenerate -> merged away + } + double preDur = 0; + for (int s = 0; s < seg.payoffShotIndex(); s++) { + preDur += seg.shots().get(s).durationSeconds(); + } + double payoffTlStart = timeline + preDur; + double payoffTlDur = seg.shots().get(seg.payoffShotIndex()).durationSeconds(); + allShots.addAll(seg.shots()); + for (MontagePlan.Shot sh : seg.shots()) { + timeline += sh.durationSeconds(); + } + String overlayText = (judgements != null && i < judgements.size()) ? judgements.get(i).overlayText() : ""; + if (overlayText != null && !overlayText.isBlank()) { + overlays.add(new MontagePlan.Overlay(overlayText, round(payoffTlStart + 0.2), + round(Math.max(payoffTlStart + 0.8, payoffTlStart + payoffTlDur - 0.2)), "lower_center_safe")); + } + prevExit = seg.exitSrc(); + } + if (allShots.isEmpty()) { + return straightCut(projectId, sourceFileName, duration); + } + return new MontagePlan(projectId, sourceFileName, "hero", genericMusic(), List.of(), overlays, allShots); + } + + private record Segment(List shots, double exitSrc, int payoffShotIndex) { + } + + /** + * One action segment around a peak: MEASURED entry (build into the peak, capped) -> slow-mo payoff on the + * peak -> MEASURED exit (motion settles). Returns null when the peak is already covered by the previous + * segment (so segments never overlap) or the span is degenerate. + */ + private Segment buildSegment(double[] m, double climaxTime, double window, double duration, double prevExit) { + int n = m.length; int climaxIdx = Math.max(1, Math.min(n - 1, (int) Math.round(climaxTime / window))); climaxTime = climaxIdx * window; - - // Onset: the biggest motion spike before the peak (the release/approach); the action shot leads in. - int actionIdx = argMax(m, Math.max(1, (int) Math.floor(0.05 * n)), - Math.max(2, (int) Math.floor((climaxTime - 1.0) / window))); - double actionTime = actionIdx * window; - - // Resolution: extend past the peak through the outcome AND the immediate reaction (the pins falling and - // the celebration; the ball settling in the net), stopping when motion settles back to baseline for a - // sustained stretch, at a camera whip, or at a cap. This is MEASURED, not positional. + if (climaxTime <= prevExit + 0.5) { + return null; // this peak is inside the previous segment + } + // Exit: extend past the peak while motion stays elevated, stop when it settles / at a whip / at a cap. double whipLevel = Math.max(1e-6, percentile(m, n, 0.4) * 4.0); int whipCutoff = n; for (int i = climaxIdx + 2; i < n; i++) { @@ -184,36 +244,40 @@ public class HighlightMontageDirector { resolution = t; if (m[i] <= settleLevel) { if (++settledRun >= 3) { - break; // motion has settled -> the action has resolved + break; } } else { settledRun = 0; } } - double endLimit = Math.min(duration, Math.max(climaxTime + 1.5, resolution + 0.3)); - - // Cap the pre-climax build. When the action spike is far from the climax (long source), the - // action+tension shot would otherwise stretch to tens of seconds — one ultra-long continuous shot - // that reads as dead air AND blows up score generation (music length == montage length). Bound the - // build to montageMaxBuildSeconds so the highlight stays tight and generatable regardless of source - // length. The build still leads INTO the climax; the setup shot precedes it. - double actionEnd = climaxTime - 0.6; - double desiredBuild = actionEnd - (actionTime - 1.5); - double build = Math.min(Math.max(0.8, desiredBuild), Math.max(1.0, properties.getMontageMaxBuildSeconds())); - double buildStart = Math.max(0.0, actionEnd - build); + double exit = Math.min(duration, Math.max(climaxTime + 1.5, resolution + 0.3)); + // Entry: build INTO the peak, bounded by the cap and never earlier than the previous segment's exit. + int actionIdx = argMax(m, Math.max(1, (int) Math.floor(0.05 * n)), + Math.max(2, (int) Math.floor((climaxTime - 1.0) / window))); + double actionTime = actionIdx * window; + double payoffStart = climaxTime - 0.3; + double desiredBuild = payoffStart - (actionTime - 1.0); + double build = Math.min(Math.max(0.6, desiredBuild), Math.max(1.0, properties.getMontageMaxBuildSeconds())); + double buildStart = Math.max(Math.max(0.0, prevExit), payoffStart - build); + double payoffEnd = Math.min(climaxTime + 2.0, exit - 0.4); + if (exit - payoffStart < 0.6) { + return null; // too short to be a segment + } + if (payoffEnd < payoffStart + 0.5) { + payoffEnd = Math.min(exit, payoffStart + 0.8); + } List shots = new ArrayList<>(); - addShot(shots, buildStart - 1.7, buildStart, 1.03, 1.0, duration); // setup - addShot(shots, buildStart, actionEnd, 1.04, 1.0, duration); // action + tension (capped) - addShot(shots, climaxTime - 0.6, climaxTime + 0.2, 1.06, 1.0, duration); // realization - double payoffEnd = Math.min(climaxTime + 2.2, endLimit - 0.9); - addShot(shots, climaxTime + 0.2, payoffEnd, 1.05, 0.7, duration); // slow-mo payoff - addShot(shots, payoffEnd, endLimit, 1.04, 0.9, duration); // resolution button - - if (shots.isEmpty()) { - return straightCut(projectId, sourceFileName, duration); + if (payoffStart - buildStart >= 0.4) { + addShot(shots, buildStart, payoffStart, 1.04, 1.0, duration); // entry / build into the peak } - return new MontagePlan(projectId, sourceFileName, "hero", genericMusic(), List.of(), List.of(), shots); + int payoffShotIndex = shots.size(); + addShot(shots, payoffStart, payoffEnd, 1.05, 0.7, duration); // slow-mo payoff on the peak + if (shots.size() <= payoffShotIndex) { + return null; // payoff shot was degenerate + } + addShot(shots, payoffEnd, exit, 1.04, 0.9, duration); // exit / resolution + return new Segment(shots, exit, payoffShotIndex); } private MontagePlan straightCut(String projectId, String sourceFileName, double duration) { diff --git a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java index 8b9c003..29c121e 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java +++ b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java @@ -229,18 +229,13 @@ public class HighlightSourceScheduler { try { double duration = analysis.source() == null ? 0.0 : analysis.source().durationSeconds(); Path visionWork = store.directorDirectory(projectId).resolve("vision-work"); - // GENERIC rule: measurement PROPOSES candidate decisive moments (intensity peaks); the vision judge - // PICKS which is the real highlight by meaning (a celebration/goal beats a loud turn-away). No - // positional band, no per-video threshold. Falls back to the strongest peak if the model declines. - HighlightMontageDirector.MomentChooser chooser = properties.isVisionDirectorEnabled() - ? candidates -> visionDirector.rankDecisiveMoment(sourcePath, candidates, visionWork) + // GENERIC rule: measurement PROPOSES candidate moments across the WHOLE clip; the vision judge rates + // each by meaning and supplies an honest overlay; the director keeps EVERY worthy moment (blended + // with action intensity) and builds a multi-segment reel. No band, no per-video threshold, no cap. + HighlightMontageDirector.MomentJudge judge = properties.isVisionDirectorEnabled() + ? candidates -> visionDirector.judgeMoments(sourcePath, candidates, visionWork) : null; - MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration, chooser); - if (properties.isVisionDirectorEnabled()) { - // Dense pass over the chosen decisive window (payoff shot through the outcome) with the honesty - // rule: never assert an action the shown window doesn't contain (a "KICK" over a cut before it). - montage = visionDirector.groundOverlay(montage, sourcePath, visionWork); - } + MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration, judge); store.writeJson(projectId, "director/montage.json", montage); // Minimal edit-plan.json so the render scanner selects the project; montage.json takes precedence. store.writeJson(projectId, "director/edit-plan.json", diff --git a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java index d03428c..1b3d70a 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java +++ b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java @@ -132,37 +132,25 @@ public class HighlightVisionDirector { } /** - * Densely captions the DECISIVE SOURCE WINDOW of a plan — the payoff shot through the final shot, i.e. the - * action AND its outcome — and attaches an honest, grounded overlay plus scene-informed music. Unlike a - * sparse whole-clip pass, this assesses several frames THROUGH the shown payoff, so the overlay reflects - * what is actually on screen. The honesty rule ({@link #honestOverlayText}) never asserts an action the - * window does not show: an anticipatory cut gets a teaser question instead. Fail-soft: on any error (model - * missing, worker failure, timeout) it returns the plan unchanged, so the Tier-1 cut always stands. + * The Tier-2 vision JUDGE for a highlight reel. Captions EVERY candidate moment in one batched pass (a + * description to rate it, plus a punchy label + a teaser question for its overlay) and returns, per + * candidate, its highlight-worthiness and a ready, honest overlay. A celebration or a scored goal rates + * high; a loud turn-away or an "about to…" build rates low — meaning, not motion. The director blends these + * ratings with measured intensity to decide which moments make the reel and how to label each. Fail-soft: + * returns an empty list on any error, so the director selects by measured intensity alone. */ - public MontagePlan groundOverlay(MontagePlan plan, Path source, Path workDir) { - int payoffIdx = payoffShotIndex(plan); - if (payoffIdx < 0 || plan.shots().isEmpty()) { - return plan; - } - // Caption the CHOSEN PEAK itself — the payoff (slow-mo) shot — not the trailing outcome/button shots. - // Sampling through to the last shot let a post-payoff reaction frame (e.g. the bowler lowering his arms - // and holding his face after the celebration) supply the overlay; confining it to the payoff shot keeps - // the label on the decisive moment the judge picked. - MontagePlan.Shot payoffShot = plan.shots().get(payoffIdx); - double winStart = payoffShot.sourceStartSeconds(); - double winEnd = payoffShot.sourceStartSeconds() + payoffShot.durationSeconds() * Math.max(0.05, payoffShot.speed()); - if (winEnd <= winStart) { - winEnd = winStart + 1.0; + public List judgeMoments(Path source, List candidateTimes, + Path workDir) { + if (candidateTimes == null || candidateTimes.isEmpty()) { + return List.of(); } try { Files.createDirectories(workDir); - int samples = 3; List manifest = new ArrayList<>(); List ids = new ArrayList<>(); - for (int i = 0; i < samples; i++) { - double t = winStart + (winEnd - winStart) * (i + 0.5) / samples; - Path frame = workDir.resolve("payoff-" + i + ".jpg"); - if (extractFrame(source, t, frame)) { + for (int i = 0; i < candidateTimes.size(); i++) { + Path frame = workDir.resolve("cand-" + i + ".jpg"); + if (extractFrame(source, candidateTimes.get(i), frame)) { String img = frame.toAbsolutePath().toString(); manifest.add(new Manifest("d" + i, img, DESCRIPTIVE_QUESTION)); manifest.add(new Manifest("l" + i, img, @@ -173,94 +161,29 @@ public class HighlightVisionDirector { } } if (ids.isEmpty()) { - return plan; + return List.of(); } List caps = runCaptioner(workDir, manifest); - // Pick the frame whose description most shows the decisive action; later frames win ties so the - // window's climax is preferred over its lead-in. - int best = ids.get(0); - double bestScore = -1; - for (int i : ids) { - double sc = semanticScore(answerFor(caps, "d" + i)); - if (sc >= bestScore) { - bestScore = sc; - best = i; - } - } - String description = answerFor(caps, "d" + best); - String overlayText = honestOverlayText(description, answerFor(caps, "l" + best), - answerFor(caps, "q" + best)); - List overlays = new ArrayList<>( - plan.overlays() == null ? List.of() : plan.overlays()); - if (!overlayText.isBlank()) { - double[] span = payoffTimeline(plan, payoffIdx); - overlays.add(new MontagePlan.Overlay(overlayText, span[0] + 0.2, - Math.max(span[0] + 0.8, span[1] - 0.2), "lower_center_safe")); - } - String music = description.isBlank() ? plan.musicDirection() - : "cinematic film score for this moment: " + description + " Build quiet tension to a " - + "triumphant climax hit at the payoff, then a short warm resolve, modern trailer score, no vocals."; - log.info("event=highlight_vision_director_completed overlay=\"{}\" payoff_description=\"{}\"", - overlayText, description); - return new MontagePlan(plan.projectId(), plan.sourceVideoFileName(), plan.grade(), music, - plan.voiceover(), overlays, plan.shots()); - } catch (RuntimeException | IOException | InterruptedException ex) { - if (ex instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.warn("event=highlight_vision_director_failed stage=overlay error_type={} message={}", - ex.getClass().getSimpleName(), ex.getMessage()); - return plan; - } - } - - /** - * The Tier-2 vision JUDGE. Given candidate decisive moments proposed by measurement, it captions each and - * returns the time of the most highlight-worthy one — a celebration or a scored goal beats a loud - * camera turn-away or an anticipatory build, because "is this THE highlight?" is a question of meaning, not - * of motion or loudness. Returns NaN on any failure so the director falls back to the strongest peak. - */ - public double rankDecisiveMoment(Path source, List candidateTimes, Path workDir) { - if (candidateTimes == null || candidateTimes.isEmpty()) { - return Double.NaN; - } - try { - Files.createDirectories(workDir); - List manifest = new ArrayList<>(); - List ids = new ArrayList<>(); + List out = new ArrayList<>(); for (int i = 0; i < candidateTimes.size(); i++) { - Path frame = workDir.resolve("cand-" + i + ".jpg"); - if (extractFrame(source, candidateTimes.get(i), frame)) { - manifest.add(new Manifest("d" + i, frame.toAbsolutePath().toString(), DESCRIPTIVE_QUESTION)); - ids.add(i); + if (!ids.contains(i)) { + out.add(new HighlightMontageDirector.Judgement(0.0, "")); // unreadable frame + continue; } + String description = answerFor(caps, "d" + i); + double worthiness = highlightWorthiness(description); + String overlay = honestOverlayText(description, answerFor(caps, "l" + i), answerFor(caps, "q" + i)); + out.add(new HighlightMontageDirector.Judgement(worthiness, overlay)); } - if (ids.isEmpty()) { - return Double.NaN; - } - List caps = runCaptioner(workDir, manifest); - int best = -1; - double bestScore = -1; - for (int i : ids) { - double score = highlightWorthiness(answerFor(caps, "d" + i)); - if (score > bestScore) { // strict >: ties keep the stronger-intensity (earlier) candidate - bestScore = score; - best = i; - } - } - if (best < 0) { - return Double.NaN; - } - log.info("event=highlight_vision_moment_ranked candidates={} chosen_time={} score={}", - ids.size(), candidateTimes.get(best), bestScore); - return candidateTimes.get(best); + log.info("event=highlight_vision_moments_judged candidates={} rated={}", candidateTimes.size(), ids.size()); + return out; } catch (RuntimeException | IOException | InterruptedException ex) { if (ex instanceof InterruptedException) { Thread.currentThread().interrupt(); } - log.warn("event=highlight_vision_director_failed stage=rank error_type={} message={}", + log.warn("event=highlight_vision_director_failed stage=judge error_type={} message={}", ex.getClass().getSimpleName(), ex.getMessage()); - return Double.NaN; + return List.of(); } } diff --git a/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java b/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java index e2bb6aa..80e639f 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java @@ -1,8 +1,10 @@ package org.example.videoclips.editing; import org.example.videoclips.config.VideoClippingProperties; +import org.example.videoclips.editing.HighlightMontageDirector.Judgement; import org.junit.jupiter.api.Test; +import java.util.ArrayList; import java.util.List; import static org.assertj.core.api.Assertions.assertThat; @@ -12,116 +14,148 @@ class HighlightMontageDirectorTest { private final HighlightMontageDirector director = new HighlightMontageDirector(new VideoClippingProperties()); @Test - void proposesEveryRealMomentAsACandidateWithoutBias() { - // 64s source: an early goal (~8s) AND a late, HIGHER-intensity aftermath (~55s: loud + high motion). - // Measurement must propose BOTH as candidates — it does not get to decide which is the highlight - // (that is the vision judge's job). Note the aftermath ranks first: measurement alone would pick it. - int n = 128; + void proposesEveryRealMomentAsACandidateInTimeOrderWithNoCap() { + // Several distinct action bursts across a long clip: measurement must propose them ALL (no cap), in + // time order. It has no opinion on which is the highlight — that is the judge's job. + int n = 160; double[] motion = new double[n]; double[] audio = new double[n]; for (int i = 0; i < n; i++) { double t = i * 0.5; motion[i] = 2.0; - if (t >= 7.0 && t <= 10.0) motion[i] = 7.0; // the goal - if (t >= 54.0 && t <= 56.0) motion[i] = 8.0; // the aftermath (higher motion) audio[i] = -30.0; - if (t >= 8.0 && t <= 11.0) audio[i] = -10.0; // goal reaction - if (t >= 54.0 && t <= 56.0) audio[i] = -12.0; // aftermath (loud) + for (double peak : new double[]{8, 22, 41, 60}) { + if (Math.abs(t - peak) < 0.6) { + motion[i] = 7.0; + audio[i] = -12.0; + } + } } + List candidates = director.candidatePeaks(motion, audio, 0.5, 80.0); - List candidates = director.candidatePeaks(motion, audio, 0.5, 64.0); - - assertThat(candidates).anySatisfy(t -> assertThat(t).isBetween(6.0, 11.0)); // the goal is a candidate - assertThat(candidates).anySatisfy(t -> assertThat(t).isBetween(53.0, 57.0)); // the aftermath too - // Distinct moments (min-separated), strongest first -> the aftermath, which a naive pick would take. - assertThat(candidates.get(0)).isBetween(53.0, 57.0); + assertThat(candidates).hasSizeGreaterThanOrEqualTo(4); + assertThat(candidates).isSorted(); // time order + for (double peak : new double[]{8, 22, 41, 60}) { + assertThat(candidates).anySatisfy(t -> assertThat(t).isCloseTo(peak, org.assertj.core.api.Assertions.within(1.0))); + } } @Test - void buildsTheActionSegmentAroundTheCHOSENPeakAndSweepsInTheOutcome() { - // The judge chose the early goal (~8s), NOT the louder late aftermath. Given that choice, the segment - // must place the slow-mo payoff on the goal and extend the window through its outcome (ball settling). - int n = 128; + void buildsAMultiSegmentReelFromEveryWorthyMoment() { + // Two DISTINCT worthy actions, far apart (~10s and ~40s). Both must become segments in ONE reel, each + // with its own slow-mo payoff and its own overlay. + int n = 120; double[] motion = new double[n]; double[] audio = new double[n]; for (int i = 0; i < n; i++) { double t = i * 0.5; motion[i] = 2.0; - if (t >= 7.0 && t <= 10.0) motion[i] = 7.0; // the goal (peak) - if (t > 10.0 && t <= 13.0) motion[i] = 4.0; // the ball settling in the net (outcome) - if (t >= 54.0 && t <= 56.0) motion[i] = 8.0; // aftermath elsewhere (not chosen) audio[i] = -30.0; - if (t >= 8.0 && t <= 11.0) audio[i] = -10.0; + if (Math.abs(t - 10.0) < 0.8) { motion[i] = 8.0; audio[i] = -10.0; } + if (Math.abs(t - 40.0) < 0.8) { motion[i] = 8.0; audio[i] = -10.0; } } + List candidates = director.candidatePeaks(motion, audio, 0.5, 60.0); + List judgements = judge(candidates, 1.0, "GOAL"); // both worthy - MontagePlan plan = director.composeMontageAt("p", "src.mp4", motion, audio, 8.5, 0.5, 64.0); + MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 60.0); - double payoff = payoffSourceStart(plan); - assertThat(payoff).isBetween(7.5, 10.0); // slow-mo on the goal - double maxSourceEnd = plan.shots().stream() + long slowMo = plan.shots().stream().filter(s -> s.speed() < 0.8).count(); + assertThat(slowMo).isEqualTo(2); // two payoffs -> two segments + assertThat(plan.overlays()).hasSize(2); // one overlay per segment + // The two payoffs sit on the two actions (~10s and ~40s of source). + List payoffs = plan.shots().stream().filter(s -> s.speed() < 0.8) + .map(MontagePlan.Shot::sourceStartSeconds).sorted().toList(); + assertThat(payoffs.get(0)).isBetween(8.0, 12.0); + assertThat(payoffs.get(1)).isBetween(38.0, 42.0); + } + + @Test + void intensityDecidesWhichSectionsMakeTheReelWhenTheJudgeCannotDiscriminate() { + // Continuous-motion content: the judge rates every moment the same (neutral 0.4, "riding"). A high + // intensity section (~10s) must make the reel; a much weaker one (~40s) must not. + int n = 120; + double[] motion = new double[n]; + double[] audio = new double[n]; + for (int i = 0; i < n; i++) { + double t = i * 0.5; + motion[i] = 2.0; + audio[i] = -30.0; + if (Math.abs(t - 10.0) < 0.8) { motion[i] = 10.0; audio[i] = -8.0; } // strong + if (Math.abs(t - 40.0) < 0.8) { motion[i] = 3.5; audio[i] = -26.0; } // weak + } + List candidates = director.candidatePeaks(motion, audio, 0.5, 60.0); + List judgements = judge(candidates, 0.4, "RIDING"); // model can't discriminate + + MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 60.0); + + double maxSrc = plan.shots().stream() .mapToDouble(s -> s.sourceStartSeconds() + s.durationSeconds() * s.speed()).max().orElse(0); - assertThat(maxSourceEnd).isGreaterThan(11.0); // outcome included - assertThat(maxSourceEnd).isLessThan(20.0); // not the far aftermath - assertThat(plan.shots().size()).isGreaterThanOrEqualTo(4); - assertThat(plan.grade()).isEqualTo("hero"); - assertThat(plan.musicDirection()).isNotBlank(); + double payoff = plan.shots().stream().filter(s -> s.speed() < 0.8) + .mapToDouble(MontagePlan.Shot::sourceStartSeconds).min().orElse(-1); + assertThat(payoff).isBetween(8.0, 12.0); // the strong section + assertThat(maxSrc).isLessThan(30.0); // the weak ~40s one excluded } @Test - void measuredFallbackUsesTheStrongestPeakWhenThereIsNoJudge() { - // With no vision judge, composeMontage builds around the strongest intensity peak (~45s here). + void mergesCandidatesThatWouldOverlapIntoOneSegment() { + // Two peaks only ~1.5s apart belong to the same action; the reel must not build two overlapping segments. + int n = 120; + double[] motion = new double[n]; + double[] audio = new double[n]; + for (int i = 0; i < n; i++) { + double t = i * 0.5; + motion[i] = 2.0; + audio[i] = -30.0; + if (Math.abs(t - 20.0) < 0.6 || Math.abs(t - 21.5) < 0.6) { motion[i] = 8.0; audio[i] = -10.0; } + } + List candidates = director.candidatePeaks(motion, audio, 0.5, 60.0); + List judgements = judge(candidates, 1.0, "MOMENT"); + + MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 60.0); + + long slowMo = plan.shots().stream().filter(s -> s.speed() < 0.8).count(); + assertThat(slowMo).isEqualTo(1); // merged into one segment + } + + @Test + void eachSegmentIncludesItsOutcomeAndTheBuildIsCapped() { int n = 128; double[] motion = new double[n]; double[] audio = new double[n]; for (int i = 0; i < n; i++) { double t = i * 0.5; motion[i] = 2.0; - if (Math.abs(t - 5.0) < 0.6) motion[i] = 6.0; - if (t >= 44.0 && t <= 46.0) motion[i] = 4.0; audio[i] = -30.0; - if (Math.abs(t - 45.0) < 0.6) audio[i] = -8.0; // strongest (loud + motion) + if (Math.abs(t - 5.0) < 0.6) motion[i] = 6.0; // early action spike (far from the peak) + if (t >= 44.0 && t <= 47.0) motion[i] = 4.0; // the action + its outcome + if (Math.abs(t - 45.0) < 0.6) audio[i] = -8.0; // the peak ~45s } + List candidates = List.of(45.0); + List judgements = List.of(new Judgement(1.0, "MOMENT")); - MontagePlan plan = director.composeMontage("p", "src.mp4", motion, audio, 0.5, 64.0); - - assertThat(payoffSourceStart(plan)).isBetween(44.0, 46.0); - } - - @Test - void capsThePreClimaxBuildOnLongSourcesSoNoSingleShotRunsAway() { - // Peak far from the pre-peak action spike; the build shot must be bounded by montage-max-build-seconds. - int n = 128; - double[] motion = new double[n]; - double[] audio = new double[n]; - for (int i = 0; i < n; i++) { - double t = i * 0.5; - motion[i] = 2.0; - if (Math.abs(t - 5.0) < 0.6) motion[i] = 6.0; // early action spike - if (t >= 44.0 && t <= 46.0) motion[i] = 4.0; - audio[i] = -30.0; - if (Math.abs(t - 45.0) < 0.6) audio[i] = -8.0; // strongest peak ~45s - } - - MontagePlan plan = director.composeMontageAt("p", "src.mp4", motion, audio, 45.0, 0.5, 64.0); + MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 64.0); double longestSpan = plan.shots().stream() .mapToDouble(s -> s.durationSeconds() * s.speed()).max().orElse(0); - assertThat(longestSpan).isLessThanOrEqualTo(6.5); - double total = plan.shots().stream().mapToDouble(MontagePlan.Shot::durationSeconds).sum(); - assertThat(total).isLessThanOrEqualTo(16.0); - assertThat(payoffSourceStart(plan)).isBetween(44.0, 46.0); + assertThat(longestSpan).isLessThanOrEqualTo(6.5); // build capped + double maxSrc = plan.shots().stream() + .mapToDouble(s -> s.sourceStartSeconds() + s.durationSeconds() * s.speed()).max().orElse(0); + assertThat(maxSrc).isGreaterThan(45.5); // outcome (~46-47s) included } @Test void fallsBackToAStraightCutForVeryShortSources() { - MontagePlan plan = director.composeMontage("p", "src.mp4", new double[]{1, 1}, new double[]{-30, -30}, - 0.5, 1.0); + MontagePlan plan = director.composeReel("p", "src.mp4", new double[]{1, 1}, new double[]{-30, -30}, + List.of(), null, 0.5, 1.0); assertThat(plan.shots()).hasSize(1); assertThat(plan.shots().get(0).sourceStartSeconds()).isEqualTo(0.0); } - private double payoffSourceStart(MontagePlan plan) { - return plan.shots().stream().filter(s -> s.speed() < 0.8).findFirst() - .map(MontagePlan.Shot::sourceStartSeconds).orElse(-1.0); + private static List judge(List candidates, double worthiness, String overlay) { + List out = new ArrayList<>(); + for (int i = 0; i < candidates.size(); i++) { + out.add(new Judgement(worthiness, overlay)); + } + return out; } }