forked from jsl/video_editing_poc
feat(director): multi-segment highlight reel with intensity-aware selection
The director produced ONE segment around a single chosen moment, so a video with several highlight-worthy actions got only one. Now it understands the whole clip and builds a REEL: - candidatePeaks proposes ALL intensity peaks across the clip (no cap), time-ordered. - The vision judge (judgeMoments) rates EVERY candidate in one batched pass and supplies an honest overlay per candidate (replaces the single-pick rankDecisiveMoment + the separate groundOverlay pass). - composeReel selects every candidate whose blended score clears the bar (0.6*judge-worthiness + 0.4*measured-intensity; the single best is always kept), builds an action segment (entry -> slow-mo peak -> exit) per selection, merges overlapping ones, and concatenates them into one video with one overlay per segment. - Intensity in the blend means continuous-motion clips (a downhill ride, where the model rates everything 'riding') still pick the most dynamic sections, not all of them. - Config: highlight-select-threshold (default 0.5). No cap on count or length. mvn verify: 295 tests green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
JSLMPR
parent
70dffaea6c
commit
5b98c0538f
|
|
@ -554,6 +554,13 @@ public class VideoClippingProperties {
|
|||
/** R14 subject-tracking reframe: follow the detected subject instead of a static centre crop. */
|
||||
private boolean subjectReframeEnabled = false;
|
||||
|
||||
/**
|
||||
* Highlight-reel selection bar (0..1). Every candidate whose blended score (0.6*judge-worthiness +
|
||||
* 0.4*action-intensity) clears this becomes its own segment in the final reel; the single best is
|
||||
* always kept. Lower = more segments. See {@code HighlightMontageDirector.composeReel}.
|
||||
*/
|
||||
private double highlightSelectThreshold = 0.5;
|
||||
|
||||
/** Offline subject-tracking tool (YOLO). YOLOv8 is AGPL-3.0 → non-commercial, like the CV worker. */
|
||||
private String subjectTrackScript = "./tools/subject_track.py";
|
||||
|
||||
|
|
@ -804,6 +811,14 @@ public class VideoClippingProperties {
|
|||
this.subjectReframeEnabled = subjectReframeEnabled;
|
||||
}
|
||||
|
||||
public double getHighlightSelectThreshold() {
|
||||
return highlightSelectThreshold;
|
||||
}
|
||||
|
||||
public void setHighlightSelectThreshold(double highlightSelectThreshold) {
|
||||
this.highlightSelectThreshold = highlightSelectThreshold;
|
||||
}
|
||||
|
||||
public String getSubjectTrackScript() {
|
||||
return subjectTrackScript;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -30,12 +30,13 @@ import java.util.List;
|
|||
public class HighlightMontageDirector {
|
||||
|
||||
static final double WINDOW_SECONDS = 0.5;
|
||||
/** How many candidate decisive moments to propose for the vision judge to rank. */
|
||||
static final int MAX_CANDIDATES = 5;
|
||||
/** Minimum spacing (seconds) between proposed candidates, so they are distinct moments. Kept short so a
|
||||
* reaction close on the heels of the action (a celebration right after the strike) stays a separate
|
||||
* candidate the vision judge can score, rather than being merged into the louder neighbouring peak. */
|
||||
static final double MIN_CANDIDATE_SEPARATION = 1.0;
|
||||
/** Selection-score blend: how the judge's meaning-rating and the measured action intensity combine. */
|
||||
static final double SELECT_SEMANTIC_WEIGHT = 0.6;
|
||||
static final double SELECT_INTENSITY_WEIGHT = 0.4;
|
||||
|
||||
private final VideoClippingProperties.Editing properties;
|
||||
|
||||
|
|
@ -43,45 +44,44 @@ public class HighlightMontageDirector {
|
|||
this.properties = properties.getEditing();
|
||||
}
|
||||
|
||||
/** Measure the source and compose a montage, choosing the decisive moment by measured intensity alone. */
|
||||
/** Measure the source and compose a reel, choosing decisive moments by measured intensity alone. */
|
||||
public MontagePlan direct(String projectId, String sourceFileName, Path source, double durationSeconds) {
|
||||
return direct(projectId, sourceFileName, source, durationSeconds, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Measure the source, propose candidate decisive moments (intensity peaks), and let {@code chooser} — the
|
||||
* Tier-2 vision judge — pick which candidate is the actual highlight. This is the generic rule: measurement
|
||||
* never decides the moment by position or threshold (a camera whip or a loud aftermath can out-score a
|
||||
* quiet celebration); it only PROPOSES, and the model JUDGES by meaning. When no chooser is supplied, or it
|
||||
* declines (NaN), the highest-intensity candidate is used as a measured fallback.
|
||||
* Compose a multi-segment HIGHLIGHT REEL. Measurement proposes candidate moments across the WHOLE clip
|
||||
* (intensity peaks); the Tier-2 vision {@code judge} rates each by meaning and supplies an overlay; the
|
||||
* director keeps EVERY moment worth showing (blending the judge's rating with measured action intensity,
|
||||
* so a dynamic section is kept even when the model can't name it), builds an action segment
|
||||
* (entry -> peak -> exit) around each, and concatenates them all into one video. No cap on how many
|
||||
* segments or how long. When no judge is supplied, selection is by measured intensity alone.
|
||||
*/
|
||||
public MontagePlan direct(String projectId, String sourceFileName, Path source, double durationSeconds,
|
||||
MomentChooser chooser) {
|
||||
MomentJudge judge) {
|
||||
double[] motion = probeCurve(source.toString(), true, durationSeconds);
|
||||
double[] audio = probeCurve(source.toString(), false, durationSeconds);
|
||||
List<Double> candidates = candidatePeaks(motion, audio, WINDOW_SECONDS, durationSeconds);
|
||||
double climax = Double.NaN;
|
||||
if (chooser != null && !candidates.isEmpty()) {
|
||||
climax = chooser.choose(List.copyOf(candidates));
|
||||
}
|
||||
if (!Double.isFinite(climax)) {
|
||||
climax = candidates.isEmpty() ? 0.0 : candidates.get(0); // measured fallback: strongest peak
|
||||
}
|
||||
return composeMontageAt(projectId, sourceFileName, motion, audio, climax, WINDOW_SECONDS, durationSeconds);
|
||||
List<Judgement> judgements = (judge != null && !candidates.isEmpty())
|
||||
? judge.judge(List.copyOf(candidates)) : null;
|
||||
return composeReel(projectId, sourceFileName, motion, audio, candidates, judgements,
|
||||
WINDOW_SECONDS, durationSeconds);
|
||||
}
|
||||
|
||||
/** Chooses which candidate moment (seconds) is the real highlight; returns NaN to defer to measurement. */
|
||||
/** Per-candidate verdict from the Tier-2 vision judge: how highlight-worthy (0..1) and a ready overlay. */
|
||||
public record Judgement(double worthiness, String overlayText) {
|
||||
}
|
||||
|
||||
/** Rates each candidate moment; returns one {@link Judgement} per candidate (same order). */
|
||||
@FunctionalInterface
|
||||
public interface MomentChooser {
|
||||
double choose(List<Double> candidateTimeSeconds);
|
||||
public interface MomentJudge {
|
||||
List<Judgement> judge(List<Double> candidateTimeSeconds);
|
||||
}
|
||||
|
||||
/**
|
||||
* Proposes candidate decisive moments as the local maxima of measured intensity (motion + audio),
|
||||
* strongest first, min-separated. GENERIC: no positional band, no per-video threshold — every real
|
||||
* event (a strike, a celebration, a turn-away, a goal, an anticipation) becomes a candidate, and the
|
||||
* vision judge decides which is the highlight. {@code motion[i]}/{@code audio[i]} cover the window at
|
||||
* {@code i * window} seconds.
|
||||
* min-separated, returned in TIME order. GENERIC: no positional band, no per-video threshold and NO cap on
|
||||
* how many — every real event becomes a candidate, and the judge + intensity decide which make the reel.
|
||||
*/
|
||||
List<Double> candidatePeaks(double[] motion, double[] audio, double window, double duration) {
|
||||
int n = Math.min(motion.length, audio.length);
|
||||
|
|
@ -117,52 +117,112 @@ public class HighlightMontageDirector {
|
|||
peaks.add(i);
|
||||
}
|
||||
}
|
||||
peaks.sort((x, y) -> Double.compare(intensity[y], intensity[x])); // strongest first
|
||||
peaks.sort((x, y) -> Double.compare(intensity[y], intensity[x])); // strongest first for dedup
|
||||
List<Double> chosen = new ArrayList<>();
|
||||
for (int idx : peaks) {
|
||||
double t = idx * window;
|
||||
boolean tooClose = chosen.stream().anyMatch(c -> Math.abs(c - t) < MIN_CANDIDATE_SEPARATION);
|
||||
if (!tooClose) {
|
||||
chosen.add(t);
|
||||
}
|
||||
if (chosen.size() >= MAX_CANDIDATES) {
|
||||
break;
|
||||
chosen.add(t); // no cap on how many candidates
|
||||
}
|
||||
}
|
||||
chosen.sort(Double::compareTo); // return in time order
|
||||
return chosen;
|
||||
}
|
||||
|
||||
MontagePlan composeMontage(String projectId, String sourceFileName, double[] motion, double[] audio,
|
||||
double window, double duration) {
|
||||
// Measured path (no vision judge): build around the strongest intensity peak.
|
||||
List<Double> candidates = candidatePeaks(motion, audio, window, duration);
|
||||
double climax = candidates.isEmpty() ? 0.0 : candidates.get(0);
|
||||
return composeMontageAt(projectId, sourceFileName, motion, audio, climax, window, duration);
|
||||
}
|
||||
|
||||
/**
|
||||
* Builds the ACTION SEGMENT (start -> peak -> outcome) around a chosen climax time. Pure and deterministic;
|
||||
* package-visible for unit testing without ffmpeg. The onset (start) and resolution (end) are MEASURED from
|
||||
* motion around the given peak — the peak itself is chosen by the caller (the vision judge).
|
||||
* Builds the multi-segment reel from the candidates. Each candidate's SELECTION SCORE blends the judge's
|
||||
* highlight-worthiness with the candidate's measured action intensity: {@code 0.6*worthiness +
|
||||
* 0.4*intensity}. Every candidate whose score clears the threshold becomes a segment (at least the best
|
||||
* one always does); each segment is an action unit (entry -> slow-mo peak -> exit) that never overlaps the
|
||||
* previous one. Pure and deterministic; package-visible for unit testing without ffmpeg.
|
||||
*/
|
||||
MontagePlan composeMontageAt(String projectId, String sourceFileName, double[] motion, double[] audio,
|
||||
double climaxTime, double window, double duration) {
|
||||
MontagePlan composeReel(String projectId, String sourceFileName, double[] motion, double[] audio,
|
||||
List<Double> candidates, List<Judgement> judgements, double window, double duration) {
|
||||
int n = Math.min(motion.length, audio.length);
|
||||
if (n < 4 || duration <= 0) {
|
||||
if (n < 4 || duration <= 0 || candidates == null || candidates.isEmpty()) {
|
||||
return straightCut(projectId, sourceFileName, duration);
|
||||
}
|
||||
double[] m = smooth(motion, n);
|
||||
double[] mN = normalize(m, n);
|
||||
double[] aN = normalize(smooth(audio, n), n);
|
||||
|
||||
// Score every candidate = 0.6 * (judge worthiness) + 0.4 * (measured intensity). When the model can't
|
||||
// discriminate (all "riding" -> neutral), intensity decides which sections are the real highlights.
|
||||
int k = candidates.size();
|
||||
double[] score = new double[k];
|
||||
for (int i = 0; i < k; i++) {
|
||||
int idx = Math.max(0, Math.min(n - 1, (int) Math.round(candidates.get(i) / window)));
|
||||
double intensity = (aN[idx] + mN[idx]) / 2.0; // 0..1
|
||||
double worthiness = (judgements != null && i < judgements.size())
|
||||
? judgements.get(i).worthiness() : 0.4; // neutral when no judge
|
||||
score[i] = SELECT_SEMANTIC_WEIGHT * worthiness + SELECT_INTENSITY_WEIGHT * intensity;
|
||||
}
|
||||
double bar = properties.getHighlightSelectThreshold();
|
||||
int best = 0;
|
||||
for (int i = 1; i < k; i++) {
|
||||
if (score[i] > score[best]) {
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
List<Integer> selected = new ArrayList<>();
|
||||
for (int i = 0; i < k; i++) {
|
||||
if (score[i] >= bar) {
|
||||
selected.add(i); // candidates are time-ordered
|
||||
}
|
||||
}
|
||||
if (selected.isEmpty()) {
|
||||
selected.add(best); // always at least the single best
|
||||
}
|
||||
|
||||
List<MontagePlan.Shot> allShots = new ArrayList<>();
|
||||
List<MontagePlan.Overlay> overlays = new ArrayList<>();
|
||||
double timeline = 0.0;
|
||||
double prevExit = Double.NEGATIVE_INFINITY;
|
||||
for (int i : selected) {
|
||||
Segment seg = buildSegment(m, candidates.get(i), window, duration, prevExit);
|
||||
if (seg == null) {
|
||||
continue; // overlapped or degenerate -> merged away
|
||||
}
|
||||
double preDur = 0;
|
||||
for (int s = 0; s < seg.payoffShotIndex(); s++) {
|
||||
preDur += seg.shots().get(s).durationSeconds();
|
||||
}
|
||||
double payoffTlStart = timeline + preDur;
|
||||
double payoffTlDur = seg.shots().get(seg.payoffShotIndex()).durationSeconds();
|
||||
allShots.addAll(seg.shots());
|
||||
for (MontagePlan.Shot sh : seg.shots()) {
|
||||
timeline += sh.durationSeconds();
|
||||
}
|
||||
String overlayText = (judgements != null && i < judgements.size()) ? judgements.get(i).overlayText() : "";
|
||||
if (overlayText != null && !overlayText.isBlank()) {
|
||||
overlays.add(new MontagePlan.Overlay(overlayText, round(payoffTlStart + 0.2),
|
||||
round(Math.max(payoffTlStart + 0.8, payoffTlStart + payoffTlDur - 0.2)), "lower_center_safe"));
|
||||
}
|
||||
prevExit = seg.exitSrc();
|
||||
}
|
||||
if (allShots.isEmpty()) {
|
||||
return straightCut(projectId, sourceFileName, duration);
|
||||
}
|
||||
return new MontagePlan(projectId, sourceFileName, "hero", genericMusic(), List.of(), overlays, allShots);
|
||||
}
|
||||
|
||||
private record Segment(List<MontagePlan.Shot> shots, double exitSrc, int payoffShotIndex) {
|
||||
}
|
||||
|
||||
/**
|
||||
* One action segment around a peak: MEASURED entry (build into the peak, capped) -> slow-mo payoff on the
|
||||
* peak -> MEASURED exit (motion settles). Returns null when the peak is already covered by the previous
|
||||
* segment (so segments never overlap) or the span is degenerate.
|
||||
*/
|
||||
private Segment buildSegment(double[] m, double climaxTime, double window, double duration, double prevExit) {
|
||||
int n = m.length;
|
||||
int climaxIdx = Math.max(1, Math.min(n - 1, (int) Math.round(climaxTime / window)));
|
||||
climaxTime = climaxIdx * window;
|
||||
|
||||
// Onset: the biggest motion spike before the peak (the release/approach); the action shot leads in.
|
||||
int actionIdx = argMax(m, Math.max(1, (int) Math.floor(0.05 * n)),
|
||||
Math.max(2, (int) Math.floor((climaxTime - 1.0) / window)));
|
||||
double actionTime = actionIdx * window;
|
||||
|
||||
// Resolution: extend past the peak through the outcome AND the immediate reaction (the pins falling and
|
||||
// the celebration; the ball settling in the net), stopping when motion settles back to baseline for a
|
||||
// sustained stretch, at a camera whip, or at a cap. This is MEASURED, not positional.
|
||||
if (climaxTime <= prevExit + 0.5) {
|
||||
return null; // this peak is inside the previous segment
|
||||
}
|
||||
// Exit: extend past the peak while motion stays elevated, stop when it settles / at a whip / at a cap.
|
||||
double whipLevel = Math.max(1e-6, percentile(m, n, 0.4) * 4.0);
|
||||
int whipCutoff = n;
|
||||
for (int i = climaxIdx + 2; i < n; i++) {
|
||||
|
|
@ -184,36 +244,40 @@ public class HighlightMontageDirector {
|
|||
resolution = t;
|
||||
if (m[i] <= settleLevel) {
|
||||
if (++settledRun >= 3) {
|
||||
break; // motion has settled -> the action has resolved
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
settledRun = 0;
|
||||
}
|
||||
}
|
||||
double endLimit = Math.min(duration, Math.max(climaxTime + 1.5, resolution + 0.3));
|
||||
|
||||
// Cap the pre-climax build. When the action spike is far from the climax (long source), the
|
||||
// action+tension shot would otherwise stretch to tens of seconds — one ultra-long continuous shot
|
||||
// that reads as dead air AND blows up score generation (music length == montage length). Bound the
|
||||
// build to montageMaxBuildSeconds so the highlight stays tight and generatable regardless of source
|
||||
// length. The build still leads INTO the climax; the setup shot precedes it.
|
||||
double actionEnd = climaxTime - 0.6;
|
||||
double desiredBuild = actionEnd - (actionTime - 1.5);
|
||||
double build = Math.min(Math.max(0.8, desiredBuild), Math.max(1.0, properties.getMontageMaxBuildSeconds()));
|
||||
double buildStart = Math.max(0.0, actionEnd - build);
|
||||
double exit = Math.min(duration, Math.max(climaxTime + 1.5, resolution + 0.3));
|
||||
// Entry: build INTO the peak, bounded by the cap and never earlier than the previous segment's exit.
|
||||
int actionIdx = argMax(m, Math.max(1, (int) Math.floor(0.05 * n)),
|
||||
Math.max(2, (int) Math.floor((climaxTime - 1.0) / window)));
|
||||
double actionTime = actionIdx * window;
|
||||
double payoffStart = climaxTime - 0.3;
|
||||
double desiredBuild = payoffStart - (actionTime - 1.0);
|
||||
double build = Math.min(Math.max(0.6, desiredBuild), Math.max(1.0, properties.getMontageMaxBuildSeconds()));
|
||||
double buildStart = Math.max(Math.max(0.0, prevExit), payoffStart - build);
|
||||
double payoffEnd = Math.min(climaxTime + 2.0, exit - 0.4);
|
||||
if (exit - payoffStart < 0.6) {
|
||||
return null; // too short to be a segment
|
||||
}
|
||||
if (payoffEnd < payoffStart + 0.5) {
|
||||
payoffEnd = Math.min(exit, payoffStart + 0.8);
|
||||
}
|
||||
|
||||
List<MontagePlan.Shot> shots = new ArrayList<>();
|
||||
addShot(shots, buildStart - 1.7, buildStart, 1.03, 1.0, duration); // setup
|
||||
addShot(shots, buildStart, actionEnd, 1.04, 1.0, duration); // action + tension (capped)
|
||||
addShot(shots, climaxTime - 0.6, climaxTime + 0.2, 1.06, 1.0, duration); // realization
|
||||
double payoffEnd = Math.min(climaxTime + 2.2, endLimit - 0.9);
|
||||
addShot(shots, climaxTime + 0.2, payoffEnd, 1.05, 0.7, duration); // slow-mo payoff
|
||||
addShot(shots, payoffEnd, endLimit, 1.04, 0.9, duration); // resolution button
|
||||
|
||||
if (shots.isEmpty()) {
|
||||
return straightCut(projectId, sourceFileName, duration);
|
||||
if (payoffStart - buildStart >= 0.4) {
|
||||
addShot(shots, buildStart, payoffStart, 1.04, 1.0, duration); // entry / build into the peak
|
||||
}
|
||||
return new MontagePlan(projectId, sourceFileName, "hero", genericMusic(), List.of(), List.of(), shots);
|
||||
int payoffShotIndex = shots.size();
|
||||
addShot(shots, payoffStart, payoffEnd, 1.05, 0.7, duration); // slow-mo payoff on the peak
|
||||
if (shots.size() <= payoffShotIndex) {
|
||||
return null; // payoff shot was degenerate
|
||||
}
|
||||
addShot(shots, payoffEnd, exit, 1.04, 0.9, duration); // exit / resolution
|
||||
return new Segment(shots, exit, payoffShotIndex);
|
||||
}
|
||||
|
||||
private MontagePlan straightCut(String projectId, String sourceFileName, double duration) {
|
||||
|
|
|
|||
|
|
@ -229,18 +229,13 @@ public class HighlightSourceScheduler {
|
|||
try {
|
||||
double duration = analysis.source() == null ? 0.0 : analysis.source().durationSeconds();
|
||||
Path visionWork = store.directorDirectory(projectId).resolve("vision-work");
|
||||
// GENERIC rule: measurement PROPOSES candidate decisive moments (intensity peaks); the vision judge
|
||||
// PICKS which is the real highlight by meaning (a celebration/goal beats a loud turn-away). No
|
||||
// positional band, no per-video threshold. Falls back to the strongest peak if the model declines.
|
||||
HighlightMontageDirector.MomentChooser chooser = properties.isVisionDirectorEnabled()
|
||||
? candidates -> visionDirector.rankDecisiveMoment(sourcePath, candidates, visionWork)
|
||||
// GENERIC rule: measurement PROPOSES candidate moments across the WHOLE clip; the vision judge rates
|
||||
// each by meaning and supplies an honest overlay; the director keeps EVERY worthy moment (blended
|
||||
// with action intensity) and builds a multi-segment reel. No band, no per-video threshold, no cap.
|
||||
HighlightMontageDirector.MomentJudge judge = properties.isVisionDirectorEnabled()
|
||||
? candidates -> visionDirector.judgeMoments(sourcePath, candidates, visionWork)
|
||||
: null;
|
||||
MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration, chooser);
|
||||
if (properties.isVisionDirectorEnabled()) {
|
||||
// Dense pass over the chosen decisive window (payoff shot through the outcome) with the honesty
|
||||
// rule: never assert an action the shown window doesn't contain (a "KICK" over a cut before it).
|
||||
montage = visionDirector.groundOverlay(montage, sourcePath, visionWork);
|
||||
}
|
||||
MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration, judge);
|
||||
store.writeJson(projectId, "director/montage.json", montage);
|
||||
// Minimal edit-plan.json so the render scanner selects the project; montage.json takes precedence.
|
||||
store.writeJson(projectId, "director/edit-plan.json",
|
||||
|
|
|
|||
|
|
@ -132,37 +132,25 @@ public class HighlightVisionDirector {
|
|||
}
|
||||
|
||||
/**
|
||||
* Densely captions the DECISIVE SOURCE WINDOW of a plan — the payoff shot through the final shot, i.e. the
|
||||
* action AND its outcome — and attaches an honest, grounded overlay plus scene-informed music. Unlike a
|
||||
* sparse whole-clip pass, this assesses several frames THROUGH the shown payoff, so the overlay reflects
|
||||
* what is actually on screen. The honesty rule ({@link #honestOverlayText}) never asserts an action the
|
||||
* window does not show: an anticipatory cut gets a teaser question instead. Fail-soft: on any error (model
|
||||
* missing, worker failure, timeout) it returns the plan unchanged, so the Tier-1 cut always stands.
|
||||
* The Tier-2 vision JUDGE for a highlight reel. Captions EVERY candidate moment in one batched pass (a
|
||||
* description to rate it, plus a punchy label + a teaser question for its overlay) and returns, per
|
||||
* candidate, its highlight-worthiness and a ready, honest overlay. A celebration or a scored goal rates
|
||||
* high; a loud turn-away or an "about to…" build rates low — meaning, not motion. The director blends these
|
||||
* ratings with measured intensity to decide which moments make the reel and how to label each. Fail-soft:
|
||||
* returns an empty list on any error, so the director selects by measured intensity alone.
|
||||
*/
|
||||
public MontagePlan groundOverlay(MontagePlan plan, Path source, Path workDir) {
|
||||
int payoffIdx = payoffShotIndex(plan);
|
||||
if (payoffIdx < 0 || plan.shots().isEmpty()) {
|
||||
return plan;
|
||||
}
|
||||
// Caption the CHOSEN PEAK itself — the payoff (slow-mo) shot — not the trailing outcome/button shots.
|
||||
// Sampling through to the last shot let a post-payoff reaction frame (e.g. the bowler lowering his arms
|
||||
// and holding his face after the celebration) supply the overlay; confining it to the payoff shot keeps
|
||||
// the label on the decisive moment the judge picked.
|
||||
MontagePlan.Shot payoffShot = plan.shots().get(payoffIdx);
|
||||
double winStart = payoffShot.sourceStartSeconds();
|
||||
double winEnd = payoffShot.sourceStartSeconds() + payoffShot.durationSeconds() * Math.max(0.05, payoffShot.speed());
|
||||
if (winEnd <= winStart) {
|
||||
winEnd = winStart + 1.0;
|
||||
public List<HighlightMontageDirector.Judgement> judgeMoments(Path source, List<Double> candidateTimes,
|
||||
Path workDir) {
|
||||
if (candidateTimes == null || candidateTimes.isEmpty()) {
|
||||
return List.of();
|
||||
}
|
||||
try {
|
||||
Files.createDirectories(workDir);
|
||||
int samples = 3;
|
||||
List<Manifest> manifest = new ArrayList<>();
|
||||
List<Integer> ids = new ArrayList<>();
|
||||
for (int i = 0; i < samples; i++) {
|
||||
double t = winStart + (winEnd - winStart) * (i + 0.5) / samples;
|
||||
Path frame = workDir.resolve("payoff-" + i + ".jpg");
|
||||
if (extractFrame(source, t, frame)) {
|
||||
for (int i = 0; i < candidateTimes.size(); i++) {
|
||||
Path frame = workDir.resolve("cand-" + i + ".jpg");
|
||||
if (extractFrame(source, candidateTimes.get(i), frame)) {
|
||||
String img = frame.toAbsolutePath().toString();
|
||||
manifest.add(new Manifest("d" + i, img, DESCRIPTIVE_QUESTION));
|
||||
manifest.add(new Manifest("l" + i, img,
|
||||
|
|
@ -173,94 +161,29 @@ public class HighlightVisionDirector {
|
|||
}
|
||||
}
|
||||
if (ids.isEmpty()) {
|
||||
return plan;
|
||||
return List.of();
|
||||
}
|
||||
List<Caption> caps = runCaptioner(workDir, manifest);
|
||||
// Pick the frame whose description most shows the decisive action; later frames win ties so the
|
||||
// window's climax is preferred over its lead-in.
|
||||
int best = ids.get(0);
|
||||
double bestScore = -1;
|
||||
for (int i : ids) {
|
||||
double sc = semanticScore(answerFor(caps, "d" + i));
|
||||
if (sc >= bestScore) {
|
||||
bestScore = sc;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
String description = answerFor(caps, "d" + best);
|
||||
String overlayText = honestOverlayText(description, answerFor(caps, "l" + best),
|
||||
answerFor(caps, "q" + best));
|
||||
List<MontagePlan.Overlay> overlays = new ArrayList<>(
|
||||
plan.overlays() == null ? List.of() : plan.overlays());
|
||||
if (!overlayText.isBlank()) {
|
||||
double[] span = payoffTimeline(plan, payoffIdx);
|
||||
overlays.add(new MontagePlan.Overlay(overlayText, span[0] + 0.2,
|
||||
Math.max(span[0] + 0.8, span[1] - 0.2), "lower_center_safe"));
|
||||
}
|
||||
String music = description.isBlank() ? plan.musicDirection()
|
||||
: "cinematic film score for this moment: " + description + " Build quiet tension to a "
|
||||
+ "triumphant climax hit at the payoff, then a short warm resolve, modern trailer score, no vocals.";
|
||||
log.info("event=highlight_vision_director_completed overlay=\"{}\" payoff_description=\"{}\"",
|
||||
overlayText, description);
|
||||
return new MontagePlan(plan.projectId(), plan.sourceVideoFileName(), plan.grade(), music,
|
||||
plan.voiceover(), overlays, plan.shots());
|
||||
} catch (RuntimeException | IOException | InterruptedException ex) {
|
||||
if (ex instanceof InterruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
log.warn("event=highlight_vision_director_failed stage=overlay error_type={} message={}",
|
||||
ex.getClass().getSimpleName(), ex.getMessage());
|
||||
return plan;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The Tier-2 vision JUDGE. Given candidate decisive moments proposed by measurement, it captions each and
|
||||
* returns the time of the most highlight-worthy one — a celebration or a scored goal beats a loud
|
||||
* camera turn-away or an anticipatory build, because "is this THE highlight?" is a question of meaning, not
|
||||
* of motion or loudness. Returns NaN on any failure so the director falls back to the strongest peak.
|
||||
*/
|
||||
public double rankDecisiveMoment(Path source, List<Double> candidateTimes, Path workDir) {
|
||||
if (candidateTimes == null || candidateTimes.isEmpty()) {
|
||||
return Double.NaN;
|
||||
}
|
||||
try {
|
||||
Files.createDirectories(workDir);
|
||||
List<Manifest> manifest = new ArrayList<>();
|
||||
List<Integer> ids = new ArrayList<>();
|
||||
List<HighlightMontageDirector.Judgement> out = new ArrayList<>();
|
||||
for (int i = 0; i < candidateTimes.size(); i++) {
|
||||
Path frame = workDir.resolve("cand-" + i + ".jpg");
|
||||
if (extractFrame(source, candidateTimes.get(i), frame)) {
|
||||
manifest.add(new Manifest("d" + i, frame.toAbsolutePath().toString(), DESCRIPTIVE_QUESTION));
|
||||
ids.add(i);
|
||||
if (!ids.contains(i)) {
|
||||
out.add(new HighlightMontageDirector.Judgement(0.0, "")); // unreadable frame
|
||||
continue;
|
||||
}
|
||||
String description = answerFor(caps, "d" + i);
|
||||
double worthiness = highlightWorthiness(description);
|
||||
String overlay = honestOverlayText(description, answerFor(caps, "l" + i), answerFor(caps, "q" + i));
|
||||
out.add(new HighlightMontageDirector.Judgement(worthiness, overlay));
|
||||
}
|
||||
if (ids.isEmpty()) {
|
||||
return Double.NaN;
|
||||
}
|
||||
List<Caption> caps = runCaptioner(workDir, manifest);
|
||||
int best = -1;
|
||||
double bestScore = -1;
|
||||
for (int i : ids) {
|
||||
double score = highlightWorthiness(answerFor(caps, "d" + i));
|
||||
if (score > bestScore) { // strict >: ties keep the stronger-intensity (earlier) candidate
|
||||
bestScore = score;
|
||||
best = i;
|
||||
}
|
||||
}
|
||||
if (best < 0) {
|
||||
return Double.NaN;
|
||||
}
|
||||
log.info("event=highlight_vision_moment_ranked candidates={} chosen_time={} score={}",
|
||||
ids.size(), candidateTimes.get(best), bestScore);
|
||||
return candidateTimes.get(best);
|
||||
log.info("event=highlight_vision_moments_judged candidates={} rated={}", candidateTimes.size(), ids.size());
|
||||
return out;
|
||||
} catch (RuntimeException | IOException | InterruptedException ex) {
|
||||
if (ex instanceof InterruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
log.warn("event=highlight_vision_director_failed stage=rank error_type={} message={}",
|
||||
log.warn("event=highlight_vision_director_failed stage=judge error_type={} message={}",
|
||||
ex.getClass().getSimpleName(), ex.getMessage());
|
||||
return Double.NaN;
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,8 +1,10 @@
|
|||
package org.example.videoclips.editing;
|
||||
|
||||
import org.example.videoclips.config.VideoClippingProperties;
|
||||
import org.example.videoclips.editing.HighlightMontageDirector.Judgement;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
|
@ -12,116 +14,148 @@ class HighlightMontageDirectorTest {
|
|||
private final HighlightMontageDirector director = new HighlightMontageDirector(new VideoClippingProperties());
|
||||
|
||||
@Test
|
||||
void proposesEveryRealMomentAsACandidateWithoutBias() {
|
||||
// 64s source: an early goal (~8s) AND a late, HIGHER-intensity aftermath (~55s: loud + high motion).
|
||||
// Measurement must propose BOTH as candidates — it does not get to decide which is the highlight
|
||||
// (that is the vision judge's job). Note the aftermath ranks first: measurement alone would pick it.
|
||||
int n = 128;
|
||||
void proposesEveryRealMomentAsACandidateInTimeOrderWithNoCap() {
|
||||
// Several distinct action bursts across a long clip: measurement must propose them ALL (no cap), in
|
||||
// time order. It has no opinion on which is the highlight — that is the judge's job.
|
||||
int n = 160;
|
||||
double[] motion = new double[n];
|
||||
double[] audio = new double[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
double t = i * 0.5;
|
||||
motion[i] = 2.0;
|
||||
if (t >= 7.0 && t <= 10.0) motion[i] = 7.0; // the goal
|
||||
if (t >= 54.0 && t <= 56.0) motion[i] = 8.0; // the aftermath (higher motion)
|
||||
audio[i] = -30.0;
|
||||
if (t >= 8.0 && t <= 11.0) audio[i] = -10.0; // goal reaction
|
||||
if (t >= 54.0 && t <= 56.0) audio[i] = -12.0; // aftermath (loud)
|
||||
for (double peak : new double[]{8, 22, 41, 60}) {
|
||||
if (Math.abs(t - peak) < 0.6) {
|
||||
motion[i] = 7.0;
|
||||
audio[i] = -12.0;
|
||||
}
|
||||
}
|
||||
}
|
||||
List<Double> candidates = director.candidatePeaks(motion, audio, 0.5, 80.0);
|
||||
|
||||
List<Double> candidates = director.candidatePeaks(motion, audio, 0.5, 64.0);
|
||||
|
||||
assertThat(candidates).anySatisfy(t -> assertThat(t).isBetween(6.0, 11.0)); // the goal is a candidate
|
||||
assertThat(candidates).anySatisfy(t -> assertThat(t).isBetween(53.0, 57.0)); // the aftermath too
|
||||
// Distinct moments (min-separated), strongest first -> the aftermath, which a naive pick would take.
|
||||
assertThat(candidates.get(0)).isBetween(53.0, 57.0);
|
||||
assertThat(candidates).hasSizeGreaterThanOrEqualTo(4);
|
||||
assertThat(candidates).isSorted(); // time order
|
||||
for (double peak : new double[]{8, 22, 41, 60}) {
|
||||
assertThat(candidates).anySatisfy(t -> assertThat(t).isCloseTo(peak, org.assertj.core.api.Assertions.within(1.0)));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void buildsTheActionSegmentAroundTheCHOSENPeakAndSweepsInTheOutcome() {
|
||||
// The judge chose the early goal (~8s), NOT the louder late aftermath. Given that choice, the segment
|
||||
// must place the slow-mo payoff on the goal and extend the window through its outcome (ball settling).
|
||||
int n = 128;
|
||||
void buildsAMultiSegmentReelFromEveryWorthyMoment() {
|
||||
// Two DISTINCT worthy actions, far apart (~10s and ~40s). Both must become segments in ONE reel, each
|
||||
// with its own slow-mo payoff and its own overlay.
|
||||
int n = 120;
|
||||
double[] motion = new double[n];
|
||||
double[] audio = new double[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
double t = i * 0.5;
|
||||
motion[i] = 2.0;
|
||||
if (t >= 7.0 && t <= 10.0) motion[i] = 7.0; // the goal (peak)
|
||||
if (t > 10.0 && t <= 13.0) motion[i] = 4.0; // the ball settling in the net (outcome)
|
||||
if (t >= 54.0 && t <= 56.0) motion[i] = 8.0; // aftermath elsewhere (not chosen)
|
||||
audio[i] = -30.0;
|
||||
if (t >= 8.0 && t <= 11.0) audio[i] = -10.0;
|
||||
if (Math.abs(t - 10.0) < 0.8) { motion[i] = 8.0; audio[i] = -10.0; }
|
||||
if (Math.abs(t - 40.0) < 0.8) { motion[i] = 8.0; audio[i] = -10.0; }
|
||||
}
|
||||
List<Double> candidates = director.candidatePeaks(motion, audio, 0.5, 60.0);
|
||||
List<Judgement> judgements = judge(candidates, 1.0, "GOAL"); // both worthy
|
||||
|
||||
MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 60.0);
|
||||
|
||||
long slowMo = plan.shots().stream().filter(s -> s.speed() < 0.8).count();
|
||||
assertThat(slowMo).isEqualTo(2); // two payoffs -> two segments
|
||||
assertThat(plan.overlays()).hasSize(2); // one overlay per segment
|
||||
// The two payoffs sit on the two actions (~10s and ~40s of source).
|
||||
List<Double> payoffs = plan.shots().stream().filter(s -> s.speed() < 0.8)
|
||||
.map(MontagePlan.Shot::sourceStartSeconds).sorted().toList();
|
||||
assertThat(payoffs.get(0)).isBetween(8.0, 12.0);
|
||||
assertThat(payoffs.get(1)).isBetween(38.0, 42.0);
|
||||
}
|
||||
|
||||
MontagePlan plan = director.composeMontageAt("p", "src.mp4", motion, audio, 8.5, 0.5, 64.0);
|
||||
@Test
|
||||
void intensityDecidesWhichSectionsMakeTheReelWhenTheJudgeCannotDiscriminate() {
|
||||
// Continuous-motion content: the judge rates every moment the same (neutral 0.4, "riding"). A high
|
||||
// intensity section (~10s) must make the reel; a much weaker one (~40s) must not.
|
||||
int n = 120;
|
||||
double[] motion = new double[n];
|
||||
double[] audio = new double[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
double t = i * 0.5;
|
||||
motion[i] = 2.0;
|
||||
audio[i] = -30.0;
|
||||
if (Math.abs(t - 10.0) < 0.8) { motion[i] = 10.0; audio[i] = -8.0; } // strong
|
||||
if (Math.abs(t - 40.0) < 0.8) { motion[i] = 3.5; audio[i] = -26.0; } // weak
|
||||
}
|
||||
List<Double> candidates = director.candidatePeaks(motion, audio, 0.5, 60.0);
|
||||
List<Judgement> judgements = judge(candidates, 0.4, "RIDING"); // model can't discriminate
|
||||
|
||||
double payoff = payoffSourceStart(plan);
|
||||
assertThat(payoff).isBetween(7.5, 10.0); // slow-mo on the goal
|
||||
double maxSourceEnd = plan.shots().stream()
|
||||
MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 60.0);
|
||||
|
||||
double maxSrc = plan.shots().stream()
|
||||
.mapToDouble(s -> s.sourceStartSeconds() + s.durationSeconds() * s.speed()).max().orElse(0);
|
||||
assertThat(maxSourceEnd).isGreaterThan(11.0); // outcome included
|
||||
assertThat(maxSourceEnd).isLessThan(20.0); // not the far aftermath
|
||||
assertThat(plan.shots().size()).isGreaterThanOrEqualTo(4);
|
||||
assertThat(plan.grade()).isEqualTo("hero");
|
||||
assertThat(plan.musicDirection()).isNotBlank();
|
||||
double payoff = plan.shots().stream().filter(s -> s.speed() < 0.8)
|
||||
.mapToDouble(MontagePlan.Shot::sourceStartSeconds).min().orElse(-1);
|
||||
assertThat(payoff).isBetween(8.0, 12.0); // the strong section
|
||||
assertThat(maxSrc).isLessThan(30.0); // the weak ~40s one excluded
|
||||
}
|
||||
|
||||
@Test
|
||||
void measuredFallbackUsesTheStrongestPeakWhenThereIsNoJudge() {
|
||||
// With no vision judge, composeMontage builds around the strongest intensity peak (~45s here).
|
||||
void mergesCandidatesThatWouldOverlapIntoOneSegment() {
|
||||
// Two peaks only ~1.5s apart belong to the same action; the reel must not build two overlapping segments.
|
||||
int n = 120;
|
||||
double[] motion = new double[n];
|
||||
double[] audio = new double[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
double t = i * 0.5;
|
||||
motion[i] = 2.0;
|
||||
audio[i] = -30.0;
|
||||
if (Math.abs(t - 20.0) < 0.6 || Math.abs(t - 21.5) < 0.6) { motion[i] = 8.0; audio[i] = -10.0; }
|
||||
}
|
||||
List<Double> candidates = director.candidatePeaks(motion, audio, 0.5, 60.0);
|
||||
List<Judgement> judgements = judge(candidates, 1.0, "MOMENT");
|
||||
|
||||
MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 60.0);
|
||||
|
||||
long slowMo = plan.shots().stream().filter(s -> s.speed() < 0.8).count();
|
||||
assertThat(slowMo).isEqualTo(1); // merged into one segment
|
||||
}
|
||||
|
||||
@Test
|
||||
void eachSegmentIncludesItsOutcomeAndTheBuildIsCapped() {
|
||||
int n = 128;
|
||||
double[] motion = new double[n];
|
||||
double[] audio = new double[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
double t = i * 0.5;
|
||||
motion[i] = 2.0;
|
||||
if (Math.abs(t - 5.0) < 0.6) motion[i] = 6.0;
|
||||
if (t >= 44.0 && t <= 46.0) motion[i] = 4.0;
|
||||
audio[i] = -30.0;
|
||||
if (Math.abs(t - 45.0) < 0.6) audio[i] = -8.0; // strongest (loud + motion)
|
||||
if (Math.abs(t - 5.0) < 0.6) motion[i] = 6.0; // early action spike (far from the peak)
|
||||
if (t >= 44.0 && t <= 47.0) motion[i] = 4.0; // the action + its outcome
|
||||
if (Math.abs(t - 45.0) < 0.6) audio[i] = -8.0; // the peak ~45s
|
||||
}
|
||||
List<Double> candidates = List.of(45.0);
|
||||
List<Judgement> judgements = List.of(new Judgement(1.0, "MOMENT"));
|
||||
|
||||
MontagePlan plan = director.composeMontage("p", "src.mp4", motion, audio, 0.5, 64.0);
|
||||
|
||||
assertThat(payoffSourceStart(plan)).isBetween(44.0, 46.0);
|
||||
}
|
||||
|
||||
@Test
|
||||
void capsThePreClimaxBuildOnLongSourcesSoNoSingleShotRunsAway() {
|
||||
// Peak far from the pre-peak action spike; the build shot must be bounded by montage-max-build-seconds.
|
||||
int n = 128;
|
||||
double[] motion = new double[n];
|
||||
double[] audio = new double[n];
|
||||
for (int i = 0; i < n; i++) {
|
||||
double t = i * 0.5;
|
||||
motion[i] = 2.0;
|
||||
if (Math.abs(t - 5.0) < 0.6) motion[i] = 6.0; // early action spike
|
||||
if (t >= 44.0 && t <= 46.0) motion[i] = 4.0;
|
||||
audio[i] = -30.0;
|
||||
if (Math.abs(t - 45.0) < 0.6) audio[i] = -8.0; // strongest peak ~45s
|
||||
}
|
||||
|
||||
MontagePlan plan = director.composeMontageAt("p", "src.mp4", motion, audio, 45.0, 0.5, 64.0);
|
||||
MontagePlan plan = director.composeReel("p", "s.mp4", motion, audio, candidates, judgements, 0.5, 64.0);
|
||||
|
||||
double longestSpan = plan.shots().stream()
|
||||
.mapToDouble(s -> s.durationSeconds() * s.speed()).max().orElse(0);
|
||||
assertThat(longestSpan).isLessThanOrEqualTo(6.5);
|
||||
double total = plan.shots().stream().mapToDouble(MontagePlan.Shot::durationSeconds).sum();
|
||||
assertThat(total).isLessThanOrEqualTo(16.0);
|
||||
assertThat(payoffSourceStart(plan)).isBetween(44.0, 46.0);
|
||||
assertThat(longestSpan).isLessThanOrEqualTo(6.5); // build capped
|
||||
double maxSrc = plan.shots().stream()
|
||||
.mapToDouble(s -> s.sourceStartSeconds() + s.durationSeconds() * s.speed()).max().orElse(0);
|
||||
assertThat(maxSrc).isGreaterThan(45.5); // outcome (~46-47s) included
|
||||
}
|
||||
|
||||
@Test
|
||||
void fallsBackToAStraightCutForVeryShortSources() {
|
||||
MontagePlan plan = director.composeMontage("p", "src.mp4", new double[]{1, 1}, new double[]{-30, -30},
|
||||
0.5, 1.0);
|
||||
MontagePlan plan = director.composeReel("p", "src.mp4", new double[]{1, 1}, new double[]{-30, -30},
|
||||
List.of(), null, 0.5, 1.0);
|
||||
assertThat(plan.shots()).hasSize(1);
|
||||
assertThat(plan.shots().get(0).sourceStartSeconds()).isEqualTo(0.0);
|
||||
}
|
||||
|
||||
private double payoffSourceStart(MontagePlan plan) {
|
||||
return plan.shots().stream().filter(s -> s.speed() < 0.8).findFirst()
|
||||
.map(MontagePlan.Shot::sourceStartSeconds).orElse(-1.0);
|
||||
private static List<Judgement> judge(List<Double> candidates, double worthiness, String overlay) {
|
||||
List<Judgement> out = new ArrayList<>();
|
||||
for (int i = 0; i < candidates.size(); i++) {
|
||||
out.add(new Judgement(worthiness, overlay));
|
||||
}
|
||||
return out;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue