fix(director): select the decisive moment, include its outcome, honest overlay
The highlight could cut off before the actual action and then assert an action that was never shown (a "KICK" overlay over a stop-before-the-kick cut). Root causes, all generic and fixed here: - Localization (A): the climax search only scanned the central 25-80% band on audio(+semantic), structurally excluding a late payoff and often landing on the anticipation. Now it searches nearly the whole clip and localizes primarily by MOTION (the decisive action is a motion event), audio equal, semantic a weak tie-breaker, with a robust percentile-based camera-whip guard. - Outcome (B): the window ended at a fixed climax+offset. It now extends past the climax until motion settles (the action AND its result), whip-guarded and capped. - Honest overlay (C): a DENSE pass over the shown payoff window (not 7 sparse whole-clip frames) picks the best action frame; an honesty rule never asserts an action the window does not show -- anticipation gets a grounded teaser question instead (HighlightVisionDirector.groundOverlay/honestOverlayText/isAnticipatory). The VLM no longer picks the moment (it mislocalized onto anticipation), only grounds the overlay. - Pacing cap: montage-max-build-seconds bounds the single pre-climax build shot so a distant action spike on a long source can't create one runaway shot (dead air plus an impractically long generated score). Also: re-ingesting a source whose name already exists in processed/ no longer fails -- moveToDirectory picks a unique "<name>-<n>.<ext>" instead of refusing. Verified end-to-end on a new 63s landscape soccer clip: the director now finds the scoring kick (ball in net), keeps the outcome, honestly labels "KICK", 24fps/420p/ -16 LUFS, beat-synced, subject-followed. mvn verify: 292 tests green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
parent
9e401870a1
commit
8b00b41d7c
|
|
@ -563,6 +563,13 @@ public class VideoClippingProperties {
|
||||||
/** Frames sampled per shot for subject tracking (more = smoother path, slower). */
|
/** Frames sampled per shot for subject tracking (more = smoother path, slower). */
|
||||||
private int subjectTrackSamples = 5;
|
private int subjectTrackSamples = 5;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Max length (seconds) of the montage's single pre-climax "action/tension" build shot. Caps the case
|
||||||
|
* where a distant action spike on a long source produces one ultra-long continuous shot (dead air +
|
||||||
|
* an impractically long generated score). Pacing bound, not a total-duration constraint.
|
||||||
|
*/
|
||||||
|
private double montageMaxBuildSeconds = 6.0;
|
||||||
|
|
||||||
@Min(1)
|
@Min(1)
|
||||||
private int audioSampleRate = 48000;
|
private int audioSampleRate = 48000;
|
||||||
|
|
||||||
|
|
@ -814,6 +821,14 @@ public class VideoClippingProperties {
|
||||||
this.subjectTrackSamples = subjectTrackSamples;
|
this.subjectTrackSamples = subjectTrackSamples;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public double getMontageMaxBuildSeconds() {
|
||||||
|
return montageMaxBuildSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setMontageMaxBuildSeconds(double montageMaxBuildSeconds) {
|
||||||
|
this.montageMaxBuildSeconds = montageMaxBuildSeconds;
|
||||||
|
}
|
||||||
|
|
||||||
public int getAudioSampleRate() {
|
public int getAudioSampleRate() {
|
||||||
return audioSampleRate;
|
return audioSampleRate;
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -71,20 +71,35 @@ public class HighlightMontageDirector {
|
||||||
double[] m = smooth(motion, n);
|
double[] m = smooth(motion, n);
|
||||||
double[] a = smooth(audio, n);
|
double[] a = smooth(audio, n);
|
||||||
|
|
||||||
// Climax = the peak of the "payoff" signal in the central band (skip the intro and the messy tail).
|
// Localize the DECISIVE MOMENT. The old code searched only the central 25-80% band on audio(+semantic),
|
||||||
// Base signal is loud sustained audio; when a Tier-2 semantic curve is present (vision captions scored
|
// which structurally EXCLUDED a payoff in the last fifth of a clip (a late kick/goal) and could land on
|
||||||
// for highlight-worthiness), blend it in so the payoff lands on the moment that is both loud AND
|
// the anticipation rather than the action. Now: search almost the whole clip and localize primarily by
|
||||||
// semantically the highlight (e.g. the celebration), not merely the loudest sound.
|
// MOTION — the decisive action is a motion event — with audio an equal partner and any semantic curve a
|
||||||
double[] payoffSignal = a;
|
// weak tie-breaker. Motion catches pure-action payoffs (a kick); audio catches reaction payoffs (a
|
||||||
if (semantic != null && semantic.length >= n) {
|
// celebration cheer). This is the generic fix for "the highlight cut off before the actual action".
|
||||||
double[] normAudio = normalize(a, n);
|
double[] mN = normalize(m, n);
|
||||||
double[] normSemantic = normalize(semantic, n);
|
double[] aN = normalize(a, n);
|
||||||
payoffSignal = new double[n];
|
boolean haveSemantic = semantic != null && semantic.length >= n;
|
||||||
|
double[] sN = haveSemantic ? normalize(semantic, n) : null;
|
||||||
|
double[] payoffSignal = new double[n];
|
||||||
for (int i = 0; i < n; i++) {
|
for (int i = 0; i < n; i++) {
|
||||||
payoffSignal[i] = normAudio[i] + 0.9 * normSemantic[i];
|
payoffSignal[i] = aN[i] + 0.6 * mN[i] + (haveSemantic ? 0.9 * sN[i] : 0.0);
|
||||||
|
}
|
||||||
|
// Exclude a late CAMERA WHIP (a brief motion spike far above the clip's own motion level — the hand-held
|
||||||
|
// pan after the action) from both the climax search and the window. The baseline is the 40th-percentile
|
||||||
|
// motion (robust to the action spikes themselves), so widening the search to nearly the whole clip can't
|
||||||
|
// land the payoff on the messy aftermath — while a genuine late action (well below whip level) stays in.
|
||||||
|
double whipLevel = Math.max(1e-6, percentile(m, n, 0.4) * 4.0);
|
||||||
|
int whipCutoff = n;
|
||||||
|
for (int i = (int) Math.floor(0.55 * n); i < n; i++) {
|
||||||
|
if (m[i] > whipLevel) {
|
||||||
|
whipCutoff = i;
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
int climaxIdx = argMax(payoffSignal, (int) Math.floor(0.25 * n), (int) Math.ceil(0.80 * n));
|
int lo = (int) Math.floor(0.08 * n);
|
||||||
|
int searchHi = Math.min((int) Math.ceil(0.95 * n), whipCutoff);
|
||||||
|
int climaxIdx = argMax(payoffSignal, lo, Math.max(lo + 1, searchHi));
|
||||||
double climaxTime = climaxIdx * window;
|
double climaxTime = climaxIdx * window;
|
||||||
|
|
||||||
// Action = the biggest motion spike before the climax; the action shot leads in ~1.5s before it so
|
// Action = the biggest motion spike before the climax; the action shot leads in ~1.5s before it so
|
||||||
|
|
@ -93,22 +108,43 @@ public class HighlightMontageDirector {
|
||||||
Math.max(2, (int) Math.floor((climaxTime - 1.0) / window)));
|
Math.max(2, (int) Math.floor((climaxTime - 1.0) / window)));
|
||||||
double actionTime = actionIdx * window;
|
double actionTime = actionIdx * window;
|
||||||
|
|
||||||
// Trim a high-motion tail (camera whip): the first strong post-climax motion rise well above the
|
// Resolution: the highlight must show the OUTCOME, not cut off at the action. Extend past the climax
|
||||||
// celebration's own motion level.
|
// while motion stays elevated (the action and its result, e.g. the ball rolling away), and stop when
|
||||||
|
// motion SETTLES back near baseline for a sustained stretch, at the whip, or at a cap.
|
||||||
double baseMotion = mean(m, 0, Math.max(1, climaxIdx));
|
double baseMotion = mean(m, 0, Math.max(1, climaxIdx));
|
||||||
double tailThreshold = baseMotion * 2.4;
|
double settleLevel = baseMotion * 1.4;
|
||||||
double tailStart = duration;
|
double maxResolution = climaxTime + 6.0;
|
||||||
for (int i = climaxIdx + 4; i < n; i++) {
|
double resolution = climaxTime + 1.0;
|
||||||
if (m[i] > tailThreshold) {
|
int settledRun = 0;
|
||||||
tailStart = i * window;
|
for (int i = climaxIdx + 1; i < Math.min(n, whipCutoff); i++) {
|
||||||
|
double t = i * window;
|
||||||
|
if (t > maxResolution) {
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
|
resolution = t;
|
||||||
|
if (m[i] <= settleLevel) {
|
||||||
|
if (++settledRun >= 3) {
|
||||||
|
break; // motion has settled -> the action has resolved
|
||||||
}
|
}
|
||||||
double endLimit = Math.min(Math.min(duration, tailStart), climaxTime + 4.5);
|
} else {
|
||||||
|
settledRun = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
double endLimit = Math.min(duration, Math.max(climaxTime + 1.5, resolution + 0.3));
|
||||||
|
|
||||||
|
// Cap the pre-climax build. When the action spike is far from the climax (long source), the
|
||||||
|
// action+tension shot would otherwise stretch to tens of seconds — one ultra-long continuous shot
|
||||||
|
// that reads as dead air AND blows up score generation (music length == montage length). Bound the
|
||||||
|
// build to montageMaxBuildSeconds so the highlight stays tight and generatable regardless of source
|
||||||
|
// length. The build still leads INTO the climax; the setup shot precedes it.
|
||||||
|
double actionEnd = climaxTime - 0.6;
|
||||||
|
double desiredBuild = actionEnd - (actionTime - 1.5);
|
||||||
|
double build = Math.min(Math.max(0.8, desiredBuild), Math.max(1.0, properties.getMontageMaxBuildSeconds()));
|
||||||
|
double buildStart = Math.max(0.0, actionEnd - build);
|
||||||
|
|
||||||
List<MontagePlan.Shot> shots = new ArrayList<>();
|
List<MontagePlan.Shot> shots = new ArrayList<>();
|
||||||
addShot(shots, actionTime - 3.2, actionTime - 1.5, 1.03, 1.0, duration); // setup
|
addShot(shots, buildStart - 1.7, buildStart, 1.03, 1.0, duration); // setup
|
||||||
addShot(shots, actionTime - 1.5, climaxTime - 0.6, 1.04, 1.0, duration); // action + tension
|
addShot(shots, buildStart, actionEnd, 1.04, 1.0, duration); // action + tension (capped)
|
||||||
addShot(shots, climaxTime - 0.6, climaxTime + 0.2, 1.06, 1.0, duration); // realization
|
addShot(shots, climaxTime - 0.6, climaxTime + 0.2, 1.06, 1.0, duration); // realization
|
||||||
double payoffEnd = Math.min(climaxTime + 2.2, endLimit - 0.9);
|
double payoffEnd = Math.min(climaxTime + 2.2, endLimit - 0.9);
|
||||||
addShot(shots, climaxTime + 0.2, payoffEnd, 1.05, 0.7, duration); // slow-mo payoff
|
addShot(shots, climaxTime + 0.2, payoffEnd, 1.05, 0.7, duration); // slow-mo payoff
|
||||||
|
|
@ -245,6 +281,14 @@ public class HighlightMontageDirector {
|
||||||
return best;
|
return best;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The {@code p}-quantile (0..1) of the first {@code n} values — a robust, spike-resistant level. */
|
||||||
|
private static double percentile(double[] values, int n, double p) {
|
||||||
|
double[] copy = java.util.Arrays.copyOf(values, n);
|
||||||
|
java.util.Arrays.sort(copy);
|
||||||
|
int idx = Math.min(n - 1, Math.max(0, (int) (p * n)));
|
||||||
|
return copy[idx];
|
||||||
|
}
|
||||||
|
|
||||||
/** Min-max normalizes the first {@code n} values to 0..1; returns zeros when the range is flat. */
|
/** Min-max normalizes the first {@code n} values to 0..1; returns zeros when the range is flat. */
|
||||||
private static double[] normalize(double[] values, int n) {
|
private static double[] normalize(double[] values, int n) {
|
||||||
double min = Double.POSITIVE_INFINITY;
|
double min = Double.POSITIVE_INFINITY;
|
||||||
|
|
|
||||||
|
|
@ -37,8 +37,6 @@ public class HighlightSourceScheduler {
|
||||||
"mp4", "mov", "m4v", "mkv", "webm", "avi"
|
"mp4", "mov", "m4v", "mkv", "webm", "avi"
|
||||||
);
|
);
|
||||||
|
|
||||||
private static final int VISION_SAMPLES = 7;
|
|
||||||
|
|
||||||
private final VideoClippingProperties.Editing.HighlightScheduler properties;
|
private final VideoClippingProperties.Editing.HighlightScheduler properties;
|
||||||
private final HighlightProjectStore store;
|
private final HighlightProjectStore store;
|
||||||
private final HighlightSourceAnalyzer analyzer;
|
private final HighlightSourceAnalyzer analyzer;
|
||||||
|
|
@ -230,19 +228,15 @@ public class HighlightSourceScheduler {
|
||||||
HighlightSourceAnalysis analysis, CinematicHighlightAnalysis cinematic, long scanId) {
|
HighlightSourceAnalysis analysis, CinematicHighlightAnalysis cinematic, long scanId) {
|
||||||
try {
|
try {
|
||||||
double duration = analysis.source() == null ? 0.0 : analysis.source().durationSeconds();
|
double duration = analysis.source() == null ? 0.0 : analysis.source().durationSeconds();
|
||||||
// Tier-2 first: caption several beat frames so the director understands the moments, then let the
|
// Localize the decisive moment from MEASURED motion + audio (robust, whole-clip). A sparse whole-clip
|
||||||
// semantic curve guide payoff selection (not just decorate it).
|
// VLM pass used to mislocalize the payoff onto the anticipation, so the VLM is now used only to
|
||||||
List<HighlightVisionDirector.TimedCaption> captions = List.of();
|
// GROUND the overlay in the decisive window (below) — never to pick the moment.
|
||||||
double[] semantic = null;
|
MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration);
|
||||||
if (properties.isVisionDirectorEnabled()) {
|
if (properties.isVisionDirectorEnabled()) {
|
||||||
captions = visionDirector.captionTimeline(sourcePath, duration,
|
// Dense pass over the decisive window (payoff shot through the outcome) with the honesty rule:
|
||||||
store.directorDirectory(projectId).resolve("vision-work"), VISION_SAMPLES);
|
// never assert an action the shown window doesn't contain (a "KICK" over a cut before the kick).
|
||||||
semantic = HighlightVisionDirector.semanticCurve(captions,
|
montage = visionDirector.groundOverlay(montage, sourcePath,
|
||||||
HighlightMontageDirector.WINDOW_SECONDS, duration);
|
store.directorDirectory(projectId).resolve("vision-work"));
|
||||||
}
|
|
||||||
MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration, semantic);
|
|
||||||
if (!captions.isEmpty()) {
|
|
||||||
montage = visionDirector.augmentFromCaptions(montage, captions);
|
|
||||||
}
|
}
|
||||||
store.writeJson(projectId, "director/montage.json", montage);
|
store.writeJson(projectId, "director/montage.json", montage);
|
||||||
// Minimal edit-plan.json so the render scanner selects the project; montage.json takes precedence.
|
// Minimal edit-plan.json so the render scanner selects the project; montage.json takes precedence.
|
||||||
|
|
@ -292,18 +286,33 @@ public class HighlightSourceScheduler {
|
||||||
}
|
}
|
||||||
|
|
||||||
private Path moveToDirectory(Path source, Path targetDirectory) {
|
private Path moveToDirectory(Path source, Path targetDirectory) {
|
||||||
Path target = targetDirectory.resolve(source.getFileName());
|
|
||||||
if (Files.exists(target)) {
|
|
||||||
throw new IllegalStateException("Refusing to overwrite highlight source file: " + target);
|
|
||||||
}
|
|
||||||
try {
|
try {
|
||||||
Files.createDirectories(targetDirectory);
|
Files.createDirectories(targetDirectory);
|
||||||
|
// Never overwrite an existing file, but also never fail on a name collision: re-processing a source
|
||||||
|
// whose name already exists here (a recurring filename) picks a unique "<name>-<n>.<ext>" instead.
|
||||||
|
Path target = uniqueTarget(targetDirectory, source.getFileName().toString());
|
||||||
return Files.move(source, target, StandardCopyOption.ATOMIC_MOVE);
|
return Files.move(source, target, StandardCopyOption.ATOMIC_MOVE);
|
||||||
} catch (IOException ex) {
|
} catch (IOException ex) {
|
||||||
throw new IllegalStateException("Unable to move highlight source file to: " + targetDirectory, ex);
|
throw new IllegalStateException("Unable to move highlight source file to: " + targetDirectory, ex);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Path uniqueTarget(Path directory, String fileName) {
|
||||||
|
Path candidate = directory.resolve(fileName);
|
||||||
|
if (!Files.exists(candidate)) {
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
String base = stripExtension(fileName);
|
||||||
|
String ext = extension(fileName);
|
||||||
|
String suffixExt = ext.isEmpty() ? "" : "." + ext;
|
||||||
|
int n = 1;
|
||||||
|
do {
|
||||||
|
candidate = directory.resolve(base + "-" + n + suffixExt);
|
||||||
|
n++;
|
||||||
|
} while (Files.exists(candidate));
|
||||||
|
return candidate;
|
||||||
|
}
|
||||||
|
|
||||||
private boolean isCandidateFile(Path path) {
|
private boolean isCandidateFile(Path path) {
|
||||||
if (!Files.isRegularFile(path)) {
|
if (!Files.isRegularFile(path)) {
|
||||||
return false;
|
return false;
|
||||||
|
|
|
||||||
|
|
@ -86,37 +86,113 @@ public class HighlightVisionDirector {
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Augments a plan with a semantic overlay + scene-informed music using ALREADY-captured timeline captions
|
* Densely captions the DECISIVE SOURCE WINDOW of a plan — the payoff shot through the final shot, i.e. the
|
||||||
* (no additional model call): the caption nearest the payoff supplies the overlay text and the music mood.
|
* action AND its outcome — and attaches an honest, grounded overlay plus scene-informed music. Unlike a
|
||||||
* Returns the input plan unchanged when there is no payoff or no usable caption.
|
* sparse whole-clip pass, this assesses several frames THROUGH the shown payoff, so the overlay reflects
|
||||||
|
* what is actually on screen. The honesty rule ({@link #honestOverlayText}) never asserts an action the
|
||||||
|
* window does not show: an anticipatory cut gets a teaser question instead. Fail-soft: on any error (model
|
||||||
|
* missing, worker failure, timeout) it returns the plan unchanged, so the Tier-1 cut always stands.
|
||||||
*/
|
*/
|
||||||
public MontagePlan augmentFromCaptions(MontagePlan plan, List<TimedCaption> captions) {
|
public MontagePlan groundOverlay(MontagePlan plan, Path source, Path workDir) {
|
||||||
int payoff = payoffShotIndex(plan);
|
int payoffIdx = payoffShotIndex(plan);
|
||||||
if (payoff < 0 || captions == null || captions.isEmpty()) {
|
if (payoffIdx < 0 || plan.shots().isEmpty()) {
|
||||||
return plan;
|
return plan;
|
||||||
}
|
}
|
||||||
MontagePlan.Shot shot = plan.shots().get(payoff);
|
MontagePlan.Shot payoffShot = plan.shots().get(payoffIdx);
|
||||||
double payoffSource = shot.sourceStartSeconds() + shot.durationSeconds() * shot.speed() / 2.0;
|
MontagePlan.Shot last = plan.shots().get(plan.shots().size() - 1);
|
||||||
TimedCaption nearest = captions.stream()
|
double winStart = payoffShot.sourceStartSeconds();
|
||||||
.min(java.util.Comparator.comparingDouble(c -> Math.abs(c.timeSeconds() - payoffSource)))
|
double winEnd = last.sourceStartSeconds() + last.durationSeconds() * Math.max(0.05, last.speed());
|
||||||
.orElse(null);
|
if (winEnd <= winStart) {
|
||||||
String label = nearest == null ? "" : nearest.label();
|
winEnd = winStart + 1.0;
|
||||||
String description = nearest == null ? "" : nearest.description();
|
}
|
||||||
String overlayText = toOverlayText(label);
|
try {
|
||||||
|
Files.createDirectories(workDir);
|
||||||
|
int samples = 3;
|
||||||
|
List<Manifest> manifest = new ArrayList<>();
|
||||||
|
List<Integer> ids = new ArrayList<>();
|
||||||
|
for (int i = 0; i < samples; i++) {
|
||||||
|
double t = winStart + (winEnd - winStart) * (i + 0.5) / samples;
|
||||||
|
Path frame = workDir.resolve("payoff-" + i + ".jpg");
|
||||||
|
if (extractFrame(source, t, frame)) {
|
||||||
|
String img = frame.toAbsolutePath().toString();
|
||||||
|
manifest.add(new Manifest("d" + i, img,
|
||||||
|
"In one sentence, describe what the person is doing with their body right now."));
|
||||||
|
manifest.add(new Manifest("l" + i, img,
|
||||||
|
"In one to three words, what is the exciting action in this moment?"));
|
||||||
|
manifest.add(new Manifest("q" + i, img,
|
||||||
|
"In three to five words, ask a suspenseful question about what happens next."));
|
||||||
|
ids.add(i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ids.isEmpty()) {
|
||||||
|
return plan;
|
||||||
|
}
|
||||||
|
List<Caption> caps = runCaptioner(workDir, manifest);
|
||||||
|
// Pick the frame whose description most shows the decisive action; later frames win ties so the
|
||||||
|
// window's climax is preferred over its lead-in.
|
||||||
|
int best = ids.get(0);
|
||||||
|
double bestScore = -1;
|
||||||
|
for (int i : ids) {
|
||||||
|
double sc = semanticScore(answerFor(caps, "d" + i));
|
||||||
|
if (sc >= bestScore) {
|
||||||
|
bestScore = sc;
|
||||||
|
best = i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
String description = answerFor(caps, "d" + best);
|
||||||
|
String overlayText = honestOverlayText(description, answerFor(caps, "l" + best),
|
||||||
|
answerFor(caps, "q" + best));
|
||||||
List<MontagePlan.Overlay> overlays = new ArrayList<>(
|
List<MontagePlan.Overlay> overlays = new ArrayList<>(
|
||||||
plan.overlays() == null ? List.of() : plan.overlays());
|
plan.overlays() == null ? List.of() : plan.overlays());
|
||||||
if (!overlayText.isBlank()) {
|
if (!overlayText.isBlank()) {
|
||||||
double[] span = payoffTimeline(plan, payoff);
|
double[] span = payoffTimeline(plan, payoffIdx);
|
||||||
overlays.add(new MontagePlan.Overlay(overlayText, span[0] + 0.2,
|
overlays.add(new MontagePlan.Overlay(overlayText, span[0] + 0.2,
|
||||||
Math.max(span[0] + 0.8, span[1] - 0.2), "lower_center_safe"));
|
Math.max(span[0] + 0.8, span[1] - 0.2), "lower_center_safe"));
|
||||||
}
|
}
|
||||||
String music = description.isBlank() ? plan.musicDirection()
|
String music = description.isBlank() ? plan.musicDirection()
|
||||||
: "cinematic film score for this moment: " + description + " Build quiet tension to a triumphant "
|
: "cinematic film score for this moment: " + description + " Build quiet tension to a "
|
||||||
+ "climax hit at the payoff, then a short warm resolve, modern trailer score, no vocals.";
|
+ "triumphant climax hit at the payoff, then a short warm resolve, modern trailer score, no vocals.";
|
||||||
log.info("event=highlight_vision_director_completed overlay=\"{}\" payoff_description=\"{}\"",
|
log.info("event=highlight_vision_director_completed overlay=\"{}\" payoff_description=\"{}\"",
|
||||||
overlayText, description);
|
overlayText, description);
|
||||||
return new MontagePlan(plan.projectId(), plan.sourceVideoFileName(), plan.grade(), music,
|
return new MontagePlan(plan.projectId(), plan.sourceVideoFileName(), plan.grade(), music,
|
||||||
plan.voiceover(), overlays, plan.shots());
|
plan.voiceover(), overlays, plan.shots());
|
||||||
|
} catch (RuntimeException | IOException | InterruptedException ex) {
|
||||||
|
if (ex instanceof InterruptedException) {
|
||||||
|
Thread.currentThread().interrupt();
|
||||||
|
}
|
||||||
|
log.warn("event=highlight_vision_director_failed stage=overlay error_type={} message={}",
|
||||||
|
ex.getClass().getSimpleName(), ex.getMessage());
|
||||||
|
return plan;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** True when a description reads as anticipation ("about to…") rather than a shown/ongoing action. */
|
||||||
|
static boolean isAnticipatory(String description) {
|
||||||
|
if (description == null || description.isBlank()) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
String d = description.toLowerCase(Locale.ROOT);
|
||||||
|
String[] anticipation = {"about to", "preparing", "getting ready", "ready to", "going to",
|
||||||
|
"approaching", "approaches", "walking up", "walking toward", "walking towards", "lining up",
|
||||||
|
"waiting", "prepares to", "poised", "is going to", "sets up", "setting up", "before"};
|
||||||
|
return containsAny(d, anticipation);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* The honest overlay: assert the action label only when the frame actually SHOWS the action; when it only
|
||||||
|
* shows anticipation, pose the grounded teaser question instead — never claim an action that is not on
|
||||||
|
* screen (a "KICK" overlay over a cut that stops before the kick). Falls back to a generic teaser when the
|
||||||
|
* model gave no usable question.
|
||||||
|
*/
|
||||||
|
static String honestOverlayText(String description, String label, String teaser) {
|
||||||
|
if (isAnticipatory(description)) {
|
||||||
|
String q = toOverlayText(teaser);
|
||||||
|
if (q.isBlank()) {
|
||||||
|
q = "WHAT HAPPENS NEXT";
|
||||||
|
}
|
||||||
|
return q.endsWith("?") ? q : q + "?";
|
||||||
|
}
|
||||||
|
return toOverlayText(label);
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|
|
||||||
|
|
@ -22,7 +22,7 @@ class HighlightMontageDirectorTest {
|
||||||
motion[i] = 2.0;
|
motion[i] = 2.0;
|
||||||
if (Math.abs(t - 4.0) < 0.6) motion[i] = 5.0; // release spike
|
if (Math.abs(t - 4.0) < 0.6) motion[i] = 5.0; // release spike
|
||||||
if (t >= 9.0 && t <= 11.0) motion[i] = 4.0; // celebration motion
|
if (t >= 9.0 && t <= 11.0) motion[i] = 4.0; // celebration motion
|
||||||
if (t >= 12.5) motion[i] = 9.0; // camera-whip tail
|
if (t >= 12.5 && t <= 13.0) motion[i] = 14.0; // brief camera-whip (far above action motion)
|
||||||
audio[i] = -30.0;
|
audio[i] = -30.0;
|
||||||
if (t >= 6.5 && t <= 9.5) audio[i] = -12.0; // celebration band
|
if (t >= 6.5 && t <= 9.5) audio[i] = -12.0; // celebration band
|
||||||
if (Math.abs(t - 8.0) < 0.6) audio[i] = -8.0; // climax peak
|
if (Math.abs(t - 8.0) < 0.6) audio[i] = -8.0; // climax peak
|
||||||
|
|
@ -53,6 +53,66 @@ class HighlightMontageDirectorTest {
|
||||||
assertThat(plan.musicDirection()).isNotBlank();
|
assertThat(plan.musicDirection()).isNotBlank();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void selectsALateDecisiveMomentAndIncludesItsOutcome() {
|
||||||
|
// 64s source: nothing happens for most of it; the decisive ACTION (a kick) is at ~53s — in the last
|
||||||
|
// fifth, which the old central-band search structurally excluded — and its OUTCOME (the ball rolling
|
||||||
|
// away) runs to ~58s. The director must select the late action AND extend the window to the outcome.
|
||||||
|
int n = 128;
|
||||||
|
double[] motion = new double[n];
|
||||||
|
double[] audio = new double[n];
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
double t = i * 0.5;
|
||||||
|
motion[i] = 2.0;
|
||||||
|
if (t >= 52.0 && t <= 55.0) motion[i] = 7.0; // the kick (a motion event)
|
||||||
|
if (t > 55.0 && t <= 58.0) motion[i] = 4.0; // the ball rolling away (the outcome)
|
||||||
|
audio[i] = -30.0;
|
||||||
|
if (t >= 53.0 && t <= 56.0) audio[i] = -10.0; // the kick / kids reacting
|
||||||
|
}
|
||||||
|
|
||||||
|
MontagePlan plan = director.composeMontage("p", "src.mp4", motion, audio, 0.5, 64.0);
|
||||||
|
|
||||||
|
// The payoff lands on the late kick (~52-55s), not the long anticipation before it.
|
||||||
|
double payoff = payoffSourceStart(plan);
|
||||||
|
assertThat(payoff).isBetween(51.0, 56.0);
|
||||||
|
// The window reaches into the outcome (ball rolling, ~56-58s) instead of cutting off at the kick.
|
||||||
|
double maxSourceEnd = plan.shots().stream()
|
||||||
|
.mapToDouble(s -> s.sourceStartSeconds() + s.durationSeconds() * s.speed()).max().orElse(0);
|
||||||
|
assertThat(maxSourceEnd).isGreaterThan(56.0);
|
||||||
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void capsThePreClimaxBuildOnLongSourcesSoNoSingleShotRunsAway() {
|
||||||
|
// 64s source: action spike early (~5s), climax far away (~45s). Without the cap the action/tension
|
||||||
|
// shot would stretch ~38s (one giant shot); the cap must keep it bounded and the montage tight.
|
||||||
|
int n = 128;
|
||||||
|
double[] motion = new double[n];
|
||||||
|
double[] audio = new double[n];
|
||||||
|
for (int i = 0; i < n; i++) {
|
||||||
|
double t = i * 0.5;
|
||||||
|
motion[i] = 2.0;
|
||||||
|
if (Math.abs(t - 5.0) < 0.6) motion[i] = 6.0; // early action spike
|
||||||
|
if (t >= 44.0 && t <= 46.0) motion[i] = 4.0; // celebration motion near the climax
|
||||||
|
audio[i] = -30.0;
|
||||||
|
if (t >= 43.0 && t <= 46.5) audio[i] = -12.0; // celebration band
|
||||||
|
if (Math.abs(t - 45.0) < 0.6) audio[i] = -8.0; // climax peak ~45s
|
||||||
|
}
|
||||||
|
|
||||||
|
MontagePlan plan = director.composeMontage("p", "src.mp4", motion, audio, 0.5, 64.0);
|
||||||
|
|
||||||
|
// No single shot's source span may exceed the configured build cap (6s default) plus a small margin.
|
||||||
|
double longestSpan = plan.shots().stream()
|
||||||
|
.mapToDouble(s -> s.durationSeconds() * s.speed()).max().orElse(0);
|
||||||
|
assertThat(longestSpan).isLessThanOrEqualTo(6.5);
|
||||||
|
// The whole highlight stays tight (a handful of short shots around the climax), not tens of seconds.
|
||||||
|
double total = plan.shots().stream().mapToDouble(MontagePlan.Shot::durationSeconds).sum();
|
||||||
|
assertThat(total).isLessThanOrEqualTo(16.0);
|
||||||
|
// The payoff still lands on the real climax (~45s), so the cap didn't move the story.
|
||||||
|
var slowMo = plan.shots().stream().filter(s -> s.speed() < 0.8).toList();
|
||||||
|
assertThat(slowMo).hasSize(1);
|
||||||
|
assertThat(slowMo.get(0).sourceStartSeconds()).isBetween(44.0, 46.0);
|
||||||
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void semanticCurveMovesThePayoffToTheMeaningfulMoment() {
|
void semanticCurveMovesThePayoffToTheMeaningfulMoment() {
|
||||||
// Audio is slightly LOUDER at t=5 (a bang) than at t=8 (the celebration). Motion has a release spike.
|
// Audio is slightly LOUDER at t=5 (a bang) than at t=8 (the celebration). Motion has a release spike.
|
||||||
|
|
|
||||||
|
|
@ -108,6 +108,22 @@ class HighlightSourceSchedulerTest {
|
||||||
assertThat(tempDir.resolve("highlight-projects/porsche-1/project.json")).exists();
|
assertThat(tempDir.resolve("highlight-projects/porsche-1/project.json")).exists();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
void movesToAUniqueNameInsteadOfFailingWhenTheTargetNameIsTaken() throws Exception {
|
||||||
|
Path dir = tempDir.resolve("processed");
|
||||||
|
Files.createDirectories(dir);
|
||||||
|
HighlightSourceScheduler scheduler = scheduler();
|
||||||
|
|
||||||
|
// A free name is used as-is.
|
||||||
|
assertThat(scheduler.uniqueTarget(dir, "soccer.mp4")).isEqualTo(dir.resolve("soccer.mp4"));
|
||||||
|
|
||||||
|
// A taken name never overwrites: it gets a "-<n>" suffix before the extension, and increments.
|
||||||
|
Files.writeString(dir.resolve("soccer.mp4"), "one");
|
||||||
|
assertThat(scheduler.uniqueTarget(dir, "soccer.mp4")).isEqualTo(dir.resolve("soccer-1.mp4"));
|
||||||
|
Files.writeString(dir.resolve("soccer-1.mp4"), "two");
|
||||||
|
assertThat(scheduler.uniqueTarget(dir, "soccer.mp4")).isEqualTo(dir.resolve("soccer-2.mp4"));
|
||||||
|
}
|
||||||
|
|
||||||
private HighlightSourceScheduler scheduler() {
|
private HighlightSourceScheduler scheduler() {
|
||||||
Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC);
|
Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC);
|
||||||
stubAnalysis("1");
|
stubAnalysis("1");
|
||||||
|
|
|
||||||
|
|
@ -60,21 +60,26 @@ class HighlightVisionDirectorTest {
|
||||||
}
|
}
|
||||||
|
|
||||||
@Test
|
@Test
|
||||||
void augmentsPlanWithOverlayAndMusicFromThePayoffCaption() {
|
void detectsAnticipatoryDescriptions() {
|
||||||
MontagePlan plan = new MontagePlan("p", "s.mp4", "hero", "generic music", List.of(), List.of(), List.of(
|
assertThat(HighlightVisionDirector.isAnticipatory("The person is about to kick the ball")).isTrue();
|
||||||
new MontagePlan.Shot(0.0, 2.0, 1.03, 1.0),
|
assertThat(HighlightVisionDirector.isAnticipatory("A boy preparing to throw")).isTrue();
|
||||||
new MontagePlan.Shot(8.0, 3.0, 1.05, 0.7))); // payoff consumes source 8.0..10.1
|
assertThat(HighlightVisionDirector.isAnticipatory("walking up to the ball")).isTrue();
|
||||||
List<HighlightVisionDirector.TimedCaption> captions = List.of(
|
assertThat(HighlightVisionDirector.isAnticipatory("kicking the ball hard")).isFalse();
|
||||||
new HighlightVisionDirector.TimedCaption(1.0, "preparing to throw", "Preparing"),
|
assertThat(HighlightVisionDirector.isAnticipatory("raising arms in celebration")).isFalse();
|
||||||
new HighlightVisionDirector.TimedCaption(9.0, "raising arms in celebration", "Strike celebration"));
|
assertThat(HighlightVisionDirector.isAnticipatory("")).isFalse();
|
||||||
|
assertThat(HighlightVisionDirector.isAnticipatory(null)).isFalse();
|
||||||
|
}
|
||||||
|
|
||||||
HighlightVisionDirector director = new HighlightVisionDirector(
|
@Test
|
||||||
new org.example.videoclips.config.VideoClippingProperties(),
|
void overlayAssertsShownActionButPosesAQuestionForAnticipation() {
|
||||||
new com.fasterxml.jackson.databind.ObjectMapper());
|
// The frame SHOWS the action -> assert the punchy label.
|
||||||
MontagePlan augmented = director.augmentFromCaptions(plan, captions);
|
assertThat(HighlightVisionDirector.honestOverlayText(
|
||||||
|
"kicking the ball hard", "Kick", "will he score")).isEqualTo("KICK");
|
||||||
assertThat(augmented.overlays()).hasSize(1);
|
// The frame only shows anticipation -> never assert "KICK"; pose the grounded teaser question instead.
|
||||||
assertThat(augmented.overlays().get(0).text()).isEqualTo("STRIKE CELEBRATION"); // from the label
|
assertThat(HighlightVisionDirector.honestOverlayText(
|
||||||
assertThat(augmented.musicDirection()).contains("raising arms in celebration"); // from the description
|
"the person is about to kick the ball", "Kick", "will he score")).isEqualTo("WILL HE SCORE?");
|
||||||
|
// Anticipation with no usable teaser -> a generic honest question, still never the false action claim.
|
||||||
|
assertThat(HighlightVisionDirector.honestOverlayText(
|
||||||
|
"preparing to shoot", "Shot", " ")).isEqualTo("WHAT HAPPENS NEXT?");
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue