diff --git a/docs/cinematic-quality-rules.md b/docs/cinematic-quality-rules.md index ad0603a..522031c 100644 --- a/docs/cinematic-quality-rules.md +++ b/docs/cinematic-quality-rules.md @@ -67,11 +67,18 @@ Legend: ✅ implemented · ⏳ planned (see `cinematic-highlight-poc-plan.md` P5 continuous action/tension (release/roll/watch, never chopped) → slow-motion payoff on the audio climax → resolution button, trimming a high-motion camera-whip tail. Enabled by `highlight-scheduler.auto-director-enabled` (on in localpoc); writes `director/montage.json` after analysis. Verified on bowling: auto cut ≈ the hand cut. -- **Tier 2 (semantic VLM director, DONE 2026-07-23):** `HighlightVisionDirector` + `tools/vision_caption.py` - run a local vision-language model (moondream2, offline) to caption the payoff frame, then AUGMENT the Tier-1 - montage with a semantic overlay and a scene-informed music direction. On bowling it read the celebration and - produced the overlay "STRIKE" and a scene-accurate music prompt — automatically. Enabled by - `highlight-scheduler.vision-director-enabled` (localpoc on); ~25s/frame on CPU; fails soft (Tier-1 stands). +- **Tier 2 (semantic VLM director, DONE 2026-07-23/24):** `HighlightVisionDirector` + `tools/vision_caption.py` + run a local vision-language model (moondream2, offline). It now captions *several* beat frames (two questions + each in one call: a discriminative description + a punchy label) and: + 1. **guides selection** — `semanticScore`/`semanticCurve` turn the descriptions into a per-window + highlight-worthiness signal that the montage director blends with audio to place the payoff on the + semantically-strongest moment (verified: on bowling the payoff moved onto moondream's detected + celebration); + 2. **decorates** — the payoff label becomes the bold overlay and the description flavors the music. + All generic: the scoring uses generic emotion/action/idle keywords (no content-specific terms), and it + fails soft to the measured cut. ~25s/frame CPU; enabled by `vision-director-enabled` (localpoc on). + **Honest limit:** a small VLM on distant subjects is only weakly discriminative — a terse question collapses + to a constant answer (use descriptive questions); a stronger VLM or clearer framing would help. moondream2 is Apache-2.0 (commercial-friendly, unlike the CC-BY-NC audio models). - **Source caveat:** a director can only cut what was filmed. If the camera never shows the pins, no tier can. diff --git a/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java b/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java index b85c61f..dc3c29c 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java +++ b/src/main/java/org/example/videoclips/editing/HighlightMontageDirector.java @@ -22,9 +22,9 @@ import java.util.List; * * *

while trimming a high-motion tail (e.g. a camera whip). Every decision is derived from a measurement of - * this source — nothing is hard-coded to one video. Semantic captions/overlays and emotional nuance (e.g. - * "he wasn't sure it was a strike") are deliberately NOT attempted here; that is the Tier-2 vision-language - * director's job. This tier gives a strong, deterministic, offline baseline cut. + * this source — nothing is hard-coded to one video or one kind of content. Semantic captions/overlays and + * emotional nuance are deliberately NOT attempted here; that is the Tier-2 vision-language director's job. + * This tier gives a strong, deterministic, offline baseline cut for any source. */ @Component public class HighlightMontageDirector { @@ -39,9 +39,23 @@ public class HighlightMontageDirector { /** Measure the source and compose an automatic montage plan. */ public MontagePlan direct(String projectId, String sourceFileName, Path source, double durationSeconds) { + return direct(projectId, sourceFileName, source, durationSeconds, null); + } + + /** + * Compose a montage plan from measured signals, optionally guided by a Tier-2 semantic curve (per-window + * "highlight-worthiness" from vision captions). When present, the semantic signal helps choose the payoff. + */ + public MontagePlan direct(String projectId, String sourceFileName, Path source, double durationSeconds, + double[] semantic) { double[] motion = probeCurve(source.toString(), true, durationSeconds); double[] audio = probeCurve(source.toString(), false, durationSeconds); - return composeMontage(projectId, sourceFileName, motion, audio, WINDOW_SECONDS, durationSeconds); + return composeMontage(projectId, sourceFileName, motion, audio, semantic, WINDOW_SECONDS, durationSeconds); + } + + MontagePlan composeMontage(String projectId, String sourceFileName, double[] motion, double[] audio, + double window, double duration) { + return composeMontage(projectId, sourceFileName, motion, audio, null, window, duration); } /** @@ -49,7 +63,7 @@ public class HighlightMontageDirector { * ffmpeg. {@code motion[i]} and {@code audio[i]} cover the window starting at {@code i * window} seconds. */ MontagePlan composeMontage(String projectId, String sourceFileName, double[] motion, double[] audio, - double window, double duration) { + double[] semantic, double window, double duration) { int n = Math.min(motion.length, audio.length); if (n < 4 || duration <= 0) { return straightCut(projectId, sourceFileName, duration); @@ -57,8 +71,20 @@ public class HighlightMontageDirector { double[] m = smooth(motion, n); double[] a = smooth(audio, n); - // Climax = loudest sustained audio in the central band (skip the intro and the messy tail). - int climaxIdx = argMax(a, (int) Math.floor(0.25 * n), (int) Math.ceil(0.80 * n)); + // Climax = the peak of the "payoff" signal in the central band (skip the intro and the messy tail). + // Base signal is loud sustained audio; when a Tier-2 semantic curve is present (vision captions scored + // for highlight-worthiness), blend it in so the payoff lands on the moment that is both loud AND + // semantically the highlight (e.g. the celebration), not merely the loudest sound. + double[] payoffSignal = a; + if (semantic != null && semantic.length >= n) { + double[] normAudio = normalize(a, n); + double[] normSemantic = normalize(semantic, n); + payoffSignal = new double[n]; + for (int i = 0; i < n; i++) { + payoffSignal[i] = normAudio[i] + 0.9 * normSemantic[i]; + } + } + int climaxIdx = argMax(payoffSignal, (int) Math.floor(0.25 * n), (int) Math.ceil(0.80 * n)); double climaxTime = climaxIdx * window; // Action = the biggest motion spike before the climax; the action shot leads in ~1.5s before it so @@ -219,6 +245,25 @@ public class HighlightMontageDirector { return best; } + /** Min-max normalizes the first {@code n} values to 0..1; returns zeros when the range is flat. */ + private static double[] normalize(double[] values, int n) { + double min = Double.POSITIVE_INFINITY; + double max = Double.NEGATIVE_INFINITY; + for (int i = 0; i < n; i++) { + min = Math.min(min, values[i]); + max = Math.max(max, values[i]); + } + double[] out = new double[n]; + double range = max - min; + if (range <= 1e-9) { + return out; + } + for (int i = 0; i < n; i++) { + out[i] = (values[i] - min) / range; + } + return out; + } + private static double mean(double[] values, int from, int to) { int lo = Math.max(0, from); int hi = Math.min(values.length, Math.max(lo + 1, to)); diff --git a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java index fb34333..aff3671 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java +++ b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java @@ -18,6 +18,7 @@ import java.nio.file.StandardCopyOption; import java.time.Clock; import java.time.Instant; import java.util.Comparator; +import java.util.List; import java.util.Locale; import java.util.Optional; import java.util.Set; @@ -36,6 +37,8 @@ public class HighlightSourceScheduler { "mp4", "mov", "m4v", "mkv", "webm", "avi" ); + private static final int VISION_SAMPLES = 7; + private final VideoClippingProperties.Editing.HighlightScheduler properties; private final HighlightProjectStore store; private final HighlightSourceAnalyzer analyzer; @@ -227,10 +230,19 @@ public class HighlightSourceScheduler { HighlightSourceAnalysis analysis, CinematicHighlightAnalysis cinematic, long scanId) { try { double duration = analysis.source() == null ? 0.0 : analysis.source().durationSeconds(); - MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration); + // Tier-2 first: caption several beat frames so the director understands the moments, then let the + // semantic curve guide payoff selection (not just decorate it). + List captions = List.of(); + double[] semantic = null; if (properties.isVisionDirectorEnabled()) { - montage = visionDirector.augment(montage, sourcePath, - store.directorDirectory(projectId).resolve("vision-work")); + captions = visionDirector.captionTimeline(sourcePath, duration, + store.directorDirectory(projectId).resolve("vision-work"), VISION_SAMPLES); + semantic = HighlightVisionDirector.semanticCurve(captions, + HighlightMontageDirector.WINDOW_SECONDS, duration); + } + MontagePlan montage = montageDirector.direct(projectId, sourceFileName, sourcePath, duration, semantic); + if (!captions.isEmpty()) { + montage = visionDirector.augmentFromCaptions(montage, captions); } store.writeJson(projectId, "director/montage.json", montage); // Minimal edit-plan.json so the render scanner selects the project; montage.json takes precedence. diff --git a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java index 0a09b72..b1941c5 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java +++ b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java @@ -40,52 +40,138 @@ public class HighlightVisionDirector { this.objectMapper = objectMapper; } - /** Returns a plan augmented with a semantic overlay + scene-informed music, or the input plan on failure. */ - public MontagePlan augment(MontagePlan plan, Path source, Path workDir) { + /** + * Captions {@code samples} frames evenly across the source with the local VLM (one worker call, model + * loaded once). Returns timed captions describing the key action/achievement at each moment, or an empty + * list on any failure (the caller then proceeds with measurement only). + */ + public List captionTimeline(Path source, double duration, Path workDir, int samples) { try { - int payoff = payoffShotIndex(plan); - if (payoff < 0) { - return plan; - } - MontagePlan.Shot shot = plan.shots().get(payoff); - double sourceMidpoint = shot.sourceStartSeconds() + shot.durationSeconds() * shot.speed() / 2.0; - Files.createDirectories(workDir); - Path frame = workDir.resolve("vision-payoff.jpg"); - if (!extractFrame(source, sourceMidpoint, frame)) { - return plan; + List manifest = new ArrayList<>(); + List times = new ArrayList<>(); + for (int i = 0; i < samples; i++) { + double t = duration * (i + 0.5) / samples; + Path frame = workDir.resolve("beat-" + i + ".jpg"); + if (extractFrame(source, t, frame)) { + // Two questions per frame in one worker call: a DESCRIPTIVE one (small VLMs give far more + // discriminative descriptions than terse verb phrases, which collapse to a constant answer + // on distant subjects) to score the moment, and a PUNCHY one for the overlay caption. + manifest.add(new Manifest("d" + i, frame.toAbsolutePath().toString(), + "In one sentence, describe what the person is doing with their body right now.")); + manifest.add(new Manifest("l" + i, frame.toAbsolutePath().toString(), + "In one to three words, what is the exciting achievement or action in this moment?")); + times.add(t); + } } - - List captions = caption(workDir, frame); - String label = captions.stream().filter(c -> "label".equals(c.id())).map(Caption::answer) - .findFirst().orElse(""); - String scene = captions.stream().filter(c -> "scene".equals(c.id())).map(Caption::answer) - .findFirst().orElse(""); - - String overlayText = toOverlayText(label); - List overlays = new ArrayList<>( - plan.overlays() == null ? List.of() : plan.overlays()); - if (!overlayText.isBlank()) { - double[] span = payoffTimeline(plan, payoff); - overlays.add(new MontagePlan.Overlay(overlayText, span[0] + 0.2, - Math.max(span[0] + 0.8, span[1] - 0.2), "lower_center_safe")); + if (manifest.isEmpty()) { + return List.of(); } - String music = scene.isBlank() ? plan.musicDirection() - : "cinematic film score for this scene: " + scene - + " Build quiet tension to a triumphant climax hit, then a short warm resolve, no vocals."; - - log.info("event=highlight_vision_director_completed overlay=\"{}\" scene_len={}", - overlayText, scene.length()); - return new MontagePlan(plan.projectId(), plan.sourceVideoFileName(), plan.grade(), music, - plan.voiceover(), overlays, plan.shots()); + List captions = runCaptioner(workDir, manifest); + List result = new ArrayList<>(); + for (int i = 0; i < times.size(); i++) { + result.add(new TimedCaption(times.get(i), answerFor(captions, "d" + i), + answerFor(captions, "l" + i))); + } + log.info("event=highlight_vision_timeline_captioned frames={}", result.size()); + return result; } catch (RuntimeException | IOException | InterruptedException ex) { if (ex instanceof InterruptedException) { Thread.currentThread().interrupt(); } - log.warn("event=highlight_vision_director_failed error_type={} message={}", + log.warn("event=highlight_vision_director_failed stage=timeline error_type={} message={}", ex.getClass().getSimpleName(), ex.getMessage()); + return List.of(); + } + } + + /** + * Augments a plan with a semantic overlay + scene-informed music using ALREADY-captured timeline captions + * (no additional model call): the caption nearest the payoff supplies the overlay text and the music mood. + * Returns the input plan unchanged when there is no payoff or no usable caption. + */ + public MontagePlan augmentFromCaptions(MontagePlan plan, List captions) { + int payoff = payoffShotIndex(plan); + if (payoff < 0 || captions == null || captions.isEmpty()) { return plan; } + MontagePlan.Shot shot = plan.shots().get(payoff); + double payoffSource = shot.sourceStartSeconds() + shot.durationSeconds() * shot.speed() / 2.0; + TimedCaption nearest = captions.stream() + .min(java.util.Comparator.comparingDouble(c -> Math.abs(c.timeSeconds() - payoffSource))) + .orElse(null); + String label = nearest == null ? "" : nearest.label(); + String description = nearest == null ? "" : nearest.description(); + String overlayText = toOverlayText(label); + List overlays = new ArrayList<>( + plan.overlays() == null ? List.of() : plan.overlays()); + if (!overlayText.isBlank()) { + double[] span = payoffTimeline(plan, payoff); + overlays.add(new MontagePlan.Overlay(overlayText, span[0] + 0.2, + Math.max(span[0] + 0.8, span[1] - 0.2), "lower_center_safe")); + } + String music = description.isBlank() ? plan.musicDirection() + : "cinematic film score for this moment: " + description + " Build quiet tension to a triumphant " + + "climax hit at the payoff, then a short warm resolve, modern trailer score, no vocals."; + log.info("event=highlight_vision_director_completed overlay=\"{}\" payoff_description=\"{}\"", + overlayText, description); + return new MontagePlan(plan.projectId(), plan.sourceVideoFileName(), plan.grade(), music, + plan.voiceover(), overlays, plan.shots()); + } + + /** + * A per-window "highlight-worthiness" curve from timeline captions (each window takes the nearest + * caption's semantic score). Feeds the montage director's payoff selection. + */ + public static double[] semanticCurve(List captions, double windowSeconds, double duration) { + int size = Math.max(1, (int) Math.ceil(duration / windowSeconds)); + double[] curve = new double[size]; + if (captions == null || captions.isEmpty()) { + return curve; + } + for (int i = 0; i < size; i++) { + double t = (i + 0.5) * windowSeconds; + TimedCaption nearest = captions.stream() + .min(java.util.Comparator.comparingDouble(c -> Math.abs(c.timeSeconds() - t))) + .orElse(null); + curve[i] = nearest == null ? 0.0 : semanticScore(nearest.description()); + } + return curve; + } + + /** + * Scores a caption for how much it looks like a highlight payoff. Emotion/celebration ranks highest, then + * action, then idle/setup. Keyword heuristic (deliberately simple and offline). + */ + static double semanticScore(String caption) { + if (caption == null || caption.isBlank()) { + return 0.4; + } + String c = caption.toLowerCase(Locale.ROOT); + // Keep keywords specific enough not to false-match (e.g. bare "win" hides inside "throwing"). + String[] payoff = {"celebrat", "cheer", "victor", "triumph", "raising", "arms up", "arms rais", + "excit", "happy", "applau", "fist", "champion", "thumbs", "dancing", "jumping"}; + String[] action = {"throw", "roll", "release", "swing", "kick", "shoot", "spin", "sliding"}; + String[] idle = {"stand", "wait", "empty", "prepare", "background", "looking", "watching", "sitting"}; + if (containsAny(c, payoff)) { + return 1.0; + } + if (containsAny(c, action)) { + return 0.65; + } + if (containsAny(c, idle)) { + return 0.2; + } + return 0.4; + } + + private static boolean containsAny(String haystack, String[] needles) { + for (String needle : needles) { + if (haystack.contains(needle)) { + return true; + } + } + return false; } /** The strongest slow-motion shot is the payoff; -1 if there is none. */ @@ -145,12 +231,8 @@ public class HighlightVisionDirector { return process.waitFor() == 0 && Files.isRegularFile(output); } - private List caption(Path workDir, Path frame) throws IOException, InterruptedException { - List manifest = List.of( - new Manifest("label", frame.toAbsolutePath().toString(), - "In one to three words, what is the exciting achievement or action in this moment?"), - new Manifest("scene", frame.toAbsolutePath().toString(), - "Describe the scene and mood in one short sentence for a film score composer.")); + private List runCaptioner(Path workDir, List manifest) + throws IOException, InterruptedException { Path manifestFile = workDir.resolve("vision-manifest.json"); Path outputFile = workDir.resolve("vision-captions.json"); objectMapper.writeValue(manifestFile.toFile(), manifest); @@ -176,4 +258,16 @@ public class HighlightVisionDirector { record Caption(String id, String answer) { } + + private static String answerFor(List captions, String id) { + return captions.stream().filter(c -> id.equals(c.id())).map(Caption::answer) + .filter(java.util.Objects::nonNull).findFirst().orElse(""); + } + + /** + * Captions for a specific moment (seconds): a full {@code description} (used to score the moment and flavor + * the music) and a punchy {@code label} (used for the overlay text). + */ + public record TimedCaption(double timeSeconds, String description, String label) { + } } diff --git a/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java b/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java index 6775b0c..496c406 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightMontageDirectorTest.java @@ -53,6 +53,43 @@ class HighlightMontageDirectorTest { assertThat(plan.musicDirection()).isNotBlank(); } + @Test + void semanticCurveMovesThePayoffToTheMeaningfulMoment() { + // Audio is slightly LOUDER at t=5 (a bang) than at t=8 (the celebration). Motion has a release spike. + int n = 32; + double[] motion = new double[n]; + double[] audio = new double[n]; + for (int i = 0; i < n; i++) { + double t = i * 0.5; + motion[i] = 2.0; + if (Math.abs(t - 4.0) < 0.6) motion[i] = 5.0; + audio[i] = -30.0; + if (Math.abs(t - 5.0) < 0.7) audio[i] = -8.0; // loudest (a wide bang), but not the highlight + if (Math.abs(t - 8.0) < 0.7) audio[i] = -14.0; // celebration, quieter + } + // Semantics say the celebration (t~8) is the highlight. + double[] semantic = new double[n]; + for (int i = 0; i < n; i++) { + double t = i * 0.5; + semantic[i] = 0.2; + if (t >= 7.5 && t <= 8.5) semantic[i] = 1.0; + } + + double payoffWithoutSemantics = payoffSourceStart( + director.composeMontage("p", "s.mp4", motion, audio, 0.5, 16.0)); + double payoffWithSemantics = payoffSourceStart( + director.composeMontage("p", "s.mp4", motion, audio, semantic, 0.5, 16.0)); + + // Without semantics the payoff sits on the loud bang (~5s); with them it moves onto the celebration (~8s). + assertThat(payoffWithoutSemantics).isLessThan(6.5); + assertThat(payoffWithSemantics).isGreaterThan(7.5); + } + + private double payoffSourceStart(MontagePlan plan) { + return plan.shots().stream().filter(s -> s.speed() < 0.8).findFirst() + .map(MontagePlan.Shot::sourceStartSeconds).orElse(-1.0); + } + @Test void fallsBackToAStraightCutForVeryShortSources() { MontagePlan plan = director.composeMontage("p", "src.mp4", new double[]{1, 1}, new double[]{-30, -30}, diff --git a/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java b/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java index 805cd37..34d43f9 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java @@ -39,4 +39,42 @@ class HighlightVisionDirectorTest { List.of(new MontagePlan.Shot(0.0, 2.0, 1.03, 1.0))); assertThat(HighlightVisionDirector.payoffShotIndex(plan)).isEqualTo(-1); } + + @Test + void scoresCaptionsByHighlightWorthiness() { + assertThat(HighlightVisionDirector.semanticScore("A man raising his arms in celebration")).isEqualTo(1.0); + assertThat(HighlightVisionDirector.semanticScore("Throwing the ball")).isEqualTo(0.65); + assertThat(HighlightVisionDirector.semanticScore("Standing and waiting")).isEqualTo(0.2); + assertThat(HighlightVisionDirector.semanticScore("A quiet room")).isEqualTo(0.4); // neutral + assertThat(HighlightVisionDirector.semanticScore("")).isEqualTo(0.4); + } + + @Test + void buildsSemanticCurveFromNearestCaption() { + List captions = List.of( + new HighlightVisionDirector.TimedCaption(1.0, "standing still", ""), // idle 0.2 + new HighlightVisionDirector.TimedCaption(5.0, "arms raised celebration", "")); // payoff 1.0 + double[] curve = HighlightVisionDirector.semanticCurve(captions, 1.0, 6.0); // 6 windows + assertThat(curve[0]).isEqualTo(0.2); // window ~0.5s -> nearest 1.0s "standing still" + assertThat(curve[5]).isEqualTo(1.0); // window ~5.5s -> nearest 5.0s "arms raised celebration" + } + + @Test + void augmentsPlanWithOverlayAndMusicFromThePayoffCaption() { + MontagePlan plan = new MontagePlan("p", "s.mp4", "hero", "generic music", List.of(), List.of(), List.of( + new MontagePlan.Shot(0.0, 2.0, 1.03, 1.0), + new MontagePlan.Shot(8.0, 3.0, 1.05, 0.7))); // payoff consumes source 8.0..10.1 + List captions = List.of( + new HighlightVisionDirector.TimedCaption(1.0, "preparing to throw", "Preparing"), + new HighlightVisionDirector.TimedCaption(9.0, "raising arms in celebration", "Strike celebration")); + + HighlightVisionDirector director = new HighlightVisionDirector( + new org.example.videoclips.config.VideoClippingProperties(), + new com.fasterxml.jackson.databind.ObjectMapper()); + MontagePlan augmented = director.augmentFromCaptions(plan, captions); + + assertThat(augmented.overlays()).hasSize(1); + assertThat(augmented.overlays().get(0).text()).isEqualTo("STRIKE CELEBRATION"); // from the label + assertThat(augmented.musicDirection()).contains("raising arms in celebration"); // from the description + } }