Add highlight scene segmentation

docs/production-cinematic-highlight-editing-plan.md

@@ -301,7 +301,8 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can
 - [x] Milestone 2 progress: Added a configurable highlight source scheduler and directory initializer for `input/highlights/source`, `working`, `processed`, and `rejected`; it claims one valid source video per scan, creates a highlight project and manifest, copies the source into the project folder, and leaves later videos for later scans.
 - [x] Milestone 3: Implement source probing, proxy generation, frame extraction, waveform extraction, and contact sheet creation.
 - [x] Milestone 3 progress: Added highlight source analysis for the new project contract, reusing FFprobe inspection, frame extraction, contact sheet generation, proxy generation, and adding waveform image generation under `analysis/`; the highlight scheduler now runs analysis immediately after creating a project.
-- [ ] Milestone 4: Implement shot and scene segmentation using FFmpeg scene detection or PySceneDetect.
+- [x] Milestone 4: Implement shot and scene segmentation using FFmpeg scene detection or PySceneDetect.
+- [x] Milestone 4 progress: Added configurable FFmpeg scene detection for highlight sources, persisted `analysis/scene-segments.json`, attached shot segments to `source-analysis.json`, and logged shot segment counts when the highlight scheduler finishes analysis.
 - [ ] Milestone 5: Implement audio analysis for loudness, silence, peaks, speech/music/noise sections, and optional ASR transcript.
 - [ ] Milestone 6: Implement visual analysis for blur, exposure, motion, faces, object labels, composition quality, and representative thumbnails.
 - [x] Milestone 7: Implement content category classification for family vlog, food vlog, car vlog, and generic fallback.

src/main/java/org/example/videoclips/config/VideoClippingProperties.java

@@ -503,6 +503,10 @@ public class VideoClippingProperties {
         @Min(1)
         private int proxyWidth = 640;
 
+        private double sceneDetectionThreshold = 0.35;
+
+        private double minimumSceneDurationSeconds = 1.0;
+
         @Min(1)
         private int targetDurationSeconds = 60;
 
@@ -618,6 +622,22 @@ public class VideoClippingProperties {
             this.proxyWidth = proxyWidth;
         }
 
+        public double getSceneDetectionThreshold() {
+            return sceneDetectionThreshold;
+        }
+
+        public void setSceneDetectionThreshold(double sceneDetectionThreshold) {
+            this.sceneDetectionThreshold = sceneDetectionThreshold;
+        }
+
+        public double getMinimumSceneDurationSeconds() {
+            return minimumSceneDurationSeconds;
+        }
+
+        public void setMinimumSceneDurationSeconds(double minimumSceneDurationSeconds) {
+            this.minimumSceneDurationSeconds = minimumSceneDurationSeconds;
+        }
+
         public int getTargetDurationSeconds() {
             return targetDurationSeconds;
         }

src/main/java/org/example/videoclips/editing/HighlightSourceAnalysis.java

@@ -11,6 +11,8 @@ public record HighlightSourceAnalysis(
         String contactSheet,
         String proxyPath,
         String waveformPath,
+        String sceneSegmentsPath,
+        List<ShotSegment> shotSegments,
         Instant createdAt
 ) {
 }

src/main/java/org/example/videoclips/editing/HighlightSourceAnalyzer.java

@@ -21,6 +21,7 @@ public class HighlightSourceAnalyzer {
     private final ContactSheetGenerator contactSheetGenerator;
     private final ProxyGenerator proxyGenerator;
     private final WaveformGenerator waveformGenerator;
+    private final ShotSceneSegmenter shotSceneSegmenter;
     private final Clock clock;
 
     @Autowired
@@ -30,9 +31,11 @@ public class HighlightSourceAnalyzer {
             ThumbnailExtractor thumbnailExtractor,
             ContactSheetGenerator contactSheetGenerator,
             ProxyGenerator proxyGenerator,
-            WaveformGenerator waveformGenerator
+            WaveformGenerator waveformGenerator,
+            ShotSceneSegmenter shotSceneSegmenter
     ) {
         this(store, inspector, thumbnailExtractor, contactSheetGenerator, proxyGenerator, waveformGenerator,
+                shotSceneSegmenter,
                 Clock.systemUTC());
     }
 
@@ -43,6 +46,7 @@ public class HighlightSourceAnalyzer {
             ContactSheetGenerator contactSheetGenerator,
             ProxyGenerator proxyGenerator,
             WaveformGenerator waveformGenerator,
+            ShotSceneSegmenter shotSceneSegmenter,
             Clock clock
     ) {
         this.store = store;
@@ -51,6 +55,7 @@ public class HighlightSourceAnalyzer {
         this.contactSheetGenerator = contactSheetGenerator;
         this.proxyGenerator = proxyGenerator;
         this.waveformGenerator = waveformGenerator;
+        this.shotSceneSegmenter = shotSceneSegmenter;
         this.clock = clock;
     }
 
@@ -68,6 +73,7 @@ public class HighlightSourceAnalyzer {
         String contactSheet = contactSheetGenerator.generate(inspected, contactSheetDirectory);
         String proxyPath = proxyGenerator.generate(inspected, proxyDirectory).orElse(null);
         String waveformPath = waveformGenerator.generate(inspected, audioDirectory);
+        List<ShotSegment> shotSegments = shotSceneSegmenter.segment(inspected);
         ClipAnalysis sourceAnalysis = enriched(inspected, thumbnails, contactSheet, proxyPath);
         HighlightSourceAnalysis analysis = new HighlightSourceAnalysis(
                 projectId,
@@ -77,9 +83,12 @@ public class HighlightSourceAnalyzer {
                 contactSheet,
                 proxyPath,
                 waveformPath,
+                "analysis/scene-segments.json",
+                List.copyOf(shotSegments),
                 Instant.now(clock)
         );
         store.writeJson(projectId, "analysis/ffprobe.json", sourceAnalysis);
+        store.writeJson(projectId, "analysis/scene-segments.json", shotSegments);
         store.writeJson(projectId, "analysis/source-analysis.json", analysis);
         return analysis;
     }

src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java

@@ -153,9 +153,9 @@ public class HighlightSourceScheduler {
                 scanId, projectId, processedFile.getFileName(), projectDirectory, "analysis/source-analysis.json",
                 elapsedMillis(startedAt));
         log.info("event=highlight_source_analyzed scan_id={} project_id={} duration_seconds={} thumbnails={} "
-                        + "proxy={} waveform={}",
+                        + "proxy={} waveform={} shot_segments={}",
                 scanId, projectId, analysis.source().durationSeconds(), analysis.thumbnails().size(),
-                analysis.proxyPath(), analysis.waveformPath());
+                analysis.proxyPath(), analysis.waveformPath(), analysis.shotSegments().size());
     }
 
     private void handleFailure(long scanId, Path activeFile, RuntimeException failure) {

src/main/java/org/example/videoclips/editing/ShotSceneSegmenter.java

@@ -0,0 +1,138 @@
+package org.example.videoclips.editing;
+
+import org.example.videoclips.config.VideoClippingProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+@Component
+@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true)
+public class ShotSceneSegmenter {
+
+    private static final Pattern PTS_TIME_PATTERN = Pattern.compile("pts_time:([0-9]+(?:\\.[0-9]+)?)");
+    private static final Pattern SCENE_SCORE_PATTERN = Pattern.compile("lavfi\\.scene_score=([0-9]+(?:\\.[0-9]+)?)");
+
+    private final VideoClippingProperties.Editing properties;
+    private final ProcessExecutor processExecutor;
+
+    @Autowired
+    public ShotSceneSegmenter(VideoClippingProperties properties) {
+        this(properties, ShotSceneSegmenter::execute);
+    }
+
+    ShotSceneSegmenter(VideoClippingProperties properties, ProcessExecutor processExecutor) {
+        this.properties = properties.getEditing();
+        this.processExecutor = processExecutor;
+    }
+
+    public List<ShotSegment> segment(ClipAnalysis source) {
+        List<String> command = List.of(
+                properties.getFfmpegBinary(),
+                "-hide_banner",
+                "-i", source.sourcePath(),
+                "-filter:v", "select='gt(scene," + properties.getSceneDetectionThreshold()
+                        + ")',metadata=print",
+                "-an",
+                "-f", "null",
+                "-"
+        );
+        ProcessResult result;
+        try {
+            result = processExecutor.execute(command);
+        } catch (InterruptedException ex) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("Interrupted while detecting shot segments", ex);
+        } catch (IOException ex) {
+            throw new IllegalStateException("Unable to run FFmpeg for shot segmentation", ex);
+        }
+        if (result.exitCode() != 0) {
+            throw new IllegalStateException("FFmpeg shot segmentation exited with code " + result.exitCode());
+        }
+        return segmentsFromCuts(source.durationSeconds(), parseCuts(result.output()));
+    }
+
+    private List<SceneCut> parseCuts(String output) {
+        List<SceneCut> cuts = new ArrayList<>();
+        for (String line : output.split("\\R")) {
+            Matcher ptsMatcher = PTS_TIME_PATTERN.matcher(line);
+            if (ptsMatcher.find()) {
+                cuts.add(new SceneCut(Double.parseDouble(ptsMatcher.group(1)), 0));
+                continue;
+            }
+            Matcher scoreMatcher = SCENE_SCORE_PATTERN.matcher(line);
+            if (scoreMatcher.find() && !cuts.isEmpty()) {
+                SceneCut last = cuts.remove(cuts.size() - 1);
+                cuts.add(new SceneCut(last.timestampSeconds(), Double.parseDouble(scoreMatcher.group(1))));
+            }
+        }
+        return cuts;
+    }
+
+    private List<ShotSegment> segmentsFromCuts(double durationSeconds, List<SceneCut> cuts) {
+        double duration = Math.max(0, durationSeconds);
+        double minSegmentSeconds = properties.getMinimumSceneDurationSeconds();
+        List<ShotSegment> shots = new ArrayList<>();
+        double start = 0;
+        double score = 0;
+        for (SceneCut cut : cuts.stream().sorted().toList()) {
+            double timestamp = Math.min(duration, Math.max(0, cut.timestampSeconds()));
+            if (timestamp - start < minSegmentSeconds || duration - timestamp < minSegmentSeconds) {
+                continue;
+            }
+            shots.add(segment(shots.size() + 1, start, timestamp, score));
+            start = timestamp;
+            score = cut.sceneScore();
+        }
+        if (duration > start) {
+            shots.add(segment(shots.size() + 1, start, duration, score));
+        }
+        if (shots.isEmpty()) {
+            shots.add(segment(1, 0, duration, 0));
+        }
+        return List.copyOf(shots);
+    }
+
+    private ShotSegment segment(int index, double startSeconds, double endSeconds, double sceneScore) {
+        double duration = Math.max(0, endSeconds - startSeconds);
+        return new ShotSegment(
+                "shot_%04d".formatted(index),
+                round(startSeconds),
+                round(endSeconds),
+                round(duration),
+                round(sceneScore),
+                round(startSeconds + duration / 2.0)
+        );
+    }
+
+    private double round(double value) {
+        return Math.round(value * 1000.0) / 1000.0;
+    }
+
+    private static ProcessResult execute(List<String> command) throws IOException, InterruptedException {
+        Process process = new ProcessBuilder(command).redirectErrorStream(true).start();
+        String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
+        return new ProcessResult(process.waitFor(), output);
+    }
+
+    private record SceneCut(double timestampSeconds, double sceneScore) implements Comparable<SceneCut> {
+        @Override
+        public int compareTo(SceneCut other) {
+            return Double.compare(timestampSeconds, other.timestampSeconds());
+        }
+    }
+
+    record ProcessResult(int exitCode, String output) {
+    }
+
+    @FunctionalInterface
+    interface ProcessExecutor {
+        ProcessResult execute(List<String> command) throws IOException, InterruptedException;
+    }
+}

src/main/java/org/example/videoclips/editing/ShotSegment.java

@@ -0,0 +1,11 @@
+package org.example.videoclips.editing;
+
+public record ShotSegment(
+        String shotId,
+        double startSeconds,
+        double endSeconds,
+        double durationSeconds,
+        double sceneScore,
+        double representativeTimestampSeconds
+) {
+}

src/main/resources/application.yml

@@ -23,6 +23,8 @@ video-clipping:
     contact-sheet-columns: ${VIDEO_EDITING_CONTACT_SHEET_COLUMNS:5}
     proxy-enabled: ${VIDEO_EDITING_PROXY_ENABLED:true}
     proxy-width: ${VIDEO_EDITING_PROXY_WIDTH:640}
+    scene-detection-threshold: ${VIDEO_EDITING_SCENE_DETECTION_THRESHOLD:0.35}
+    minimum-scene-duration-seconds: ${VIDEO_EDITING_MINIMUM_SCENE_DURATION_SECONDS:1.0}
     target-duration-seconds: ${VIDEO_EDITING_TARGET_DURATION_SECONDS:60}
     output-width: ${VIDEO_EDITING_OUTPUT_WIDTH:1920}
     output-height: ${VIDEO_EDITING_OUTPUT_HEIGHT:1080}

src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java

@@ -27,6 +27,8 @@ class VideoClippingPropertiesTest {
             assertThat(properties.getEditing().getContactSheetColumns()).isEqualTo(5);
             assertThat(properties.getEditing().isProxyEnabled()).isTrue();
             assertThat(properties.getEditing().getProxyWidth()).isEqualTo(640);
+            assertThat(properties.getEditing().getSceneDetectionThreshold()).isEqualTo(0.35);
+            assertThat(properties.getEditing().getMinimumSceneDurationSeconds()).isEqualTo(1.0);
             assertThat(properties.getEditing().getTargetDurationSeconds()).isEqualTo(60);
             assertThat(properties.getEditing().getOutputWidth()).isEqualTo(1920);
             assertThat(properties.getEditing().getOutputHeight()).isEqualTo(1080);
@@ -71,6 +73,8 @@ class VideoClippingPropertiesTest {
                         "video-clipping.editing.project-directory=/tmp/edit-projects",
                         "video-clipping.editing.highlight-project-directory=/tmp/highlight-projects",
                         "video-clipping.editing.thumbnail-count-per-clip=7",
+                        "video-clipping.editing.scene-detection-threshold=0.42",
+                        "video-clipping.editing.minimum-scene-duration-seconds=2.5",
                         "video-clipping.editing.voiceover-provider=remote",
                         "video-clipping.editing.loudness-target-i=-14",
                         "video-clipping.editing.loudness-true-peak=-2",
@@ -101,6 +105,8 @@ class VideoClippingPropertiesTest {
                     assertThat(properties.getEditing().getProjectDirectory()).isEqualTo("/tmp/edit-projects");
                     assertThat(properties.getEditing().getHighlightProjectDirectory()).isEqualTo("/tmp/highlight-projects");
                     assertThat(properties.getEditing().getThumbnailCountPerClip()).isEqualTo(7);
+                    assertThat(properties.getEditing().getSceneDetectionThreshold()).isEqualTo(0.42);
+                    assertThat(properties.getEditing().getMinimumSceneDurationSeconds()).isEqualTo(2.5);
                     assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("remote");
                     assertThat(properties.getEditing().getLoudnessTargetI()).isEqualTo(-14.0);
                     assertThat(properties.getEditing().getLoudnessTruePeak()).isEqualTo(-2.0);

src/test/java/org/example/videoclips/editing/HighlightSourceAnalyzerTest.java

@@ -59,13 +59,20 @@ class HighlightSourceAnalyzerTest {
                 .isEqualTo(store.analysisDirectory("porsche").resolve("proxies/porsche.mp4").toString());
         assertThat(result.waveformPath())
                 .isEqualTo(store.analysisDirectory("porsche").resolve("audio/porsche-waveform.png").toString());
+        assertThat(result.sceneSegmentsPath()).isEqualTo("analysis/scene-segments.json");
+        assertThat(result.shotSegments()).containsExactly(
+                new ShotSegment("shot_0001", 0, 12, 12, 0, 6),
+                new ShotSegment("shot_0002", 12, 42, 30, 0.61, 27)
+        );
         assertThat(result.createdAt()).isEqualTo(Instant.parse("2026-07-11T08:00:00Z"));
 
         ClipAnalysis ffprobe = store.readJson("porsche", "analysis/ffprobe.json", ClipAnalysis.class);
+        ShotSegment[] shots = store.readJson("porsche", "analysis/scene-segments.json", ShotSegment[].class);
         HighlightSourceAnalysis persisted = store.readJson("porsche", "analysis/source-analysis.json",
                 HighlightSourceAnalysis.class);
         assertThat(ffprobe.sourcePath()).isEqualTo(source.toString());
         assertThat(ffprobe.thumbnails()).hasSize(2);
+        assertThat(shots).containsExactly(result.shotSegments().toArray(ShotSegment[]::new));
         assertThat(persisted).isEqualTo(result);
     }
 
@@ -102,7 +109,14 @@ class HighlightSourceAnalyzerTest {
         when(waveform.generate(any(), any())).thenAnswer(invocation ->
                 invocation.<Path>getArgument(1).resolve("porsche-waveform.png").toString());
 
+        ShotSceneSegmenter shotSceneSegmenter = mock(ShotSceneSegmenter.class);
+        when(shotSceneSegmenter.segment(any())).thenReturn(List.of(
+                new ShotSegment("shot_0001", 0, 12, 12, 0, 6),
+                new ShotSegment("shot_0002", 12, 42, 30, 0.61, 27)
+        ));
+
         Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC);
-        return new HighlightSourceAnalyzer(store, inspector, thumbnails, contactSheet, proxy, waveform, clock);
+        return new HighlightSourceAnalyzer(store, inspector, thumbnails, contactSheet, proxy, waveform,
+                shotSceneSegmenter, clock);
     }
 }

src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java

@@ -115,6 +115,7 @@ class HighlightSourceSchedulerTest {
         ClipAnalysis source = new ClipAnalysis(projectId, store.sourceDirectory(projectId).resolve(projectId + ".mp4")
                 .toString(), 42.0, "h264", "aac", 1920, 1080, 30.0, List.of(), null, null, 0, 0, 0);
         return new HighlightSourceAnalysis(projectId, projectId + ".mp4", source, List.of(), null, null, null,
+                "analysis/scene-segments.json", List.of(new ShotSegment("shot_0001", 0, 42, 42, 0, 21)),
                 Instant.parse("2026-07-11T08:00:00Z"));
     }
 }

src/test/java/org/example/videoclips/editing/ShotSceneSegmenterTest.java

@@ -0,0 +1,90 @@
+package org.example.videoclips.editing;
+
+import org.example.videoclips.config.VideoClippingProperties;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ShotSceneSegmenterTest {
+
+    @Test
+    void detectsShotSegmentsFromFfmpegSceneMetadata() {
+        AtomicReference<List<String>> command = new AtomicReference<>();
+        VideoClippingProperties properties = new VideoClippingProperties();
+        properties.getEditing().setSceneDetectionThreshold(0.42);
+        properties.getEditing().setMinimumSceneDurationSeconds(1.5);
+        ShotSceneSegmenter segmenter = new ShotSceneSegmenter(properties, arguments -> {
+            command.set(arguments);
+            return new ShotSceneSegmenter.ProcessResult(0, """
+                    frame:1 pts:72000 pts_time:2.400
+                    lavfi.scene_score=0.510
+                    frame:2 pts:180000 pts_time:6.000
+                    lavfi.scene_score=0.730
+                    """);
+        });
+
+        List<ShotSegment> shots = segmenter.segment(clip(9));
+
+        assertThat(shots).containsExactly(
+                new ShotSegment("shot_0001", 0, 2.4, 2.4, 0, 1.2),
+                new ShotSegment("shot_0002", 2.4, 6, 3.6, 0.51, 4.2),
+                new ShotSegment("shot_0003", 6, 9, 3, 0.73, 7.5)
+        );
+        assertCommandPair(command.get(), "-i", "source.mp4");
+        assertCommandPair(command.get(), "-filter:v", "select='gt(scene,0.42)',metadata=print");
+    }
+
+    @Test
+    void returnsSingleShotWhenNoSceneCutPassesMinimumDurationRules() {
+        ShotSceneSegmenter segmenter = new ShotSceneSegmenter(new VideoClippingProperties(), arguments ->
+                new ShotSceneSegmenter.ProcessResult(0, """
+                        frame:1 pts:12000 pts_time:0.400
+                        lavfi.scene_score=0.900
+                        """));
+
+        assertThat(segmenter.segment(clip(4))).containsExactly(
+                new ShotSegment("shot_0001", 0, 4, 4, 0, 2)
+        );
+    }
+
+    @Test
+    void rejectsFfmpegFailures() {
+        ShotSceneSegmenter segmenter = new ShotSceneSegmenter(new VideoClippingProperties(), command ->
+                new ShotSceneSegmenter.ProcessResult(1, "failed"));
+
+        assertThrows(IllegalStateException.class, () -> segmenter.segment(clip(9)));
+    }
+
+    @Test
+    void reportsProcessStartupAndInterruptionFailures() {
+        assertThrows(IllegalStateException.class, () -> new ShotSceneSegmenter(new VideoClippingProperties(), command -> {
+            throw new IOException("missing executable");
+        }).segment(clip(9)));
+
+        try {
+            assertThrows(IllegalStateException.class, () -> new ShotSceneSegmenter(new VideoClippingProperties(), command -> {
+                throw new InterruptedException("stopped");
+            }).segment(clip(9)));
+            assertTrue(Thread.currentThread().isInterrupted());
+        } finally {
+            Thread.interrupted();
+        }
+    }
+
+    private ClipAnalysis clip(double durationSeconds) {
+        return new ClipAnalysis("source", "source.mp4", durationSeconds, "h264", "aac",
+                1920, 1080, 30.0, List.of(), null, null, 0, 0, 0);
+    }
+
+    private void assertCommandPair(List<String> command, String option, String value) {
+        int index = command.indexOf(option);
+        assertThat(index).isGreaterThanOrEqualTo(0);
+        assertThat(command.get(index + 1)).isEqualTo(value);
+    }
+}