Add highlight source analysis

2026-07-11 09:59:55 +02:00 · 2026-07-11 09:59:55 +02:00 · ed337c1268
parent 0b6e5afddf
commit ed337c1268
8 changed files with 415 additions and 7 deletions
--- a/docs/production-cinematic-highlight-editing-plan.md
+++ b/docs/production-cinematic-highlight-editing-plan.md
@ -299,7 +299,8 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can
 - [x] Milestone 1 progress: Added a dedicated highlight project domain, standard folder contract, manifest model, configurable `output/highlight-projects` root, filesystem store, and tests for directory creation, JSON round trips, and path safety.
 - [x] Milestone 2: Add a local scheduler that accepts exactly one source video at a time from the highlight source folder.
 - [x] Milestone 2 progress: Added a configurable highlight source scheduler and directory initializer for `input/highlights/source`, `working`, `processed`, and `rejected`; it claims one valid source video per scan, creates a highlight project and manifest, copies the source into the project folder, and leaves later videos for later scans.
- [ ] Milestone 3: Implement source probing, proxy generation, frame extraction, waveform extraction, and contact sheet creation.
+- [x] Milestone 3: Implement source probing, proxy generation, frame extraction, waveform extraction, and contact sheet creation.
+- [x] Milestone 3 progress: Added highlight source analysis for the new project contract, reusing FFprobe inspection, frame extraction, contact sheet generation, proxy generation, and adding waveform image generation under `analysis/`; the highlight scheduler now runs analysis immediately after creating a project.
 - [ ] Milestone 4: Implement shot and scene segmentation using FFmpeg scene detection or PySceneDetect.
 - [ ] Milestone 5: Implement audio analysis for loudness, silence, peaks, speech/music/noise sections, and optional ASR transcript.
 - [ ] Milestone 6: Implement visual analysis for blur, exposure, motion, faces, object labels, composition quality, and representative thumbnails.
--- a/src/main/java/org/example/videoclips/editing/HighlightSourceAnalysis.java
+++ b/src/main/java/org/example/videoclips/editing/HighlightSourceAnalysis.java
@ -0,0 +1,16 @@
+package org.example.videoclips.editing;
+
+import java.time.Instant;
+import java.util.List;
+
+public record HighlightSourceAnalysis(
+        String projectId,
+        String sourceVideoFileName,
+        ClipAnalysis source,
+        List<String> thumbnails,
+        String contactSheet,
+        String proxyPath,
+        String waveformPath,
+        Instant createdAt
+) {
+}
--- a/src/main/java/org/example/videoclips/editing/HighlightSourceAnalyzer.java
+++ b/src/main/java/org/example/videoclips/editing/HighlightSourceAnalyzer.java
@ -0,0 +1,122 @@
+package org.example.videoclips.editing;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Service;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Instant;
+import java.util.Comparator;
+import java.util.List;
+
+@Service
+@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true)
+public class HighlightSourceAnalyzer {
+
+    private final HighlightProjectStore store;
+    private final FfmpegClipInspector inspector;
+    private final ThumbnailExtractor thumbnailExtractor;
+    private final ContactSheetGenerator contactSheetGenerator;
+    private final ProxyGenerator proxyGenerator;
+    private final WaveformGenerator waveformGenerator;
+    private final Clock clock;
+
+    @Autowired
+    public HighlightSourceAnalyzer(
+            HighlightProjectStore store,
+            FfmpegClipInspector inspector,
+            ThumbnailExtractor thumbnailExtractor,
+            ContactSheetGenerator contactSheetGenerator,
+            ProxyGenerator proxyGenerator,
+            WaveformGenerator waveformGenerator
+    ) {
+        this(store, inspector, thumbnailExtractor, contactSheetGenerator, proxyGenerator, waveformGenerator,
+                Clock.systemUTC());
+    }
+
+    HighlightSourceAnalyzer(
+            HighlightProjectStore store,
+            FfmpegClipInspector inspector,
+            ThumbnailExtractor thumbnailExtractor,
+            ContactSheetGenerator contactSheetGenerator,
+            ProxyGenerator proxyGenerator,
+            WaveformGenerator waveformGenerator,
+            Clock clock
+    ) {
+        this.store = store;
+        this.inspector = inspector;
+        this.thumbnailExtractor = thumbnailExtractor;
+        this.contactSheetGenerator = contactSheetGenerator;
+        this.proxyGenerator = proxyGenerator;
+        this.waveformGenerator = waveformGenerator;
+        this.clock = clock;
+    }
+
+    public HighlightSourceAnalysis analyze(String projectId) {
+        HighlightProject project = store.readJson(projectId, "project.json", HighlightProject.class);
+        Path source = sourceVideo(projectId);
+        ClipAnalysis inspected = inspector.inspect(source);
+        Path analysisDirectory = store.analysisDirectory(projectId);
+        Path frameDirectory = analysisDirectory.resolve("frames");
+        Path contactSheetDirectory = analysisDirectory.resolve("contact-sheets");
+        Path proxyDirectory = analysisDirectory.resolve("proxies");
+        Path audioDirectory = analysisDirectory.resolve("audio");
+
+        List<String> thumbnails = thumbnailExtractor.extract(inspected, frameDirectory);
+        String contactSheet = contactSheetGenerator.generate(inspected, contactSheetDirectory);
+        String proxyPath = proxyGenerator.generate(inspected, proxyDirectory).orElse(null);
+        String waveformPath = waveformGenerator.generate(inspected, audioDirectory);
+        ClipAnalysis sourceAnalysis = enriched(inspected, thumbnails, contactSheet, proxyPath);
+        HighlightSourceAnalysis analysis = new HighlightSourceAnalysis(
+                projectId,
+                project.sourceVideoFileName(),
+                sourceAnalysis,
+                List.copyOf(thumbnails),
+                contactSheet,
+                proxyPath,
+                waveformPath,
+                Instant.now(clock)
+        );
+        store.writeJson(projectId, "analysis/ffprobe.json", sourceAnalysis);
+        store.writeJson(projectId, "analysis/source-analysis.json", analysis);
+        return analysis;
+    }
+
+    private Path sourceVideo(String projectId) {
+        Path sourceDirectory = store.sourceDirectory(projectId);
+        try (var files = Files.list(sourceDirectory)) {
+            return files.filter(Files::isRegularFile)
+                    .sorted(Comparator.comparing(path -> path.getFileName().toString()))
+                    .findFirst()
+                    .orElseThrow(() -> new IllegalStateException("Highlight project has no source video: " + projectId));
+        } catch (java.io.IOException ex) {
+            throw new IllegalStateException("Unable to read highlight project source directory: " + sourceDirectory, ex);
+        }
+    }
+
+    private ClipAnalysis enriched(
+            ClipAnalysis inspected,
+            List<String> thumbnails,
+            String contactSheet,
+            String proxyPath
+    ) {
+        return new ClipAnalysis(
+                inspected.clipId(),
+                inspected.sourcePath(),
+                inspected.durationSeconds(),
+                inspected.videoCodec(),
+                inspected.audioCodec(),
+                inspected.width(),
+                inspected.height(),
+                inspected.frameRate(),
+                thumbnails,
+                contactSheet,
+                proxyPath,
+                inspected.motionScore(),
+                inspected.brightnessScore(),
+                inspected.sharpnessScore()
+        );
+    }
+}
--- a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java
+++ b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java
@ -38,18 +38,29 @@ public class HighlightSourceScheduler {

    private final VideoClippingProperties.Editing.HighlightScheduler properties;
    private final HighlightProjectStore store;
+    private final HighlightSourceAnalyzer analyzer;
    private final Clock clock;
    private final AtomicBoolean scanning = new AtomicBoolean(false);
    private final AtomicLong scanSequence = new AtomicLong();

    @Autowired
-    public HighlightSourceScheduler(VideoClippingProperties properties, HighlightProjectStore store) {
-        this(properties, store, Clock.systemUTC());
+    public HighlightSourceScheduler(
+            VideoClippingProperties properties,
+            HighlightProjectStore store,
+            HighlightSourceAnalyzer analyzer
+    ) {
+        this(properties, store, analyzer, Clock.systemUTC());
    }

-    HighlightSourceScheduler(VideoClippingProperties properties, HighlightProjectStore store, Clock clock) {
+    HighlightSourceScheduler(
+            VideoClippingProperties properties,
+            HighlightProjectStore store,
+            HighlightSourceAnalyzer analyzer,
+            Clock clock
+    ) {
        this.properties = properties.getEditing().getHighlightScheduler();
        this.store = store;
+        this.analyzer = analyzer;
        this.clock = clock;
    }

@ -135,10 +146,16 @@ public class HighlightSourceScheduler {
        );
        Path projectDirectory = store.createProject(project, manifest);
        copySourceToProject(workingFile, store.sourceDirectory(projectId).resolve(workingFile.getFileName()));
+        HighlightSourceAnalysis analysis = analyzer.analyze(projectId);
        Path processedFile = moveToDirectory(workingFile, Path.of(properties.getProcessedDirectory()));
        log.info("event=highlight_project_created scan_id={} project_id={} source_file={} project_directory={} "
-                        + "elapsed_ms={}",
-                scanId, projectId, processedFile.getFileName(), projectDirectory, elapsedMillis(startedAt));
+                        + "analysis_file={} elapsed_ms={}",
+                scanId, projectId, processedFile.getFileName(), projectDirectory, "analysis/source-analysis.json",
+                elapsedMillis(startedAt));
+        log.info("event=highlight_source_analyzed scan_id={} project_id={} duration_seconds={} thumbnails={} "
+                        + "proxy={} waveform={}",
+                scanId, projectId, analysis.source().durationSeconds(), analysis.thumbnails().size(),
+                analysis.proxyPath(), analysis.waveformPath());
    }

    private void handleFailure(long scanId, Path activeFile, RuntimeException failure) {
--- a/src/main/java/org/example/videoclips/editing/WaveformGenerator.java
+++ b/src/main/java/org/example/videoclips/editing/WaveformGenerator.java
@ -0,0 +1,74 @@
+package org.example.videoclips.editing;
+
+import org.example.videoclips.config.VideoClippingProperties;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.stereotype.Component;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+@Component
+@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true)
+public class WaveformGenerator {
+
+    private final VideoClippingProperties.Editing properties;
+    private final ProcessExecutor processExecutor;
+
+    @Autowired
+    public WaveformGenerator(VideoClippingProperties properties) {
+        this(properties, WaveformGenerator::execute);
+    }
+
+    WaveformGenerator(VideoClippingProperties properties, ProcessExecutor processExecutor) {
+        this.properties = properties.getEditing();
+        this.processExecutor = processExecutor;
+    }
+
+    public String generate(ClipAnalysis source, Path audioDirectory) {
+        try {
+            Files.createDirectories(audioDirectory);
+        } catch (IOException ex) {
+            throw new IllegalStateException("Unable to create waveform directory", ex);
+        }
+        Path output = audioDirectory.resolve(source.clipId() + "-waveform.png");
+        List<String> command = List.of(
+                properties.getFfmpegBinary(),
+                "-hide_banner", "-y",
+                "-i", source.sourcePath(),
+                "-filter_complex", "aformat=channel_layouts=mono,showwavespic=s=1280x240:colors=DodgerBlue",
+                "-frames:v", "1",
+                output.toString()
+        );
+        ProcessResult result;
+        try {
+            result = processExecutor.execute(command);
+        } catch (InterruptedException ex) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("Interrupted while generating waveform", ex);
+        } catch (IOException ex) {
+            throw new IllegalStateException("Unable to run FFmpeg for waveform generation", ex);
+        }
+        if (result.exitCode() != 0) {
+            throw new IllegalStateException("FFmpeg waveform generation exited with code " + result.exitCode());
+        }
+        return output.toString();
+    }
+
+    private static ProcessResult execute(List<String> command) throws IOException, InterruptedException {
+        Process process = new ProcessBuilder(command).redirectErrorStream(true).start();
+        String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
+        return new ProcessResult(process.waitFor(), output);
+    }
+
+    record ProcessResult(int exitCode, String output) {
+    }
+
+    @FunctionalInterface
+    interface ProcessExecutor {
+        ProcessResult execute(List<String> command) throws IOException, InterruptedException;
+    }
+}
--- a/src/test/java/org/example/videoclips/editing/HighlightSourceAnalyzerTest.java
+++ b/src/test/java/org/example/videoclips/editing/HighlightSourceAnalyzerTest.java
@ -0,0 +1,108 @@
+package org.example.videoclips.editing;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import org.example.videoclips.config.VideoClippingProperties;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Clock;
+import java.time.Instant;
+import java.time.ZoneOffset;
+import java.util.List;
+import java.util.Optional;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class HighlightSourceAnalyzerTest {
+
+    @TempDir
+    Path tempDir;
+
+    private FileSystemHighlightProjectStore store;
+
+    @BeforeEach
+    void setUp() {
+        VideoClippingProperties properties = new VideoClippingProperties();
+        properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString());
+        store = new FileSystemHighlightProjectStore(properties, new ObjectMapper().findAndRegisterModules());
+        Instant now = Instant.parse("2026-07-11T08:00:00Z");
+        HighlightProject project = new HighlightProject("porsche", "Porsche", HighlightProjectStatus.CREATED,
+                "porsche.mp4", store.projectDirectory("porsche").toString(), now, now, null);
+        HighlightProjectManifest manifest = new HighlightProjectManifest("porsche", "porsche.mp4",
+                HighlightFolderContract.standard(), now);
+        store.createProject(project, manifest);
+    }
+
+    @Test
+    void writesSourceAnalysisArtifactsUnderAnalysisContract() throws Exception {
+        Path source = Files.writeString(store.sourceDirectory("porsche").resolve("porsche.mp4"), "video");
+        HighlightSourceAnalyzer analyzer = analyzer(source);
+
+        HighlightSourceAnalysis result = analyzer.analyze("porsche");
+
+        assertThat(result.projectId()).isEqualTo("porsche");
+        assertThat(result.sourceVideoFileName()).isEqualTo("porsche.mp4");
+        assertThat(result.thumbnails()).containsExactly(
+                store.analysisDirectory("porsche").resolve("frames/porsche_0001.jpg").toString(),
+                store.analysisDirectory("porsche").resolve("frames/porsche_0002.jpg").toString()
+        );
+        assertThat(result.contactSheet())
+                .isEqualTo(store.analysisDirectory("porsche").resolve("contact-sheets/porsche.jpg").toString());
+        assertThat(result.proxyPath())
+                .isEqualTo(store.analysisDirectory("porsche").resolve("proxies/porsche.mp4").toString());
+        assertThat(result.waveformPath())
+                .isEqualTo(store.analysisDirectory("porsche").resolve("audio/porsche-waveform.png").toString());
+        assertThat(result.createdAt()).isEqualTo(Instant.parse("2026-07-11T08:00:00Z"));
+
+        ClipAnalysis ffprobe = store.readJson("porsche", "analysis/ffprobe.json", ClipAnalysis.class);
+        HighlightSourceAnalysis persisted = store.readJson("porsche", "analysis/source-analysis.json",
+                HighlightSourceAnalysis.class);
+        assertThat(ffprobe.sourcePath()).isEqualTo(source.toString());
+        assertThat(ffprobe.thumbnails()).hasSize(2);
+        assertThat(persisted).isEqualTo(result);
+    }
+
+    @Test
+    void failsWhenProjectHasNoSourceVideo() {
+        HighlightSourceAnalyzer analyzer = analyzer(Path.of("missing.mp4"));
+
+        assertThrows(IllegalStateException.class, () -> analyzer.analyze("porsche"));
+    }
+
+    private HighlightSourceAnalyzer analyzer(Path source) {
+        FfmpegClipInspector inspector = mock(FfmpegClipInspector.class);
+        when(inspector.inspect(any())).thenReturn(new ClipAnalysis("porsche", source.toString(), 42.0,
+                "h264", "aac", 1920, 1080, 30.0, List.of(), null, null, 0.4, 0.5, 0.6));
+
+        ThumbnailExtractor thumbnails = mock(ThumbnailExtractor.class);
+        when(thumbnails.extract(any(), any())).thenAnswer(invocation -> {
+            Path directory = invocation.getArgument(1);
+            return List.of(
+                    directory.resolve("porsche_0001.jpg").toString(),
+                    directory.resolve("porsche_0002.jpg").toString()
+            );
+        });
+
+        ContactSheetGenerator contactSheet = mock(ContactSheetGenerator.class);
+        when(contactSheet.generate(any(), any())).thenAnswer(invocation ->
+                invocation.<Path>getArgument(1).resolve("porsche.jpg").toString());
+
+        ProxyGenerator proxy = mock(ProxyGenerator.class);
+        when(proxy.generate(any(), any())).thenAnswer(invocation ->
+                Optional.of(invocation.<Path>getArgument(1).resolve("porsche.mp4").toString()));
+
+        WaveformGenerator waveform = mock(WaveformGenerator.class);
+        when(waveform.generate(any(), any())).thenAnswer(invocation ->
+                invocation.<Path>getArgument(1).resolve("porsche-waveform.png").toString());
+
+        Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC);
+        return new HighlightSourceAnalyzer(store, inspector, thumbnails, contactSheet, proxy, waveform, clock);
+    }
+}
--- a/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java
+++ b/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java
@ -11,9 +11,13 @@ import java.nio.file.Path;
 import java.time.Clock;
 import java.time.Instant;
 import java.time.ZoneOffset;
+import java.util.List;
 import java.util.Optional;

 import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;

 class HighlightSourceSchedulerTest {

@ -22,6 +26,7 @@ class HighlightSourceSchedulerTest {

    private VideoClippingProperties properties;
    private FileSystemHighlightProjectStore store;
+    private HighlightSourceAnalyzer analyzer;

    @BeforeEach
    void setUp() {
@ -34,6 +39,7 @@ class HighlightSourceSchedulerTest {
        scheduler.setRejectedDirectory(tempDir.resolve("rejected").toString());
        new HighlightProjectDirectoryInitializer(properties).initialize();
        store = new FileSystemHighlightProjectStore(properties, new ObjectMapper().findAndRegisterModules());
+        analyzer = mock(HighlightSourceAnalyzer.class);
    }

    @Test
@ -66,6 +72,7 @@ class HighlightSourceSchedulerTest {
        assertThat(tempDir.resolve("highlight-projects/1/manifest.json")).exists();
        assertThat(tempDir.resolve("highlight-projects/1/source/1.mp4")).exists();
        assertThat(tempDir.resolve("highlight-projects/2")).doesNotExist();
+        verify(analyzer).analyze("1");
    }

    @Test
@ -97,6 +104,17 @@ class HighlightSourceSchedulerTest {

    private HighlightSourceScheduler scheduler() {
        Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC);
-        return new HighlightSourceScheduler(properties, store, clock);
+        when(analyzer.analyze("1")).thenReturn(analysis("1"));
+        when(analyzer.analyze("porsche")).thenReturn(analysis("porsche"));
+        when(analyzer.analyze("porsche-1")).thenReturn(analysis("porsche-1"));
+        when(analyzer.analyze("porsche-drive")).thenReturn(analysis("porsche-drive"));
+        return new HighlightSourceScheduler(properties, store, analyzer, clock);
+    }
+
+    private HighlightSourceAnalysis analysis(String projectId) {
+        ClipAnalysis source = new ClipAnalysis(projectId, store.sourceDirectory(projectId).resolve(projectId + ".mp4")
+                .toString(), 42.0, "h264", "aac", 1920, 1080, 30.0, List.of(), null, null, 0, 0, 0);
+        return new HighlightSourceAnalysis(projectId, projectId + ".mp4", source, List.of(), null, null, null,
+                Instant.parse("2026-07-11T08:00:00Z"));
    }
 }
--- a/src/test/java/org/example/videoclips/editing/WaveformGeneratorTest.java
+++ b/src/test/java/org/example/videoclips/editing/WaveformGeneratorTest.java
@ -0,0 +1,52 @@
+package org.example.videoclips.editing;
+
+import org.example.videoclips.config.VideoClippingProperties;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class WaveformGeneratorTest {
+
+    @TempDir
+    Path tempDir;
+
+    @Test
+    void generatesWaveformImageWithShowwavespicFilter() {
+        VideoClippingProperties properties = new VideoClippingProperties();
+        WaveformGenerator generator = new WaveformGenerator(properties, command -> {
+            assertThat(command).containsSubsequence(
+                    "ffmpeg",
+                    "-hide_banner", "-y",
+                    "-i", "source.mp4",
+                    "-filter_complex"
+            );
+            assertThat(command).contains("aformat=channel_layouts=mono,showwavespic=s=1280x240:colors=DodgerBlue");
+            Files.writeString(Path.of(command.get(command.size() - 1)), "waveform");
+            return new WaveformGenerator.ProcessResult(0, "ok");
+        });
+
+        String waveform = generator.generate(clip(), tempDir.resolve("audio"));
+
+        assertThat(waveform).isEqualTo(tempDir.resolve("audio/source-waveform.png").toString());
+        assertThat(Path.of(waveform)).exists();
+    }
+
+    @Test
+    void failsWhenFfmpegFails() {
+        WaveformGenerator generator = new WaveformGenerator(new VideoClippingProperties(),
+                command -> new WaveformGenerator.ProcessResult(1, "failed"));
+
+        assertThrows(IllegalStateException.class, () -> generator.generate(clip(), tempDir.resolve("audio")));
+    }
+
+    private ClipAnalysis clip() {
+        return new ClipAnalysis("source", "source.mp4", 10.0, "h264", "aac", 1920, 1080,
+                30.0, List.of(), null, null, 0, 0, 0);
+    }
+}