diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index a7fcd86..e4711f0 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -299,7 +299,8 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 1 progress: Added a dedicated highlight project domain, standard folder contract, manifest model, configurable `output/highlight-projects` root, filesystem store, and tests for directory creation, JSON round trips, and path safety. - [x] Milestone 2: Add a local scheduler that accepts exactly one source video at a time from the highlight source folder. - [x] Milestone 2 progress: Added a configurable highlight source scheduler and directory initializer for `input/highlights/source`, `working`, `processed`, and `rejected`; it claims one valid source video per scan, creates a highlight project and manifest, copies the source into the project folder, and leaves later videos for later scans. -- [ ] Milestone 3: Implement source probing, proxy generation, frame extraction, waveform extraction, and contact sheet creation. +- [x] Milestone 3: Implement source probing, proxy generation, frame extraction, waveform extraction, and contact sheet creation. +- [x] Milestone 3 progress: Added highlight source analysis for the new project contract, reusing FFprobe inspection, frame extraction, contact sheet generation, proxy generation, and adding waveform image generation under `analysis/`; the highlight scheduler now runs analysis immediately after creating a project. - [ ] Milestone 4: Implement shot and scene segmentation using FFmpeg scene detection or PySceneDetect. - [ ] Milestone 5: Implement audio analysis for loudness, silence, peaks, speech/music/noise sections, and optional ASR transcript. - [ ] Milestone 6: Implement visual analysis for blur, exposure, motion, faces, object labels, composition quality, and representative thumbnails. diff --git a/src/main/java/org/example/videoclips/editing/HighlightSourceAnalysis.java b/src/main/java/org/example/videoclips/editing/HighlightSourceAnalysis.java new file mode 100644 index 0000000..3fb00b5 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightSourceAnalysis.java @@ -0,0 +1,16 @@ +package org.example.videoclips.editing; + +import java.time.Instant; +import java.util.List; + +public record HighlightSourceAnalysis( + String projectId, + String sourceVideoFileName, + ClipAnalysis source, + List thumbnails, + String contactSheet, + String proxyPath, + String waveformPath, + Instant createdAt +) { +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightSourceAnalyzer.java b/src/main/java/org/example/videoclips/editing/HighlightSourceAnalyzer.java new file mode 100644 index 0000000..88b5f00 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightSourceAnalyzer.java @@ -0,0 +1,122 @@ +package org.example.videoclips.editing; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Service; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.util.Comparator; +import java.util.List; + +@Service +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HighlightSourceAnalyzer { + + private final HighlightProjectStore store; + private final FfmpegClipInspector inspector; + private final ThumbnailExtractor thumbnailExtractor; + private final ContactSheetGenerator contactSheetGenerator; + private final ProxyGenerator proxyGenerator; + private final WaveformGenerator waveformGenerator; + private final Clock clock; + + @Autowired + public HighlightSourceAnalyzer( + HighlightProjectStore store, + FfmpegClipInspector inspector, + ThumbnailExtractor thumbnailExtractor, + ContactSheetGenerator contactSheetGenerator, + ProxyGenerator proxyGenerator, + WaveformGenerator waveformGenerator + ) { + this(store, inspector, thumbnailExtractor, contactSheetGenerator, proxyGenerator, waveformGenerator, + Clock.systemUTC()); + } + + HighlightSourceAnalyzer( + HighlightProjectStore store, + FfmpegClipInspector inspector, + ThumbnailExtractor thumbnailExtractor, + ContactSheetGenerator contactSheetGenerator, + ProxyGenerator proxyGenerator, + WaveformGenerator waveformGenerator, + Clock clock + ) { + this.store = store; + this.inspector = inspector; + this.thumbnailExtractor = thumbnailExtractor; + this.contactSheetGenerator = contactSheetGenerator; + this.proxyGenerator = proxyGenerator; + this.waveformGenerator = waveformGenerator; + this.clock = clock; + } + + public HighlightSourceAnalysis analyze(String projectId) { + HighlightProject project = store.readJson(projectId, "project.json", HighlightProject.class); + Path source = sourceVideo(projectId); + ClipAnalysis inspected = inspector.inspect(source); + Path analysisDirectory = store.analysisDirectory(projectId); + Path frameDirectory = analysisDirectory.resolve("frames"); + Path contactSheetDirectory = analysisDirectory.resolve("contact-sheets"); + Path proxyDirectory = analysisDirectory.resolve("proxies"); + Path audioDirectory = analysisDirectory.resolve("audio"); + + List thumbnails = thumbnailExtractor.extract(inspected, frameDirectory); + String contactSheet = contactSheetGenerator.generate(inspected, contactSheetDirectory); + String proxyPath = proxyGenerator.generate(inspected, proxyDirectory).orElse(null); + String waveformPath = waveformGenerator.generate(inspected, audioDirectory); + ClipAnalysis sourceAnalysis = enriched(inspected, thumbnails, contactSheet, proxyPath); + HighlightSourceAnalysis analysis = new HighlightSourceAnalysis( + projectId, + project.sourceVideoFileName(), + sourceAnalysis, + List.copyOf(thumbnails), + contactSheet, + proxyPath, + waveformPath, + Instant.now(clock) + ); + store.writeJson(projectId, "analysis/ffprobe.json", sourceAnalysis); + store.writeJson(projectId, "analysis/source-analysis.json", analysis); + return analysis; + } + + private Path sourceVideo(String projectId) { + Path sourceDirectory = store.sourceDirectory(projectId); + try (var files = Files.list(sourceDirectory)) { + return files.filter(Files::isRegularFile) + .sorted(Comparator.comparing(path -> path.getFileName().toString())) + .findFirst() + .orElseThrow(() -> new IllegalStateException("Highlight project has no source video: " + projectId)); + } catch (java.io.IOException ex) { + throw new IllegalStateException("Unable to read highlight project source directory: " + sourceDirectory, ex); + } + } + + private ClipAnalysis enriched( + ClipAnalysis inspected, + List thumbnails, + String contactSheet, + String proxyPath + ) { + return new ClipAnalysis( + inspected.clipId(), + inspected.sourcePath(), + inspected.durationSeconds(), + inspected.videoCodec(), + inspected.audioCodec(), + inspected.width(), + inspected.height(), + inspected.frameRate(), + thumbnails, + contactSheet, + proxyPath, + inspected.motionScore(), + inspected.brightnessScore(), + inspected.sharpnessScore() + ); + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java index 8346e11..acc40fc 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java +++ b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java @@ -38,18 +38,29 @@ public class HighlightSourceScheduler { private final VideoClippingProperties.Editing.HighlightScheduler properties; private final HighlightProjectStore store; + private final HighlightSourceAnalyzer analyzer; private final Clock clock; private final AtomicBoolean scanning = new AtomicBoolean(false); private final AtomicLong scanSequence = new AtomicLong(); @Autowired - public HighlightSourceScheduler(VideoClippingProperties properties, HighlightProjectStore store) { - this(properties, store, Clock.systemUTC()); + public HighlightSourceScheduler( + VideoClippingProperties properties, + HighlightProjectStore store, + HighlightSourceAnalyzer analyzer + ) { + this(properties, store, analyzer, Clock.systemUTC()); } - HighlightSourceScheduler(VideoClippingProperties properties, HighlightProjectStore store, Clock clock) { + HighlightSourceScheduler( + VideoClippingProperties properties, + HighlightProjectStore store, + HighlightSourceAnalyzer analyzer, + Clock clock + ) { this.properties = properties.getEditing().getHighlightScheduler(); this.store = store; + this.analyzer = analyzer; this.clock = clock; } @@ -135,10 +146,16 @@ public class HighlightSourceScheduler { ); Path projectDirectory = store.createProject(project, manifest); copySourceToProject(workingFile, store.sourceDirectory(projectId).resolve(workingFile.getFileName())); + HighlightSourceAnalysis analysis = analyzer.analyze(projectId); Path processedFile = moveToDirectory(workingFile, Path.of(properties.getProcessedDirectory())); log.info("event=highlight_project_created scan_id={} project_id={} source_file={} project_directory={} " - + "elapsed_ms={}", - scanId, projectId, processedFile.getFileName(), projectDirectory, elapsedMillis(startedAt)); + + "analysis_file={} elapsed_ms={}", + scanId, projectId, processedFile.getFileName(), projectDirectory, "analysis/source-analysis.json", + elapsedMillis(startedAt)); + log.info("event=highlight_source_analyzed scan_id={} project_id={} duration_seconds={} thumbnails={} " + + "proxy={} waveform={}", + scanId, projectId, analysis.source().durationSeconds(), analysis.thumbnails().size(), + analysis.proxyPath(), analysis.waveformPath()); } private void handleFailure(long scanId, Path activeFile, RuntimeException failure) { diff --git a/src/main/java/org/example/videoclips/editing/WaveformGenerator.java b/src/main/java/org/example/videoclips/editing/WaveformGenerator.java new file mode 100644 index 0000000..aaf7d5c --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/WaveformGenerator.java @@ -0,0 +1,74 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class WaveformGenerator { + + private final VideoClippingProperties.Editing properties; + private final ProcessExecutor processExecutor; + + @Autowired + public WaveformGenerator(VideoClippingProperties properties) { + this(properties, WaveformGenerator::execute); + } + + WaveformGenerator(VideoClippingProperties properties, ProcessExecutor processExecutor) { + this.properties = properties.getEditing(); + this.processExecutor = processExecutor; + } + + public String generate(ClipAnalysis source, Path audioDirectory) { + try { + Files.createDirectories(audioDirectory); + } catch (IOException ex) { + throw new IllegalStateException("Unable to create waveform directory", ex); + } + Path output = audioDirectory.resolve(source.clipId() + "-waveform.png"); + List command = List.of( + properties.getFfmpegBinary(), + "-hide_banner", "-y", + "-i", source.sourcePath(), + "-filter_complex", "aformat=channel_layouts=mono,showwavespic=s=1280x240:colors=DodgerBlue", + "-frames:v", "1", + output.toString() + ); + ProcessResult result; + try { + result = processExecutor.execute(command); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while generating waveform", ex); + } catch (IOException ex) { + throw new IllegalStateException("Unable to run FFmpeg for waveform generation", ex); + } + if (result.exitCode() != 0) { + throw new IllegalStateException("FFmpeg waveform generation exited with code " + result.exitCode()); + } + return output.toString(); + } + + private static ProcessResult execute(List command) throws IOException, InterruptedException { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + return new ProcessResult(process.waitFor(), output); + } + + record ProcessResult(int exitCode, String output) { + } + + @FunctionalInterface + interface ProcessExecutor { + ProcessResult execute(List command) throws IOException, InterruptedException; + } +} diff --git a/src/test/java/org/example/videoclips/editing/HighlightSourceAnalyzerTest.java b/src/test/java/org/example/videoclips/editing/HighlightSourceAnalyzerTest.java new file mode 100644 index 0000000..44a6fd8 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/HighlightSourceAnalyzerTest.java @@ -0,0 +1,108 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Clock; +import java.time.Instant; +import java.time.ZoneOffset; +import java.util.List; +import java.util.Optional; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class HighlightSourceAnalyzerTest { + + @TempDir + Path tempDir; + + private FileSystemHighlightProjectStore store; + + @BeforeEach + void setUp() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + store = new FileSystemHighlightProjectStore(properties, new ObjectMapper().findAndRegisterModules()); + Instant now = Instant.parse("2026-07-11T08:00:00Z"); + HighlightProject project = new HighlightProject("porsche", "Porsche", HighlightProjectStatus.CREATED, + "porsche.mp4", store.projectDirectory("porsche").toString(), now, now, null); + HighlightProjectManifest manifest = new HighlightProjectManifest("porsche", "porsche.mp4", + HighlightFolderContract.standard(), now); + store.createProject(project, manifest); + } + + @Test + void writesSourceAnalysisArtifactsUnderAnalysisContract() throws Exception { + Path source = Files.writeString(store.sourceDirectory("porsche").resolve("porsche.mp4"), "video"); + HighlightSourceAnalyzer analyzer = analyzer(source); + + HighlightSourceAnalysis result = analyzer.analyze("porsche"); + + assertThat(result.projectId()).isEqualTo("porsche"); + assertThat(result.sourceVideoFileName()).isEqualTo("porsche.mp4"); + assertThat(result.thumbnails()).containsExactly( + store.analysisDirectory("porsche").resolve("frames/porsche_0001.jpg").toString(), + store.analysisDirectory("porsche").resolve("frames/porsche_0002.jpg").toString() + ); + assertThat(result.contactSheet()) + .isEqualTo(store.analysisDirectory("porsche").resolve("contact-sheets/porsche.jpg").toString()); + assertThat(result.proxyPath()) + .isEqualTo(store.analysisDirectory("porsche").resolve("proxies/porsche.mp4").toString()); + assertThat(result.waveformPath()) + .isEqualTo(store.analysisDirectory("porsche").resolve("audio/porsche-waveform.png").toString()); + assertThat(result.createdAt()).isEqualTo(Instant.parse("2026-07-11T08:00:00Z")); + + ClipAnalysis ffprobe = store.readJson("porsche", "analysis/ffprobe.json", ClipAnalysis.class); + HighlightSourceAnalysis persisted = store.readJson("porsche", "analysis/source-analysis.json", + HighlightSourceAnalysis.class); + assertThat(ffprobe.sourcePath()).isEqualTo(source.toString()); + assertThat(ffprobe.thumbnails()).hasSize(2); + assertThat(persisted).isEqualTo(result); + } + + @Test + void failsWhenProjectHasNoSourceVideo() { + HighlightSourceAnalyzer analyzer = analyzer(Path.of("missing.mp4")); + + assertThrows(IllegalStateException.class, () -> analyzer.analyze("porsche")); + } + + private HighlightSourceAnalyzer analyzer(Path source) { + FfmpegClipInspector inspector = mock(FfmpegClipInspector.class); + when(inspector.inspect(any())).thenReturn(new ClipAnalysis("porsche", source.toString(), 42.0, + "h264", "aac", 1920, 1080, 30.0, List.of(), null, null, 0.4, 0.5, 0.6)); + + ThumbnailExtractor thumbnails = mock(ThumbnailExtractor.class); + when(thumbnails.extract(any(), any())).thenAnswer(invocation -> { + Path directory = invocation.getArgument(1); + return List.of( + directory.resolve("porsche_0001.jpg").toString(), + directory.resolve("porsche_0002.jpg").toString() + ); + }); + + ContactSheetGenerator contactSheet = mock(ContactSheetGenerator.class); + when(contactSheet.generate(any(), any())).thenAnswer(invocation -> + invocation.getArgument(1).resolve("porsche.jpg").toString()); + + ProxyGenerator proxy = mock(ProxyGenerator.class); + when(proxy.generate(any(), any())).thenAnswer(invocation -> + Optional.of(invocation.getArgument(1).resolve("porsche.mp4").toString())); + + WaveformGenerator waveform = mock(WaveformGenerator.class); + when(waveform.generate(any(), any())).thenAnswer(invocation -> + invocation.getArgument(1).resolve("porsche-waveform.png").toString()); + + Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC); + return new HighlightSourceAnalyzer(store, inspector, thumbnails, contactSheet, proxy, waveform, clock); + } +} diff --git a/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java b/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java index c516497..fa29562 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java @@ -11,9 +11,13 @@ import java.nio.file.Path; import java.time.Clock; import java.time.Instant; import java.time.ZoneOffset; +import java.util.List; import java.util.Optional; import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; class HighlightSourceSchedulerTest { @@ -22,6 +26,7 @@ class HighlightSourceSchedulerTest { private VideoClippingProperties properties; private FileSystemHighlightProjectStore store; + private HighlightSourceAnalyzer analyzer; @BeforeEach void setUp() { @@ -34,6 +39,7 @@ class HighlightSourceSchedulerTest { scheduler.setRejectedDirectory(tempDir.resolve("rejected").toString()); new HighlightProjectDirectoryInitializer(properties).initialize(); store = new FileSystemHighlightProjectStore(properties, new ObjectMapper().findAndRegisterModules()); + analyzer = mock(HighlightSourceAnalyzer.class); } @Test @@ -66,6 +72,7 @@ class HighlightSourceSchedulerTest { assertThat(tempDir.resolve("highlight-projects/1/manifest.json")).exists(); assertThat(tempDir.resolve("highlight-projects/1/source/1.mp4")).exists(); assertThat(tempDir.resolve("highlight-projects/2")).doesNotExist(); + verify(analyzer).analyze("1"); } @Test @@ -97,6 +104,17 @@ class HighlightSourceSchedulerTest { private HighlightSourceScheduler scheduler() { Clock clock = Clock.fixed(Instant.parse("2026-07-11T08:00:00Z"), ZoneOffset.UTC); - return new HighlightSourceScheduler(properties, store, clock); + when(analyzer.analyze("1")).thenReturn(analysis("1")); + when(analyzer.analyze("porsche")).thenReturn(analysis("porsche")); + when(analyzer.analyze("porsche-1")).thenReturn(analysis("porsche-1")); + when(analyzer.analyze("porsche-drive")).thenReturn(analysis("porsche-drive")); + return new HighlightSourceScheduler(properties, store, analyzer, clock); + } + + private HighlightSourceAnalysis analysis(String projectId) { + ClipAnalysis source = new ClipAnalysis(projectId, store.sourceDirectory(projectId).resolve(projectId + ".mp4") + .toString(), 42.0, "h264", "aac", 1920, 1080, 30.0, List.of(), null, null, 0, 0, 0); + return new HighlightSourceAnalysis(projectId, projectId + ".mp4", source, List.of(), null, null, null, + Instant.parse("2026-07-11T08:00:00Z")); } } diff --git a/src/test/java/org/example/videoclips/editing/WaveformGeneratorTest.java b/src/test/java/org/example/videoclips/editing/WaveformGeneratorTest.java new file mode 100644 index 0000000..2b97354 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/WaveformGeneratorTest.java @@ -0,0 +1,52 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class WaveformGeneratorTest { + + @TempDir + Path tempDir; + + @Test + void generatesWaveformImageWithShowwavespicFilter() { + VideoClippingProperties properties = new VideoClippingProperties(); + WaveformGenerator generator = new WaveformGenerator(properties, command -> { + assertThat(command).containsSubsequence( + "ffmpeg", + "-hide_banner", "-y", + "-i", "source.mp4", + "-filter_complex" + ); + assertThat(command).contains("aformat=channel_layouts=mono,showwavespic=s=1280x240:colors=DodgerBlue"); + Files.writeString(Path.of(command.get(command.size() - 1)), "waveform"); + return new WaveformGenerator.ProcessResult(0, "ok"); + }); + + String waveform = generator.generate(clip(), tempDir.resolve("audio")); + + assertThat(waveform).isEqualTo(tempDir.resolve("audio/source-waveform.png").toString()); + assertThat(Path.of(waveform)).exists(); + } + + @Test + void failsWhenFfmpegFails() { + WaveformGenerator generator = new WaveformGenerator(new VideoClippingProperties(), + command -> new WaveformGenerator.ProcessResult(1, "failed")); + + assertThrows(IllegalStateException.class, () -> generator.generate(clip(), tempDir.resolve("audio"))); + } + + private ClipAnalysis clip() { + return new ClipAnalysis("source", "source.mp4", 10.0, "h264", "aac", 1920, 1080, + 30.0, List.of(), null, null, 0, 0, 0); + } +}