From 43fccb7dc6efc5c8dc5236781d33a7aec2db5d76 Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Sat, 11 Jul 2026 01:06:38 +0200 Subject: [PATCH] Render cinematic text overlays --- ...uction-cinematic-highlight-editing-plan.md | 1 + .../editing/FfmpegEditRenderer.java | 104 ++++++++++++++++-- .../videoclips/editing/RenderManifest.java | 17 +++ .../editing/FfmpegEditRendererTest.java | 22 ++++ .../editing/RenderManifestTest.java | 5 +- 5 files changed, 138 insertions(+), 11 deletions(-) diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index 26f2ac5..bc93498 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -308,6 +308,7 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 11: Implement asset provider interfaces for voiceover, music, SFX, fonts, and LUTs. - [x] Milestone 12: Add a local licensed asset library with category tags and deterministic asset selection. - [ ] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests. +- [ ] Milestone 13 progress: Renderer now applies timed text overlays and records resolved/planned edit assets in the render manifest. Dynamic crop keyframes, audio ducking, and loudness normalization remain. - [ ] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters. - [ ] Milestone 15: Add an operator runbook for placing a video, running the service locally, choosing a model, reviewing the director plan, and finding final clips. - [ ] Milestone 16: Add integration tests with small fixture videos for family, food, car, and generic content. diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index 64fc5e7..9b6f096 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -3,6 +3,7 @@ package org.example.videoclips.editing; import org.example.videoclips.api.dto.EditProjectResponse; import org.example.videoclips.config.VideoClippingProperties; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; @@ -28,41 +29,46 @@ public class FfmpegEditRenderer implements EditRenderer { private final EditPlanValidator validator; private final ProcessExecutor executor; private final VoiceoverGenerator voiceoverGenerator; + private final EditAssetProvider assetProvider; private final EditObservability observability; public FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store, EditProjectService projectService, EditPlanValidator validator) { - this(properties, store, projectService, validator, null, null, FfmpegEditRenderer::execute); + this(properties, store, projectService, validator, null, null, null, FfmpegEditRenderer::execute); } public FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store, EditProjectService projectService, EditPlanValidator validator, VoiceoverGenerator voiceoverGenerator) { - this(properties, store, projectService, validator, voiceoverGenerator, null, FfmpegEditRenderer::execute); + this(properties, store, projectService, validator, voiceoverGenerator, null, null, + FfmpegEditRenderer::execute); } @Autowired public FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store, EditProjectService projectService, EditPlanValidator validator, - VoiceoverGenerator voiceoverGenerator, EditObservability observability) { - this(properties, store, projectService, validator, voiceoverGenerator, observability, + VoiceoverGenerator voiceoverGenerator, ObjectProvider assetProvider, + EditObservability observability) { + this(properties, store, projectService, validator, voiceoverGenerator, assetProvider.getIfAvailable(), + observability, FfmpegEditRenderer::execute); } FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store, EditProjectService projectService, EditPlanValidator validator, ProcessExecutor executor) { - this(properties, store, projectService, validator, null, null, executor); + this(properties, store, projectService, validator, null, null, null, executor); } FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store, EditProjectService projectService, EditPlanValidator validator, VoiceoverGenerator voiceoverGenerator, ProcessExecutor executor) { - this(properties, store, projectService, validator, voiceoverGenerator, null, executor); + this(properties, store, projectService, validator, voiceoverGenerator, null, null, executor); } FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store, EditProjectService projectService, EditPlanValidator validator, - VoiceoverGenerator voiceoverGenerator, EditObservability observability, + VoiceoverGenerator voiceoverGenerator, EditAssetProvider assetProvider, + EditObservability observability, ProcessExecutor executor) { this.properties = properties.getEditing(); this.store = store; @@ -70,6 +76,7 @@ public class FfmpegEditRenderer implements EditRenderer { this.validator = validator; this.executor = executor; this.voiceoverGenerator = voiceoverGenerator; + this.assetProvider = assetProvider; this.observability = observability; } @@ -124,24 +131,30 @@ public class FfmpegEditRenderer implements EditRenderer { writeConcatFile(concatFile, segments); Path timeline = work.resolve("timeline-video.mp4"); runAndRecord(concatCommand(concatFile, timeline), commands); + Path videoTimeline = timeline; + if (!plan.overlays().isEmpty()) { + videoTimeline = work.resolve("timeline-overlays.mp4"); + runAndRecord(overlayCommand(timeline, plan.overlays(), videoTimeline), commands); + } Path output = store.projectDirectory(projectId).resolve("final.mp4"); Path audioDirectory = store.projectDirectory(projectId).resolve("audio"); Path music = audioDirectory.resolve("music.wav"); Path voiceover = audioDirectory.resolve("voiceover.wav"); + List assets = resolvedAssets(plan, audioDirectory); List soundEffects = plan.audioCues().stream() .filter(cue -> "sfx".equals(cue.type())) .map(cue -> new SfxInput(audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav"), cue)) .toList(); if (Files.isRegularFile(music) || Files.isRegularFile(voiceover) || !soundEffects.isEmpty()) { - runAndRecord(audioMixCommand(timeline, Files.isRegularFile(music) ? music : null, + runAndRecord(audioMixCommand(videoTimeline, Files.isRegularFile(music) ? music : null, Files.isRegularFile(voiceover) ? voiceover : null, soundEffects, output), commands); } else { - copy(timeline, output); + copy(videoTimeline, output); } double duration = plan.decisions().get(plan.decisions().size() - 1).timelineEndSeconds(); RenderManifest manifest = new RenderManifest(projectId, plan.decisions().stream().map(EditDecision::clipId).toList(), output.toString(), duration, - List.copyOf(commands), Instant.now()); + List.copyOf(commands), assets, Instant.now()); store.writeJson(projectId, "render-manifest.json", manifest); projectService.updateProject(projectId, EditProjectStatus.RENDERED, Path.of(project.inputDirectory()), null); if (observability != null) { @@ -200,6 +213,48 @@ public class FfmpegEditRenderer implements EditRenderer { "-i", concatFile.toString(), "-c", "copy", output.toString()); } + List overlayCommand(Path input, List overlays, Path output) { + return List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", input.toString(), + "-vf", overlayFilter(overlays), + "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", + "-c:a", "copy", output.toString()); + } + + String overlayFilter(List overlays) { + return overlays.stream() + .map(this::drawTextFilter) + .collect(Collectors.joining(",")); + } + + private String drawTextFilter(TextOverlay overlay) { + return "drawtext=text='%s':x=%s:y=%s:fontsize=54:fontcolor=white:borderw=2:bordercolor=black@0.75:enable='between(t\\,%s\\,%s)'" + .formatted(escapeDrawText(overlay.text()), overlayX(overlay.placement()), overlayY(overlay.placement()), + overlay.timelineStartSeconds(), overlay.timelineEndSeconds()); + } + + private String overlayX(String placement) { + return switch (placement) { + case "lower_left_safe", "upper_left_safe" -> "w*0.06"; + case "upper_right_safe" -> "w-text_w-w*0.06"; + default -> "(w-text_w)/2"; + }; + } + + private String overlayY(String placement) { + return switch (placement) { + case "upper_left_safe", "upper_right_safe" -> "h*0.08"; + case "center_safe" -> "(h-text_h)/2"; + default -> "h-text_h-h*0.10"; + }; + } + + private String escapeDrawText(String text) { + return text.replace("\\", "\\\\") + .replace("'", "\\'") + .replace(":", "\\:") + .replace("%", "\\%"); + } + List audioMixCommand(Path timeline, Path music, Path voiceover, Path output) { return audioMixCommand(timeline, music, voiceover, List.of(), output); } @@ -243,6 +298,35 @@ public class FfmpegEditRenderer implements EditRenderer { return List.copyOf(command); } + private List resolvedAssets(EditPlan plan, Path audioDirectory) { + List assets = new ArrayList<>(); + Path music = audioDirectory.resolve("music.wav"); + if (Files.isRegularFile(music)) { + assets.add(new ResolvedEditAsset(EditAssetType.MUSIC, "music", music.toString(), + "project-local", "project-audio")); + } + Path voiceover = audioDirectory.resolve("voiceover.wav"); + if (Files.isRegularFile(voiceover)) { + assets.add(new ResolvedEditAsset(EditAssetType.VOICEOVER, "voiceover", voiceover.toString(), + "project-local", "project-audio")); + } + for (AudioCue cue : plan.audioCues()) { + if (assetProvider != null) { + assetProvider.resolve(new EditAssetRequest(EditAssetType.valueOf(cue.type().toUpperCase()), + cue.assetKey(), null, cue.timelineEndSeconds() - cue.timelineStartSeconds(), cue.notes())) + .ifPresent(assets::add); + } + if ("sfx".equals(cue.type())) { + Path sfx = audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav"); + if (Files.isRegularFile(sfx)) { + assets.add(new ResolvedEditAsset(EditAssetType.SFX, cue.assetKey(), sfx.toString(), + "project-local", "project-audio")); + } + } + } + return List.copyOf(assets); + } + private void writeConcatFile(Path file, List segments) { String content = segments.stream() .map(path -> "file '" + path.toAbsolutePath().toString().replace("'", "'\\''") + "'") diff --git a/src/main/java/org/example/videoclips/editing/RenderManifest.java b/src/main/java/org/example/videoclips/editing/RenderManifest.java index ac82a32..321ba09 100644 --- a/src/main/java/org/example/videoclips/editing/RenderManifest.java +++ b/src/main/java/org/example/videoclips/editing/RenderManifest.java @@ -9,6 +9,23 @@ public record RenderManifest( String outputPath, double durationSeconds, List> commands, + List assets, Instant completedAt ) { + public RenderManifest { + clipIds = clipIds == null ? List.of() : List.copyOf(clipIds); + commands = commands == null ? List.of() : List.copyOf(commands); + assets = assets == null ? List.of() : List.copyOf(assets); + } + + public RenderManifest( + String projectId, + List clipIds, + String outputPath, + double durationSeconds, + List> commands, + Instant completedAt + ) { + this(projectId, clipIds, outputPath, durationSeconds, commands, List.of(), completedAt); + } } diff --git a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java index aaa610b..3663328 100644 --- a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java +++ b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java @@ -4,6 +4,7 @@ import org.example.videoclips.config.VideoClippingProperties; import org.junit.jupiter.api.Test; import java.nio.file.Path; +import java.util.List; import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.Mockito.mock; @@ -102,6 +103,27 @@ class FfmpegEditRendererTest { + "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2[a]"); } + @Test + void buildsOverlayCommandWithSafeDrawTextFilters() { + VideoClippingProperties properties = new VideoClippingProperties(); + FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class), + mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null); + + var command = renderer.overlayCommand(Path.of("timeline.mp4"), List.of( + new TextOverlay("PRECISION: 911%", 1, 3, "lower_left_safe", "fade_slide_up", "title"), + new TextOverlay("FINAL HERO", 4, 6, "center_safe", "fade_in", "ending")), + Path.of("timeline-overlays.mp4")); + + assertThat(command).containsSubsequence("-i", "timeline.mp4", "-vf"); + assertThat(command).anySatisfy(argument -> assertThat(argument) + .contains("drawtext=text='PRECISION\\: 911\\%'") + .contains("x=w*0.06:y=h-text_h-h*0.10") + .contains("enable='between(t\\,1.0\\,3.0)'") + .contains("drawtext=text='FINAL HERO'") + .contains("x=(w-text_w)/2:y=(h-text_h)/2")); + assertThat(command).containsSubsequence("-c:v", "libx264", "-c:a", "copy"); + } + @Test void chainsAudioTempoAtQuarterSpeed() { VideoClippingProperties properties = new VideoClippingProperties(); diff --git a/src/test/java/org/example/videoclips/editing/RenderManifestTest.java b/src/test/java/org/example/videoclips/editing/RenderManifestTest.java index 78f067c..03bb80f 100644 --- a/src/test/java/org/example/videoclips/editing/RenderManifestTest.java +++ b/src/test/java/org/example/videoclips/editing/RenderManifestTest.java @@ -15,11 +15,14 @@ class RenderManifestTest { ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); RenderManifest manifest = new RenderManifest("project", List.of("clip-1", "clip-2"), "final.mp4", 8.0, List.of(List.of("ffmpeg", "-i", "clip.mp4", "final.mp4")), + List.of(new ResolvedEditAsset(EditAssetType.MUSIC, "music-bed", "music.wav", + "licensed", "local-filesystem", ContentCategory.CAR_VLOG, List.of("premium"))), Instant.parse("2026-07-10T10:00:00Z")); String json = mapper.writeValueAsString(manifest); - assertThat(json).contains("\"clipIds\"").contains("\"commands\"").contains("final.mp4"); + assertThat(json).contains("\"clipIds\"").contains("\"commands\"").contains("\"assets\"") + .contains("music-bed").contains("final.mp4"); assertThat(mapper.readValue(json, RenderManifest.class)).isEqualTo(manifest); } }