jsl
/
video_editing_poc
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Render cinematic text overlays

This commit is contained in:
JSLMPR 2026-07-11 01:06:38 +02:00
parent 376bc41685
commit 43fccb7dc6
5 changed files with 138 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/production-cinematic-highlight-editing-plan.md
View File

@ -308,6 +308,7 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can
- [x] Milestone 11: Implement asset provider interfaces for voiceover, music, SFX, fonts, and LUTs.
- [x] Milestone 12: Add a local licensed asset library with category tags and deterministic asset selection.
- [ ] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests.
- [ ] Milestone 13 progress: Renderer now applies timed text overlays and records resolved/planned edit assets in the render manifest. Dynamic crop keyframes, audio ducking, and loudness normalization remain.
- [ ] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters.
- [ ] Milestone 15: Add an operator runbook for placing a video, running the service locally, choosing a model, reviewing the director plan, and finding final clips.
- [ ] Milestone 16: Add integration tests with small fixture videos for family, food, car, and generic content.

104
src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java
View File

@ -3,6 +3,7 @@ package org.example.videoclips.editing;
import org.example.videoclips.api.dto.EditProjectResponse;
import org.example.videoclips.config.VideoClippingProperties;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.ObjectProvider;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
@ -28,41 +29,46 @@ public class FfmpegEditRenderer implements EditRenderer {
private final EditPlanValidator validator;
private final ProcessExecutor executor;
private final VoiceoverGenerator voiceoverGenerator;
private final EditAssetProvider assetProvider;
private final EditObservability observability;
public FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store,
EditProjectService projectService, EditPlanValidator validator) {
this(properties, store, projectService, validator, null, null, FfmpegEditRenderer::execute);
this(properties, store, projectService, validator, null, null, null, FfmpegEditRenderer::execute);
}
public FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store,
EditProjectService projectService, EditPlanValidator validator,
VoiceoverGenerator voiceoverGenerator) {
this(properties, store, projectService, validator, voiceoverGenerator, null, FfmpegEditRenderer::execute);
this(properties, store, projectService, validator, voiceoverGenerator, null, null,
FfmpegEditRenderer::execute);
}
@Autowired
public FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store,
EditProjectService projectService, EditPlanValidator validator,
VoiceoverGenerator voiceoverGenerator, EditObservability observability) {
this(properties, store, projectService, validator, voiceoverGenerator, observability,
VoiceoverGenerator voiceoverGenerator, ObjectProvider<EditAssetProvider> assetProvider,
EditObservability observability) {
this(properties, store, projectService, validator, voiceoverGenerator, assetProvider.getIfAvailable(),
observability,
FfmpegEditRenderer::execute);
}
FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store,
EditProjectService projectService, EditPlanValidator validator, ProcessExecutor executor) {
this(properties, store, projectService, validator, null, null, executor);
this(properties, store, projectService, validator, null, null, null, executor);
}
FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store,
EditProjectService projectService, EditPlanValidator validator,
VoiceoverGenerator voiceoverGenerator, ProcessExecutor executor) {
this(properties, store, projectService, validator, voiceoverGenerator, null, executor);
this(properties, store, projectService, validator, voiceoverGenerator, null, null, executor);
}
FfmpegEditRenderer(VideoClippingProperties properties, EditProjectStore store,
EditProjectService projectService, EditPlanValidator validator,
VoiceoverGenerator voiceoverGenerator, EditObservability observability,
VoiceoverGenerator voiceoverGenerator, EditAssetProvider assetProvider,
EditObservability observability,
ProcessExecutor executor) {
this.properties = properties.getEditing();
this.store = store;
@ -70,6 +76,7 @@ public class FfmpegEditRenderer implements EditRenderer {
this.validator = validator;
this.executor = executor;
this.voiceoverGenerator = voiceoverGenerator;
this.assetProvider = assetProvider;
this.observability = observability;
}
@ -124,24 +131,30 @@ public class FfmpegEditRenderer implements EditRenderer {
writeConcatFile(concatFile, segments);
Path timeline = work.resolve("timeline-video.mp4");
runAndRecord(concatCommand(concatFile, timeline), commands);
Path videoTimeline = timeline;
if (!plan.overlays().isEmpty()) {
videoTimeline = work.resolve("timeline-overlays.mp4");
runAndRecord(overlayCommand(timeline, plan.overlays(), videoTimeline), commands);
}
Path output = store.projectDirectory(projectId).resolve("final.mp4");
Path audioDirectory = store.projectDirectory(projectId).resolve("audio");
Path music = audioDirectory.resolve("music.wav");
Path voiceover = audioDirectory.resolve("voiceover.wav");
List<ResolvedEditAsset> assets = resolvedAssets(plan, audioDirectory);
List<SfxInput> soundEffects = plan.audioCues().stream()
.filter(cue -> "sfx".equals(cue.type()))
.map(cue -> new SfxInput(audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav"), cue))
.toList();
if (Files.isRegularFile(music) || Files.isRegularFile(voiceover) || !soundEffects.isEmpty()) {
runAndRecord(audioMixCommand(timeline, Files.isRegularFile(music) ? music : null,
runAndRecord(audioMixCommand(videoTimeline, Files.isRegularFile(music) ? music : null,
Files.isRegularFile(voiceover) ? voiceover : null, soundEffects, output), commands);
} else {
copy(timeline, output);
copy(videoTimeline, output);
}
double duration = plan.decisions().get(plan.decisions().size() - 1).timelineEndSeconds();
RenderManifest manifest = new RenderManifest(projectId,
plan.decisions().stream().map(EditDecision::clipId).toList(), output.toString(), duration,
List.copyOf(commands), Instant.now());
List.copyOf(commands), assets, Instant.now());
store.writeJson(projectId, "render-manifest.json", manifest);
projectService.updateProject(projectId, EditProjectStatus.RENDERED, Path.of(project.inputDirectory()), null);
if (observability != null) {
@ -200,6 +213,48 @@ public class FfmpegEditRenderer implements EditRenderer {
"-i", concatFile.toString(), "-c", "copy", output.toString());
}
List<String> overlayCommand(Path input, List<TextOverlay> overlays, Path output) {
return List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", input.toString(),
"-vf", overlayFilter(overlays),
"-c:v", "libx264", "-preset", "veryfast", "-crf", "18",
"-c:a", "copy", output.toString());
}
String overlayFilter(List<TextOverlay> overlays) {
return overlays.stream()
.map(this::drawTextFilter)
.collect(Collectors.joining(","));
}
private String drawTextFilter(TextOverlay overlay) {
return "drawtext=text='%s':x=%s:y=%s:fontsize=54:fontcolor=white:borderw=2:bordercolor=black@0.75:enable='between(t\\,%s\\,%s)'"
.formatted(escapeDrawText(overlay.text()), overlayX(overlay.placement()), overlayY(overlay.placement()),
overlay.timelineStartSeconds(), overlay.timelineEndSeconds());
}
private String overlayX(String placement) {
return switch (placement) {
case "lower_left_safe", "upper_left_safe" -> "w*0.06";
case "upper_right_safe" -> "w-text_w-w*0.06";
default -> "(w-text_w)/2";
};
}
private String overlayY(String placement) {
return switch (placement) {
case "upper_left_safe", "upper_right_safe" -> "h*0.08";
case "center_safe" -> "(h-text_h)/2";
default -> "h-text_h-h*0.10";
};
}
private String escapeDrawText(String text) {
return text.replace("\\", "\\\\")
.replace("'", "\\'")
.replace(":", "\\:")
.replace("%", "\\%");
}
List<String> audioMixCommand(Path timeline, Path music, Path voiceover, Path output) {
return audioMixCommand(timeline, music, voiceover, List.of(), output);
}
@ -243,6 +298,35 @@ public class FfmpegEditRenderer implements EditRenderer {
return List.copyOf(command);
}
private List<ResolvedEditAsset> resolvedAssets(EditPlan plan, Path audioDirectory) {
List<ResolvedEditAsset> assets = new ArrayList<>();
Path music = audioDirectory.resolve("music.wav");
if (Files.isRegularFile(music)) {
assets.add(new ResolvedEditAsset(EditAssetType.MUSIC, "music", music.toString(),
"project-local", "project-audio"));
}
Path voiceover = audioDirectory.resolve("voiceover.wav");
if (Files.isRegularFile(voiceover)) {
assets.add(new ResolvedEditAsset(EditAssetType.VOICEOVER, "voiceover", voiceover.toString(),
"project-local", "project-audio"));
}
for (AudioCue cue : plan.audioCues()) {
if (assetProvider != null) {
assetProvider.resolve(new EditAssetRequest(EditAssetType.valueOf(cue.type().toUpperCase()),
cue.assetKey(), null, cue.timelineEndSeconds() - cue.timelineStartSeconds(), cue.notes()))
.ifPresent(assets::add);
}
if ("sfx".equals(cue.type())) {
Path sfx = audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav");
if (Files.isRegularFile(sfx)) {
assets.add(new ResolvedEditAsset(EditAssetType.SFX, cue.assetKey(), sfx.toString(),
"project-local", "project-audio"));
}
}
}
return List.copyOf(assets);
}
private void writeConcatFile(Path file, List<Path> segments) {
String content = segments.stream()
.map(path -> "file '" + path.toAbsolutePath().toString().replace("'", "'\\''") + "'")

17
src/main/java/org/example/videoclips/editing/RenderManifest.java
View File

@ -9,6 +9,23 @@ public record RenderManifest(
String outputPath,
double durationSeconds,
List<List<String>> commands,
List<ResolvedEditAsset> assets,
Instant completedAt
) {
public RenderManifest {
clipIds = clipIds == null ? List.of() : List.copyOf(clipIds);
commands = commands == null ? List.of() : List.copyOf(commands);
assets = assets == null ? List.of() : List.copyOf(assets);
}
public RenderManifest(
String projectId,
List<String> clipIds,
String outputPath,
double durationSeconds,
List<List<String>> commands,
Instant completedAt
) {
this(projectId, clipIds, outputPath, durationSeconds, commands, List.of(), completedAt);
}
}

22
src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java
View File

@ -4,6 +4,7 @@ import org.example.videoclips.config.VideoClippingProperties;
import org.junit.jupiter.api.Test;
import java.nio.file.Path;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
@ -102,6 +103,27 @@ class FfmpegEditRendererTest {
+ "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2[a]");
}
@Test
void buildsOverlayCommandWithSafeDrawTextFilters() {
VideoClippingProperties properties = new VideoClippingProperties();
FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class),
mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null);
var command = renderer.overlayCommand(Path.of("timeline.mp4"), List.of(
new TextOverlay("PRECISION: 911%", 1, 3, "lower_left_safe", "fade_slide_up", "title"),
new TextOverlay("FINAL HERO", 4, 6, "center_safe", "fade_in", "ending")),
Path.of("timeline-overlays.mp4"));
assertThat(command).containsSubsequence("-i", "timeline.mp4", "-vf");
assertThat(command).anySatisfy(argument -> assertThat(argument)
.contains("drawtext=text='PRECISION\\: 911\\%'")
.contains("x=w*0.06:y=h-text_h-h*0.10")
.contains("enable='between(t\\,1.0\\,3.0)'")
.contains("drawtext=text='FINAL HERO'")
.contains("x=(w-text_w)/2:y=(h-text_h)/2"));
assertThat(command).containsSubsequence("-c:v", "libx264", "-c:a", "copy");
}
@Test
void chainsAudioTempoAtQuarterSpeed() {
VideoClippingProperties properties = new VideoClippingProperties();

5
src/test/java/org/example/videoclips/editing/RenderManifestTest.java
View File

@ -15,11 +15,14 @@ class RenderManifestTest {
ObjectMapper mapper = new ObjectMapper().findAndRegisterModules();
RenderManifest manifest = new RenderManifest("project", List.of("clip-1", "clip-2"), "final.mp4", 8.0,
List.of(List.of("ffmpeg", "-i", "clip.mp4", "final.mp4")),
List.of(new ResolvedEditAsset(EditAssetType.MUSIC, "music-bed", "music.wav",
"licensed", "local-filesystem", ContentCategory.CAR_VLOG, List.of("premium"))),
Instant.parse("2026-07-10T10:00:00Z"));
String json = mapper.writeValueAsString(manifest);
assertThat(json).contains("\"clipIds\"").contains("\"commands\"").contains("final.mp4");
assertThat(json).contains("\"clipIds\"").contains("\"commands\"").contains("\"assets\"")
.contains("music-bed").contains("final.mp4");
assertThat(mapper.readValue(json, RenderManifest.class)).isEqualTo(manifest);
}
}