diff --git a/docs/cinematic-video-editing-service-plan.md b/docs/cinematic-video-editing-service-plan.md index 9a0b9ff..fb34bb7 100644 --- a/docs/cinematic-video-editing-service-plan.md +++ b/docs/cinematic-video-editing-service-plan.md @@ -802,7 +802,7 @@ It writes the script only and requires an imported audio file. 19. [x] Add music and voiceover audio mixing. 20. [x] Add render manifest with command metadata and output details. 21. [x] Add basic transition support: cut, crossfade, fade in, fade out. -22. [ ] Add SFX cue support. +22. [x] Add SFX cue support. 23. [ ] Add optional generated voiceover adapter interface. 24. [ ] Add structured logs and metrics for analysis, planning, and rendering durations. 25. [ ] Add integration test with generated fixture clips and a short edit plan. diff --git a/src/main/java/org/example/videoclips/editing/EditPlanValidator.java b/src/main/java/org/example/videoclips/editing/EditPlanValidator.java index 591a0b8..8c07a04 100644 --- a/src/main/java/org/example/videoclips/editing/EditPlanValidator.java +++ b/src/main/java/org/example/videoclips/editing/EditPlanValidator.java @@ -6,6 +6,9 @@ import org.springframework.stereotype.Component; import java.util.Map; import java.util.Set; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.regex.Pattern; import java.util.function.Function; import java.util.stream.Collectors; @@ -16,6 +19,7 @@ public class EditPlanValidator { private static final Set TRANSITIONS = Set.of("cut", "crossfade", "fade-in", "fade-out", "none"); private static final double EPSILON = 0.001; private static final double TARGET_TOLERANCE_SECONDS = 1.0; + private static final Pattern ASSET_KEY = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,127}"); private final EditProjectStore store; @@ -58,6 +62,7 @@ public class EditPlanValidator { validateTransition(decision.transitionOut()); previousEnd = decision.timelineEndSeconds(); } + validateAudioCues(projectId, plan); if (previousEnd > project.targetDurationSeconds() + TARGET_TOLERANCE_SECONDS || Math.abs(plan.targetDurationSeconds() - project.targetDurationSeconds()) > EPSILON) { reject("Edit plan duration exceeds or does not match the project target duration"); @@ -65,6 +70,31 @@ public class EditPlanValidator { return plan; } + private void validateAudioCues(String projectId, EditPlan plan) { + for (AudioCue cue : plan.audioCues()) { + if (cue == null || cue.type() == null || cue.timelineStartSeconds() < 0 + || cue.timelineEndSeconds() <= cue.timelineStartSeconds() + || cue.timelineEndSeconds() > plan.targetDurationSeconds() + EPSILON) { + reject("Audio cue has an invalid type or timeline range"); + } + if (!Set.of("music", "sfx").contains(cue.type())) { + reject("Unsupported audio cue type: " + cue.type()); + } + if (!"sfx".equals(cue.type())) { + continue; + } + if (cue.assetKey() == null || !ASSET_KEY.matcher(cue.assetKey()).matches() + || cue.assetKey().contains("..")) { + reject("SFX assetKey must be a safe plain file name"); + } + Path asset = store.projectDirectory(projectId).resolve("audio/sfx") + .resolve(cue.assetKey() + ".wav"); + if (!Files.isRegularFile(asset)) { + reject("SFX file does not exist: " + cue.assetKey()); + } + } + } + private void validateTransition(String transition) { if (transition == null || !TRANSITIONS.contains(transition)) { reject("Unsupported transition: " + transition); diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index d404faa..b83ec4e 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -69,9 +69,13 @@ public class FfmpegEditRenderer implements EditRenderer { Path audioDirectory = store.projectDirectory(projectId).resolve("audio"); Path music = audioDirectory.resolve("music.wav"); Path voiceover = audioDirectory.resolve("voiceover.wav"); - if (Files.isRegularFile(music) || Files.isRegularFile(voiceover)) { + List soundEffects = plan.audioCues().stream() + .filter(cue -> "sfx".equals(cue.type())) + .map(cue -> new SfxInput(audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav"), cue)) + .toList(); + if (Files.isRegularFile(music) || Files.isRegularFile(voiceover) || !soundEffects.isEmpty()) { runAndRecord(audioMixCommand(timeline, Files.isRegularFile(music) ? music : null, - Files.isRegularFile(voiceover) ? voiceover : null, output), commands); + Files.isRegularFile(voiceover) ? voiceover : null, soundEffects, output), commands); } else { copy(timeline, output); } @@ -118,6 +122,10 @@ public class FfmpegEditRenderer implements EditRenderer { } List audioMixCommand(Path timeline, Path music, Path voiceover, Path output) { + return audioMixCommand(timeline, music, voiceover, List.of(), output); + } + + List audioMixCommand(Path timeline, Path music, Path voiceover, List soundEffects, Path output) { List command = new ArrayList<>(List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", timeline.toString())); List labels = new ArrayList<>(List.of("[0:a]")); @@ -133,6 +141,20 @@ public class FfmpegEditRenderer implements EditRenderer { command.addAll(List.of("-i", voiceover.toString())); filters.append("[").append(input).append(":a]volume=1.0[voice];"); labels.add("[voice]"); + input++; + } + for (int index = 0; index < soundEffects.size(); index++) { + SfxInput soundEffect = soundEffects.get(index); + AudioCue cue = soundEffect.cue(); + String label = "sfx" + index; + long delayMillis = Math.round(cue.timelineStartSeconds() * 1000); + double duration = cue.timelineEndSeconds() - cue.timelineStartSeconds(); + command.addAll(List.of("-i", soundEffect.path().toString())); + filters.append("[").append(input).append(":a]atrim=duration=").append(duration) + .append(",volume=").append(cue.gainDb()).append("dB,adelay=") + .append(delayMillis).append("|").append(delayMillis).append("[").append(label).append("];"); + labels.add("[" + label + "]"); + input++; } filters.append(String.join("", labels)).append("amix=inputs=").append(labels.size()) .append(":duration=first:dropout_transition=2[a]"); @@ -196,6 +218,8 @@ public class FfmpegEditRenderer implements EditRenderer { record ProcessResult(int exitCode, String output) { } + record SfxInput(Path path, AudioCue cue) { } + @FunctionalInterface interface ProcessExecutor { ProcessResult execute(List command) throws IOException, InterruptedException; diff --git a/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java b/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java index aaad6b6..e39704d 100644 --- a/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java +++ b/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java @@ -52,6 +52,26 @@ class EditPlanValidatorTest { rejects(plan(d)); } + @Test void rejectsMissingSfxFile() { + EditPlan original = plan(decision("clip-1", 0, 4, 0, 4, "cut")); + EditPlan withSfx = new EditPlan(original.projectId(), original.style(), original.targetDurationSeconds(), + original.decisions(), List.of(new AudioCue("sfx", "engine-rev", 1, 2, -3, "accent")), + original.voiceover(), original.renderProfile(), original.summary()); + + rejects(withSfx); + } + + @Test void acceptsExistingSfxFile() throws Exception { + java.nio.file.Files.createDirectories(tempDir.resolve("project/audio/sfx")); + java.nio.file.Files.writeString(tempDir.resolve("project/audio/sfx/engine-rev.wav"), "wav"); + EditPlan original = plan(decision("clip-1", 0, 4, 0, 4, "cut")); + EditPlan withSfx = new EditPlan(original.projectId(), original.style(), original.targetDurationSeconds(), + original.decisions(), List.of(new AudioCue("sfx", "engine-rev", 1, 2, -3, "accent")), + original.voiceover(), original.renderProfile(), original.summary()); + + assertThat(validator.validate("project", withSfx)).isEqualTo(withSfx); + } + private void rejects(EditPlan plan) { assertThatThrownBy(() -> validator.validate("project", plan)).isInstanceOf(BadRequestException.class); } diff --git a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java index 48e9b16..5b1628a 100644 --- a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java +++ b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java @@ -70,4 +70,20 @@ class FfmpegEditRendererTest { + "fade=t=in:st=0:d=0.5,fade=t=out:st=1.5:d=0.5"); assertThat(command).containsSubsequence("-af", "atempo=2.0"); } + + @Test + void addsDelayedSfxInputToFinalMix() { + VideoClippingProperties properties = new VideoClippingProperties(); + FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class), + mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null); + AudioCue cue = new AudioCue("sfx", "engine-rev", 1.5, 3.0, -3, "acceleration"); + + var command = renderer.audioMixCommand(Path.of("timeline.mp4"), null, null, + java.util.List.of(new FfmpegEditRenderer.SfxInput(Path.of("engine-rev.wav"), cue)), + Path.of("final.mp4")); + + assertThat(command).containsSubsequence("-i", "timeline.mp4", "-i", "engine-rev.wav"); + assertThat(command).contains("[1:a]atrim=duration=1.5,volume=-3.0dB,adelay=1500|1500[sfx0];" + + "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2[a]"); + } }