jsl
/
video_editing_poc
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add edit sound effect cues

This commit is contained in:
JSLMPR 2026-07-10 18:47:18 +02:00
parent e2037a7629
commit 1c3db181a3
5 changed files with 93 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
docs/cinematic-video-editing-service-plan.md
View File

@ -802,7 +802,7 @@ It writes the script only and requires an imported audio file.
19. [x] Add music and voiceover audio mixing.
20. [x] Add render manifest with command metadata and output details.
21. [x] Add basic transition support: cut, crossfade, fade in, fade out.
22. [ ] Add SFX cue support.
22. [x] Add SFX cue support.
23. [ ] Add optional generated voiceover adapter interface.
24. [ ] Add structured logs and metrics for analysis, planning, and rendering durations.
25. [ ] Add integration test with generated fixture clips and a short edit plan.

30
src/main/java/org/example/videoclips/editing/EditPlanValidator.java
View File

@ -6,6 +6,9 @@ import org.springframework.stereotype.Component;
import java.util.Map;
import java.util.Set;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.regex.Pattern;
import java.util.function.Function;
import java.util.stream.Collectors;
@ -16,6 +19,7 @@ public class EditPlanValidator {
private static final Set<String> TRANSITIONS = Set.of("cut", "crossfade", "fade-in", "fade-out", "none");
private static final double EPSILON = 0.001;
private static final double TARGET_TOLERANCE_SECONDS = 1.0;
private static final Pattern ASSET_KEY = Pattern.compile("[A-Za-z0-9][A-Za-z0-9._-]{0,127}");
private final EditProjectStore store;
@ -58,6 +62,7 @@ public class EditPlanValidator {
validateTransition(decision.transitionOut());
previousEnd = decision.timelineEndSeconds();
}
validateAudioCues(projectId, plan);
if (previousEnd > project.targetDurationSeconds() + TARGET_TOLERANCE_SECONDS
|| Math.abs(plan.targetDurationSeconds() - project.targetDurationSeconds()) > EPSILON) {
reject("Edit plan duration exceeds or does not match the project target duration");
@ -65,6 +70,31 @@ public class EditPlanValidator {
return plan;
}
private void validateAudioCues(String projectId, EditPlan plan) {
for (AudioCue cue : plan.audioCues()) {
if (cue == null || cue.type() == null || cue.timelineStartSeconds() < 0
|| cue.timelineEndSeconds() <= cue.timelineStartSeconds()
|| cue.timelineEndSeconds() > plan.targetDurationSeconds() + EPSILON) {
reject("Audio cue has an invalid type or timeline range");
}
if (!Set.of("music", "sfx").contains(cue.type())) {
reject("Unsupported audio cue type: " + cue.type());
}
if (!"sfx".equals(cue.type())) {
continue;
}
if (cue.assetKey() == null || !ASSET_KEY.matcher(cue.assetKey()).matches()
|| cue.assetKey().contains("..")) {
reject("SFX assetKey must be a safe plain file name");
}
Path asset = store.projectDirectory(projectId).resolve("audio/sfx")
.resolve(cue.assetKey() + ".wav");
if (!Files.isRegularFile(asset)) {
reject("SFX file does not exist: " + cue.assetKey());
}
}
}
private void validateTransition(String transition) {
if (transition == null || !TRANSITIONS.contains(transition)) {
reject("Unsupported transition: " + transition);

28
src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java
View File

@ -69,9 +69,13 @@ public class FfmpegEditRenderer implements EditRenderer {
Path audioDirectory = store.projectDirectory(projectId).resolve("audio");
Path music = audioDirectory.resolve("music.wav");
Path voiceover = audioDirectory.resolve("voiceover.wav");
if (Files.isRegularFile(music) || Files.isRegularFile(voiceover)) {
List<SfxInput> soundEffects = plan.audioCues().stream()
.filter(cue -> "sfx".equals(cue.type()))
.map(cue -> new SfxInput(audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav"), cue))
.toList();
if (Files.isRegularFile(music) || Files.isRegularFile(voiceover) || !soundEffects.isEmpty()) {
runAndRecord(audioMixCommand(timeline, Files.isRegularFile(music) ? music : null,
Files.isRegularFile(voiceover) ? voiceover : null, output), commands);
Files.isRegularFile(voiceover) ? voiceover : null, soundEffects, output), commands);
} else {
copy(timeline, output);
}
@ -118,6 +122,10 @@ public class FfmpegEditRenderer implements EditRenderer {
}
List<String> audioMixCommand(Path timeline, Path music, Path voiceover, Path output) {
return audioMixCommand(timeline, music, voiceover, List.of(), output);
}
List<String> audioMixCommand(Path timeline, Path music, Path voiceover, List<SfxInput> soundEffects, Path output) {
List<String> command = new ArrayList<>(List.of(properties.getFfmpegBinary(), "-hide_banner", "-y",
"-i", timeline.toString()));
List<String> labels = new ArrayList<>(List.of("[0:a]"));
@ -133,6 +141,20 @@ public class FfmpegEditRenderer implements EditRenderer {
command.addAll(List.of("-i", voiceover.toString()));
filters.append("[").append(input).append(":a]volume=1.0[voice];");
labels.add("[voice]");
input++;
}
for (int index = 0; index < soundEffects.size(); index++) {
SfxInput soundEffect = soundEffects.get(index);
AudioCue cue = soundEffect.cue();
String label = "sfx" + index;
long delayMillis = Math.round(cue.timelineStartSeconds() * 1000);
double duration = cue.timelineEndSeconds() - cue.timelineStartSeconds();
command.addAll(List.of("-i", soundEffect.path().toString()));
filters.append("[").append(input).append(":a]atrim=duration=").append(duration)
.append(",volume=").append(cue.gainDb()).append("dB,adelay=")
.append(delayMillis).append("|").append(delayMillis).append("[").append(label).append("];");
labels.add("[" + label + "]");
input++;
}
filters.append(String.join("", labels)).append("amix=inputs=").append(labels.size())
.append(":duration=first:dropout_transition=2[a]");
@ -196,6 +218,8 @@ public class FfmpegEditRenderer implements EditRenderer {
record ProcessResult(int exitCode, String output) { }
record SfxInput(Path path, AudioCue cue) { }
@FunctionalInterface
interface ProcessExecutor {
ProcessResult execute(List<String> command) throws IOException, InterruptedException;

20
src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java
View File

@ -52,6 +52,26 @@ class EditPlanValidatorTest {
rejects(plan(d));
}
@Test void rejectsMissingSfxFile() {
EditPlan original = plan(decision("clip-1", 0, 4, 0, 4, "cut"));
EditPlan withSfx = new EditPlan(original.projectId(), original.style(), original.targetDurationSeconds(),
original.decisions(), List.of(new AudioCue("sfx", "engine-rev", 1, 2, -3, "accent")),
original.voiceover(), original.renderProfile(), original.summary());
rejects(withSfx);
}
@Test void acceptsExistingSfxFile() throws Exception {
java.nio.file.Files.createDirectories(tempDir.resolve("project/audio/sfx"));
java.nio.file.Files.writeString(tempDir.resolve("project/audio/sfx/engine-rev.wav"), "wav");
EditPlan original = plan(decision("clip-1", 0, 4, 0, 4, "cut"));
EditPlan withSfx = new EditPlan(original.projectId(), original.style(), original.targetDurationSeconds(),
original.decisions(), List.of(new AudioCue("sfx", "engine-rev", 1, 2, -3, "accent")),
original.voiceover(), original.renderProfile(), original.summary());
assertThat(validator.validate("project", withSfx)).isEqualTo(withSfx);
}
private void rejects(EditPlan plan) {
assertThatThrownBy(() -> validator.validate("project", plan)).isInstanceOf(BadRequestException.class);
}

16
src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java
View File

@ -70,4 +70,20 @@ class FfmpegEditRendererTest {
+ "fade=t=in:st=0:d=0.5,fade=t=out:st=1.5:d=0.5");
assertThat(command).containsSubsequence("-af", "atempo=2.0");
}
@Test
void addsDelayedSfxInputToFinalMix() {
VideoClippingProperties properties = new VideoClippingProperties();
FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class),
mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null);
AudioCue cue = new AudioCue("sfx", "engine-rev", 1.5, 3.0, -3, "acceleration");
var command = renderer.audioMixCommand(Path.of("timeline.mp4"), null, null,
java.util.List.of(new FfmpegEditRenderer.SfxInput(Path.of("engine-rev.wav"), cue)),
Path.of("final.mp4"));
assertThat(command).containsSubsequence("-i", "timeline.mp4", "-i", "engine-rev.wav");
assertThat(command).contains("[1:a]atrim=duration=1.5,volume=-3.0dB,adelay=1500|1500[sfx0];"
+ "[0:a][sfx0]amix=inputs=2:duration=first:dropout_transition=2[a]");
}
}