diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index 3fa98e6..caf3736 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -305,7 +305,7 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 8: Implement category-specific highlight candidate scoring. - [x] Milestone 9: Implement strict AI director prompts and JSON schemas for category-aware highlight edit plans. - [x] Milestone 10: Implement edit plan validation so impossible timestamps, missing assets, invalid overlays, and unsupported effects fail before render. -- [ ] Milestone 11: Implement asset provider interfaces for voiceover, music, SFX, fonts, and LUTs. +- [x] Milestone 11: Implement asset provider interfaces for voiceover, music, SFX, fonts, and LUTs. - [ ] Milestone 12: Add a local licensed asset library with category tags and deterministic asset selection. - [ ] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests. - [ ] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters. diff --git a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java index a20a2ce..c7eee84 100644 --- a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java +++ b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java @@ -520,6 +520,10 @@ public class VideoClippingProperties { private String audioBitrate = "192k"; + private String voiceoverProvider = "noop"; + + private final Assets assets = new Assets(); + private final LocalDirector localDirector = new LocalDirector(); public boolean isEnabled() { @@ -642,10 +646,74 @@ public class VideoClippingProperties { this.audioBitrate = audioBitrate; } + public String getVoiceoverProvider() { + return voiceoverProvider; + } + + public void setVoiceoverProvider(String voiceoverProvider) { + this.voiceoverProvider = voiceoverProvider; + } + + public Assets getAssets() { + return assets; + } + public LocalDirector getLocalDirector() { return localDirector; } + public static class Assets { + private String musicFolder = "./input/highlights/assets/music"; + + private String sfxFolder = "./input/highlights/assets/sfx"; + + private String fontsFolder = "./input/highlights/assets/fonts"; + + private String lutsFolder = "./input/highlights/assets/luts"; + + private String voiceoverFolder = "./output/highlight-projects/_voiceover-cache"; + + public String getMusicFolder() { + return musicFolder; + } + + public void setMusicFolder(String musicFolder) { + this.musicFolder = musicFolder; + } + + public String getSfxFolder() { + return sfxFolder; + } + + public void setSfxFolder(String sfxFolder) { + this.sfxFolder = sfxFolder; + } + + public String getFontsFolder() { + return fontsFolder; + } + + public void setFontsFolder(String fontsFolder) { + this.fontsFolder = fontsFolder; + } + + public String getLutsFolder() { + return lutsFolder; + } + + public void setLutsFolder(String lutsFolder) { + this.lutsFolder = lutsFolder; + } + + public String getVoiceoverFolder() { + return voiceoverFolder; + } + + public void setVoiceoverFolder(String voiceoverFolder) { + this.voiceoverFolder = voiceoverFolder; + } + } + public static class LocalDirector { private boolean enabled = true; diff --git a/src/main/java/org/example/videoclips/editing/EditAssetProvider.java b/src/main/java/org/example/videoclips/editing/EditAssetProvider.java new file mode 100644 index 0000000..02069fa --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/EditAssetProvider.java @@ -0,0 +1,11 @@ +package org.example.videoclips.editing; + +import java.util.List; +import java.util.Optional; + +public interface EditAssetProvider { + + Optional resolve(EditAssetRequest request); + + List list(EditAssetType type, ContentCategory category); +} diff --git a/src/main/java/org/example/videoclips/editing/EditAssetRequest.java b/src/main/java/org/example/videoclips/editing/EditAssetRequest.java new file mode 100644 index 0000000..10928e8 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/EditAssetRequest.java @@ -0,0 +1,10 @@ +package org.example.videoclips.editing; + +public record EditAssetRequest( + EditAssetType type, + String assetKey, + ContentCategory category, + double durationSeconds, + String mood +) { +} diff --git a/src/main/java/org/example/videoclips/editing/EditAssetType.java b/src/main/java/org/example/videoclips/editing/EditAssetType.java new file mode 100644 index 0000000..28e8497 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/EditAssetType.java @@ -0,0 +1,9 @@ +package org.example.videoclips.editing; + +public enum EditAssetType { + VOICEOVER, + MUSIC, + SFX, + FONT, + LUT +} diff --git a/src/main/java/org/example/videoclips/editing/LocalEditAssetProvider.java b/src/main/java/org/example/videoclips/editing/LocalEditAssetProvider.java new file mode 100644 index 0000000..6a7fe91 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/LocalEditAssetProvider.java @@ -0,0 +1,147 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Stream; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class LocalEditAssetProvider implements EditAssetProvider { + + private static final Set AUDIO_EXTENSIONS = Set.of("wav", "mp3", "aac", "m4a", "flac"); + private static final Set FONT_EXTENSIONS = Set.of("ttf", "otf"); + private static final Set LUT_EXTENSIONS = Set.of("cube", "3dl"); + + private final VideoClippingProperties.Editing.Assets assets; + + public LocalEditAssetProvider(VideoClippingProperties properties) { + this.assets = properties.getEditing().getAssets(); + } + + @Override + public Optional resolve(EditAssetRequest request) { + if (request == null || request.type() == null || !isSafeAssetKey(request.assetKey())) { + return Optional.empty(); + } + return candidateRoots(request.type(), request.category()).stream() + .flatMap(root -> resolveInRoot(root, request).stream()) + .findFirst(); + } + + @Override + public List list(EditAssetType type, ContentCategory category) { + if (type == null) { + return List.of(); + } + return candidateRoots(type, category).stream() + .filter(Files::isDirectory) + .flatMap(root -> listRoot(type, root).stream()) + .sorted(Comparator.comparing(ResolvedEditAsset::assetKey)) + .toList(); + } + + private Optional resolveInRoot(Path root, EditAssetRequest request) { + if (!Files.isDirectory(root)) { + return Optional.empty(); + } + List extensions = extensions(request.type()); + Path direct = root.resolve(request.assetKey()).normalize(); + if (direct.startsWith(root) && Files.isRegularFile(direct) && hasExtension(direct, extensions)) { + return Optional.of(asset(request.type(), direct)); + } + for (String extension : extensions) { + Path candidate = root.resolve(request.assetKey() + "." + extension).normalize(); + if (candidate.startsWith(root) && Files.isRegularFile(candidate)) { + return Optional.of(asset(request.type(), candidate)); + } + } + return Optional.empty(); + } + + private List listRoot(EditAssetType type, Path root) { + try (Stream files = Files.list(root)) { + return files.filter(Files::isRegularFile) + .filter(path -> hasExtension(path, extensions(type))) + .map(path -> asset(type, path)) + .toList(); + } catch (IOException ex) { + throw new IllegalStateException("Unable to list edit assets in: " + root, ex); + } + } + + private ResolvedEditAsset asset(EditAssetType type, Path path) { + return new ResolvedEditAsset(type, stripExtension(path.getFileName().toString()), path.toString(), + license(path), "local-filesystem"); + } + + private String license(Path path) { + Path sidecar = path.resolveSibling(path.getFileName() + ".license.txt"); + if (!Files.isRegularFile(sidecar)) { + return "untracked-local-asset"; + } + try { + String license = Files.readString(sidecar).strip(); + return license.isBlank() ? "untracked-local-asset" : license; + } catch (IOException ex) { + throw new IllegalStateException("Unable to read edit asset license sidecar: " + sidecar, ex); + } + } + + private List candidateRoots(EditAssetType type, ContentCategory category) { + Path base = root(type); + if (category == null) { + return List.of(base); + } + return List.of(base.resolve(category.name().toLowerCase(Locale.ROOT)), base); + } + + private Path root(EditAssetType type) { + return switch (type) { + case VOICEOVER -> Path.of(assets.getVoiceoverFolder()).normalize(); + case MUSIC -> Path.of(assets.getMusicFolder()).normalize(); + case SFX -> Path.of(assets.getSfxFolder()).normalize(); + case FONT -> Path.of(assets.getFontsFolder()).normalize(); + case LUT -> Path.of(assets.getLutsFolder()).normalize(); + }; + } + + private List extensions(EditAssetType type) { + return switch (type) { + case VOICEOVER, MUSIC, SFX -> AUDIO_EXTENSIONS.stream().sorted().toList(); + case FONT -> FONT_EXTENSIONS.stream().sorted().toList(); + case LUT -> LUT_EXTENSIONS.stream().sorted().toList(); + }; + } + + private boolean hasExtension(Path path, List extensions) { + String fileName = path.getFileName().toString(); + int separator = fileName.lastIndexOf('.'); + if (separator < 0) { + return false; + } + return extensions.contains(fileName.substring(separator + 1).toLowerCase(Locale.ROOT)); + } + + private String stripExtension(String fileName) { + int separator = fileName.lastIndexOf('.'); + return separator > 0 ? fileName.substring(0, separator) : fileName; + } + + private boolean isSafeAssetKey(String assetKey) { + return assetKey != null + && !assetKey.isBlank() + && !assetKey.contains("..") + && !assetKey.contains("/") + && !assetKey.contains("\\"); + } +} diff --git a/src/main/java/org/example/videoclips/editing/ResolvedEditAsset.java b/src/main/java/org/example/videoclips/editing/ResolvedEditAsset.java new file mode 100644 index 0000000..7ea0a94 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/ResolvedEditAsset.java @@ -0,0 +1,10 @@ +package org.example.videoclips.editing; + +public record ResolvedEditAsset( + EditAssetType type, + String assetKey, + String path, + String license, + String provider +) { +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3702db0..dbc958c 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -30,6 +30,12 @@ video-clipping: video-bitrate: ${VIDEO_EDITING_VIDEO_BITRATE:12000k} audio-bitrate: ${VIDEO_EDITING_AUDIO_BITRATE:192k} voiceover-provider: ${VIDEO_EDITING_VOICEOVER_PROVIDER:noop} + assets: + music-folder: ${VIDEO_EDITING_ASSETS_MUSIC_FOLDER:./input/highlights/assets/music} + sfx-folder: ${VIDEO_EDITING_ASSETS_SFX_FOLDER:./input/highlights/assets/sfx} + fonts-folder: ${VIDEO_EDITING_ASSETS_FONTS_FOLDER:./input/highlights/assets/fonts} + luts-folder: ${VIDEO_EDITING_ASSETS_LUTS_FOLDER:./input/highlights/assets/luts} + voiceover-folder: ${VIDEO_EDITING_ASSETS_VOICEOVER_FOLDER:./output/highlight-projects/_voiceover-cache} local-director: enabled: ${VIDEO_EDITING_LOCAL_DIRECTOR_ENABLED:true} source-directory: ${VIDEO_EDITING_LOCAL_DIRECTOR_SOURCE_DIRECTORY:./input/editing/source} diff --git a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java index fc3f378..2ecce85 100644 --- a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java +++ b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java @@ -33,6 +33,15 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().getAudioSampleRate()).isEqualTo(48000); assertThat(properties.getEditing().getVideoBitrate()).isEqualTo("12000k"); assertThat(properties.getEditing().getAudioBitrate()).isEqualTo("192k"); + assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("noop"); + assertThat(properties.getEditing().getAssets().getMusicFolder()) + .isEqualTo("./input/highlights/assets/music"); + assertThat(properties.getEditing().getAssets().getSfxFolder()).isEqualTo("./input/highlights/assets/sfx"); + assertThat(properties.getEditing().getAssets().getFontsFolder()) + .isEqualTo("./input/highlights/assets/fonts"); + assertThat(properties.getEditing().getAssets().getLutsFolder()).isEqualTo("./input/highlights/assets/luts"); + assertThat(properties.getEditing().getAssets().getVoiceoverFolder()) + .isEqualTo("./output/highlight-projects/_voiceover-cache"); assertThat(properties.getEditing().getLocalDirector().isEnabled()).isTrue(); assertThat(properties.getEditing().getLocalDirector().getSourceDirectory()) .isEqualTo("./input/editing/source"); @@ -47,6 +56,12 @@ class VideoClippingPropertiesTest { "video-clipping.editing.enabled=false", "video-clipping.editing.project-directory=/tmp/edit-projects", "video-clipping.editing.thumbnail-count-per-clip=7", + "video-clipping.editing.voiceover-provider=remote", + "video-clipping.editing.assets.music-folder=/tmp/music", + "video-clipping.editing.assets.sfx-folder=/tmp/sfx", + "video-clipping.editing.assets.fonts-folder=/tmp/fonts", + "video-clipping.editing.assets.luts-folder=/tmp/luts", + "video-clipping.editing.assets.voiceover-folder=/tmp/voiceover", "video-clipping.editing.local-director.enabled=false", "video-clipping.editing.local-director.source-directory=/tmp/source", "video-clipping.editing.local-director.poll-interval-ms=9000", @@ -58,6 +73,12 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().isEnabled()).isFalse(); assertThat(properties.getEditing().getProjectDirectory()).isEqualTo("/tmp/edit-projects"); assertThat(properties.getEditing().getThumbnailCountPerClip()).isEqualTo(7); + assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("remote"); + assertThat(properties.getEditing().getAssets().getMusicFolder()).isEqualTo("/tmp/music"); + assertThat(properties.getEditing().getAssets().getSfxFolder()).isEqualTo("/tmp/sfx"); + assertThat(properties.getEditing().getAssets().getFontsFolder()).isEqualTo("/tmp/fonts"); + assertThat(properties.getEditing().getAssets().getLutsFolder()).isEqualTo("/tmp/luts"); + assertThat(properties.getEditing().getAssets().getVoiceoverFolder()).isEqualTo("/tmp/voiceover"); assertThat(properties.getEditing().getLocalDirector().isEnabled()).isFalse(); assertThat(properties.getEditing().getLocalDirector().getSourceDirectory()).isEqualTo("/tmp/source"); assertThat(properties.getEditing().getLocalDirector().getPollIntervalMs()).isEqualTo(9000); diff --git a/src/test/java/org/example/videoclips/editing/LocalEditAssetProviderTest.java b/src/test/java/org/example/videoclips/editing/LocalEditAssetProviderTest.java new file mode 100644 index 0000000..d2df522 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/LocalEditAssetProviderTest.java @@ -0,0 +1,94 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; + +class LocalEditAssetProviderTest { + + @TempDir + Path tempDir; + + @Test + void resolvesCategorySpecificAssetBeforeBaseAsset() throws Exception { + VideoClippingProperties properties = properties(); + Path base = Files.createDirectories(tempDir.resolve("music")); + Path car = Files.createDirectories(base.resolve("car_vlog")); + Files.writeString(base.resolve("drive.wav"), "base"); + Files.writeString(car.resolve("drive.wav"), "car"); + Files.writeString(car.resolve("drive.wav.license.txt"), "licensed-local-car-track"); + + LocalEditAssetProvider provider = new LocalEditAssetProvider(properties); + + var resolved = provider.resolve(new EditAssetRequest( + EditAssetType.MUSIC, "drive", ContentCategory.CAR_VLOG, 30, "premium drive")); + + assertThat(resolved).hasValueSatisfying(asset -> { + assertThat(asset.type()).isEqualTo(EditAssetType.MUSIC); + assertThat(asset.assetKey()).isEqualTo("drive"); + assertThat(asset.path()).endsWith("music/car_vlog/drive.wav"); + assertThat(asset.license()).isEqualTo("licensed-local-car-track"); + assertThat(asset.provider()).isEqualTo("local-filesystem"); + }); + } + + @Test + void resolvesAssetsWithExplicitExtensionAndListsSupportedFiles() throws Exception { + VideoClippingProperties properties = properties(); + Path sfx = Files.createDirectories(tempDir.resolve("sfx")); + Files.writeString(sfx.resolve("hit.wav"), "wav"); + Files.writeString(sfx.resolve("noise.txt"), "txt"); + + LocalEditAssetProvider provider = new LocalEditAssetProvider(properties); + + assertThat(provider.resolve(new EditAssetRequest( + EditAssetType.SFX, "hit.wav", null, 1, "impact"))).isPresent(); + assertThat(provider.list(EditAssetType.SFX, null)) + .extracting(ResolvedEditAsset::assetKey) + .containsExactly("hit"); + } + + @Test + void rejectsUnsafeAssetKeysAndMissingAssets() throws Exception { + VideoClippingProperties properties = properties(); + Files.createDirectories(tempDir.resolve("luts")); + LocalEditAssetProvider provider = new LocalEditAssetProvider(properties); + + assertThat(provider.resolve(new EditAssetRequest( + EditAssetType.LUT, "../secret", null, 0, null))).isEmpty(); + assertThat(provider.resolve(new EditAssetRequest( + EditAssetType.LUT, "missing", null, 0, null))).isEmpty(); + } + + @Test + void resolvesFontsLutsAndVoiceoverAssets() throws Exception { + VideoClippingProperties properties = properties(); + Files.createDirectories(tempDir.resolve("fonts")); + Files.createDirectories(tempDir.resolve("luts")); + Files.createDirectories(tempDir.resolve("voiceover")); + Files.writeString(tempDir.resolve("fonts/title.otf"), "font"); + Files.writeString(tempDir.resolve("luts/warm.cube"), "lut"); + Files.writeString(tempDir.resolve("voiceover/line-001.mp3"), "voice"); + LocalEditAssetProvider provider = new LocalEditAssetProvider(properties); + + assertThat(provider.resolve(new EditAssetRequest(EditAssetType.FONT, "title", null, 0, null))).isPresent(); + assertThat(provider.resolve(new EditAssetRequest(EditAssetType.LUT, "warm", null, 0, null))).isPresent(); + assertThat(provider.resolve(new EditAssetRequest(EditAssetType.VOICEOVER, "line-001", null, 2, null))) + .isPresent(); + } + + private VideoClippingProperties properties() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().getAssets().setMusicFolder(tempDir.resolve("music").toString()); + properties.getEditing().getAssets().setSfxFolder(tempDir.resolve("sfx").toString()); + properties.getEditing().getAssets().setFontsFolder(tempDir.resolve("fonts").toString()); + properties.getEditing().getAssets().setLutsFolder(tempDir.resolve("luts").toString()); + properties.getEditing().getAssets().setVoiceoverFolder(tempDir.resolve("voiceover").toString()); + return properties; + } +}