diff --git a/docs/cinematic-highlight-operator-checklist.md b/docs/cinematic-highlight-operator-checklist.md new file mode 100644 index 0000000..caac7a4 --- /dev/null +++ b/docs/cinematic-highlight-operator-checklist.md @@ -0,0 +1,98 @@ +# Cinematic Highlight Operator Checklist + +Use this when you want one source video turned into a cinematic highlight project. + +## Start Here + +1. Start the service with the cinematic local profile: + +```bash +mvn spring-boot:run -Dspring-boot.run.profiles=cinematic-editing-local +``` + +2. Put one valid source video in: + +```text +input/highlights/source/ +``` + +3. Watch the logs for: + +- `event=highlight_scan_started` +- `event=highlight_candidate_selected` +- `event=highlight_project_created` +- `event=highlight_director_prompt_generated` +- `event=highlight_scan_completed` + +## Exact Project Folders + +Project root: + +```text +output/highlight-projects// +``` + +Files and folders the service writes: + +- `project.json` +- `source/` +- `analysis/source-analysis.json` +- `analysis/category.json` +- `analysis/highlight-candidates.json` +- `director/director-brief.md` +- `director/director-prompt.md` +- `director/edit-plan.json` +- `highlights//storyboard.md` +- `highlights//visual-effects.json` +- `highlights//assets/requests/` +- `highlights//assets/music/` +- `highlights//assets/sfx/` +- `highlights//assets/voiceover/` +- `highlights//rendered-clips/` +- `highlights//preview.mp4` +- `highlights//final.mp4` +- `highlights//render-manifest.json` +- `highlights//qa-report.json` + +## Exact Operator Flow + +1. Wait for the service to create the highlight project. +2. Open: + +```text +output/highlight-projects//director/director-prompt.md +``` + +3. Run Codex, Claude, or another filesystem-capable AI instance on that prompt. +4. Make the AI write the finished plan to: + +```text +output/highlight-projects//director/edit-plan.json +``` + +5. Confirm `project.json` moved to `WAITING_FOR_DIRECTOR` after prompt generation. +6. The service now turns the director plan into `highlights//visual-effects.json`. +7. The asset bridge writes request files and the local asset worker resolves what it can into `highlights//assets/`. +8. The renderer publishes `rendered-clips/clip_*.mp4`, `preview.mp4`, `final.mp4`, `render-manifest.json`, and `qa-report.json`. + +## What Is Automatic Today + +- The scheduler analyzes the source video. +- The service writes the director prompt and brief. +- Existing analyzed projects are backfilled on startup if the prompt is missing. +- The visual-effects stage writes explicit renderer instructions. +- The local asset worker tries to resolve or generate reusable music, SFX, and voiceover assets. + +## What Is Still Manual + +- Running the AI director. +- Triggering any downstream render step that your current flow requires. + +## If Output Is Missing + +- No file in `input/highlights/source/` +- Unsupported video file +- The scheduler has not run yet +- `output/highlight-projects//director/director-prompt.md` is missing +- `director/edit-plan.json` was not written by the AI +- Rendering is not wired for the current highlight flow yet diff --git a/docs/local-cinematic-asset-worker-prompt.md b/docs/local-cinematic-asset-worker-prompt.md new file mode 100644 index 0000000..87eabf2 --- /dev/null +++ b/docs/local-cinematic-asset-worker-prompt.md @@ -0,0 +1,204 @@ +# Local Cinematic Asset Worker Prompt + +Use this prompt for the local asset worker that runs on the VPS after the director plan has been written. + +## Purpose + +Generate production-grade local assets for a cinematic video edit: + +- spoken voiceover audio +- cinematic music bed +- sound effects +- a machine-readable asset manifest + +The worker must use the best local model available for each job: + +- `Piper` or another local neural TTS engine for voiceover +- `MusicGen` via `AudioCraft` for music +- `AudioGen` via `AudioCraft` for SFX + +If a preferred model is unavailable, fall back to the next best local option, but do not invent fake assets or silently skip required outputs. + +## Inputs + +Work only with the current project folder and its generated plan: + +- `director/edit-plan.json` +- `highlights//edit-plan.json` +- `highlights//storyboard.md` +- `highlights//visual-effects.json` +- `analysis/` +- `assets/requests/` +- `assets/music/` +- `assets/sfx/` +- `assets/voiceover/` + +Use the analysis artifacts and director plan as the source of truth. Do not invent scene facts, car specs, people, locations, or actions that are not visible in the footage or explicitly stated in the plan. + +## Quality Bar + +The generated assets must feel intentional and cinematic: + +- voiceover must sound natural, premium, and grounded in visible footage +- music must match the project category and energy curve +- SFX must support cuts, reveals, transitions, and emphasis without sounding noisy or random +- assets must be reusable in future projects when possible +- the result must be good enough for public upload, not placeholder quality + +## Model Selection + +Prefer this local model split: + +1. Voiceover + - Use a local neural TTS model first. + - Prefer `Piper` for fast, reliable, local synthesis. + - If a higher-quality local TTS engine is available on the VPS, use it only if it clearly improves speech naturalness and pronunciation. + - Keep delivery human, clear, and premium. + +2. Music + - Use `MusicGen` through `AudioCraft` for text-to-music generation. + - Generate a loopable or long-form bed that can cover the timeline cleanly. + - Preserve room for voiceover and key SFX. + +3. SFX + - Use `AudioGen` through `AudioCraft` for text-to-sound generation. + - If a specific SFX prompt produces weak results, retry with a shorter, more concrete prompt. + - Use category-aware effects such as whooshes, rises, impacts, camera accents, engine tones, texture hits, and ambience. + +## Instructions + +You are the local cinematic asset worker. + +Your job is to create the best possible reusable asset set for this project. + +Follow these rules: + +- Read the edit plan first. +- Read the storyboard and visual effects plan second. +- Use the project category to drive style, pacing, tone, and sound design. +- Keep voiceover grounded in visible footage. +- Keep music and SFX supportive, not distracting. +- Generate assets in stable filenames so the renderer can reuse them. +- Write a manifest that records every generated, reused, and skipped asset. +- Prefer generating one strong asset over several weak variants. +- If the prompt asks for a voiceover line, synthesize it to WAV. +- If the prompt asks for music, synthesize a WAV bed that spans the intended duration. +- If the prompt asks for SFX, synthesize one WAV per cue. +- If generation fails, retry once with a tighter prompt. +- If it still fails, write a request record and mark the asset as pending. + +## Voiceover Requirements + +When voiceover is enabled: + +- synthesize every required line to `assets/voiceover/voiceover.wav` +- if the plan contains multiple lines, synthesize them into a single clean narration track unless the renderer expects split assets +- choose a voice that fits the category +- use confident, concise, polished delivery +- do not sound salesy, robotic, or exaggerated +- keep wording faithful to visible footage + +Voiceover style by category: + +- `car_vlog`: confident, premium, controlled, slightly aspirational +- `family_vlog`: warm, human, intimate, gentle +- `food_vlog`: sensory, warm, appetizing, precise +- `generic_vlog`: clean, observational, polished, grounded + +## Music Requirements + +When music is needed: + +- generate a cinematic bed that matches the story arc +- build energy from intro to payoff +- leave a pocket for voiceover clarity +- avoid constant high-intensity density +- choose tone based on category and scene + +Music style by category: + +- `car_vlog`: confident pulse, modern cinematic drive, controlled bass, sleek tension +- `family_vlog`: emotional, warm, reflective, soft rise +- `food_vlog`: light groove, tactile warmth, subtle motion +- `generic_vlog`: tasteful atmospheric score with restrained lift + +## SFX Requirements + +When SFX are needed: + +- align each effect to a cut, reveal, beat change, or visual emphasis +- use short, clean transients rather than heavy clutter +- keep effects category-aware +- avoid overusing impacts + +Useful SFX prompts by category: + +- `car_vlog`: soft whoosh, bass hit, subtle rise, mechanical accent, road ambience +- `family_vlog`: gentle whoosh, soft room tone, warm ambience, subtle scene transition +- `food_vlog`: tactile foley, pour, chop, sizzle, plate tap, light texture accents +- `generic_vlog`: clean whoosh, understated hit, transition swell, ambient bed + +## Output Files + +Create these files when applicable: + +- `assets/voiceover/voiceover.wav` +- `assets/music/music.wav` +- `assets/sfx/.wav` +- `assets/generated-assets.json` +- `assets/asset-generation-manifest.json` + +Also create request files for anything still missing: + +- `assets/requests/-.md` +- `assets/requests/-.json` + +## Manifest Requirements + +The manifest must include: + +- project id +- highlight id +- asset type +- asset key +- source model or provider +- prompt text +- output path +- status: `generated`, `reused`, or `pending` +- reason if not generated + +## Suggested Prompt Templates + +### Voiceover Prompt + +Write a short, polished narration for this footage. +Keep it grounded in what is visibly shown. +Match the category tone. +Avoid unsupported facts. +Avoid generic praise. +Make the result sound premium and human. + +### Music Prompt + +Create a cinematic background track for this edit. +Match the category tone and the pacing curve in the director plan. +Keep the track polished, modern, and supportive of voiceover. +Use a clear intro, a controlled build, and a satisfying payoff. + +### SFX Prompt + +Create a precise cinematic sound effect for the described edit moment. +Make it short, clean, and category-appropriate. +Use the effect to support the cut or reveal, not to overwhelm the scene. + +## Final Acceptance Criteria + +The asset stage is done only when: + +- voiceover audio exists if voiceover is enabled +- music audio exists if music is enabled +- required SFX files exist or are explicitly marked pending +- the manifest records what was generated and what was reused +- filenames are stable and project-local +- assets are ready for the renderer without manual cleanup + diff --git a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java index 3e9808d..6c0a50b 100644 --- a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java +++ b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java @@ -530,7 +530,7 @@ public class VideoClippingProperties { private String audioBitrate = "192k"; - private String voiceoverProvider = "noop"; + private String voiceoverProvider = "local"; private double loudnessTargetI = -16.0; @@ -552,6 +552,8 @@ public class VideoClippingProperties { private final VisualAnalysis visualAnalysis = new VisualAnalysis(); + private final LocalAssetWorker localAssetWorker = new LocalAssetWorker(); + private final LocalDirector localDirector = new LocalDirector(); private final HighlightScheduler highlightScheduler = new HighlightScheduler(); @@ -788,6 +790,10 @@ public class VideoClippingProperties { return visualAnalysis; } + public LocalAssetWorker getLocalAssetWorker() { + return localAssetWorker; + } + public LocalDirector getLocalDirector() { return localDirector; } @@ -951,6 +957,130 @@ public class VideoClippingProperties { } } + public static class LocalAssetWorker { + private boolean autoStart = false; + + private boolean strictRuntime = false; + + private String bootstrapScript = "./tools/run_local_asset_worker.sh"; + + private String script = "./tools/local_asset_worker.py"; + + @Min(0) + private long startupWaitMs = 0; + + private String healthPath = "/health"; + + @Min(1) + private long healthCheckIntervalMs = 1000; + + private String pythonBinary = "./.venv-local-asset/bin/python"; + + private String piperBinary = "piper"; + + private String piperModelPath = ""; + + private String musicModel = "musicgen-small"; + + private String sfxModel = "audiogen-medium"; + + public boolean isAutoStart() { + return autoStart; + } + + public void setAutoStart(boolean autoStart) { + this.autoStart = autoStart; + } + + public boolean isStrictRuntime() { + return strictRuntime; + } + + public void setStrictRuntime(boolean strictRuntime) { + this.strictRuntime = strictRuntime; + } + + public String getBootstrapScript() { + return bootstrapScript; + } + + public void setBootstrapScript(String bootstrapScript) { + this.bootstrapScript = bootstrapScript; + } + + public String getScript() { + return script; + } + + public void setScript(String script) { + this.script = script; + } + + public long getStartupWaitMs() { + return startupWaitMs; + } + + public void setStartupWaitMs(long startupWaitMs) { + this.startupWaitMs = startupWaitMs; + } + + public String getHealthPath() { + return healthPath; + } + + public void setHealthPath(String healthPath) { + this.healthPath = healthPath; + } + + public long getHealthCheckIntervalMs() { + return healthCheckIntervalMs; + } + + public void setHealthCheckIntervalMs(long healthCheckIntervalMs) { + this.healthCheckIntervalMs = healthCheckIntervalMs; + } + + public String getPythonBinary() { + return pythonBinary; + } + + public void setPythonBinary(String pythonBinary) { + this.pythonBinary = pythonBinary; + } + + public String getPiperBinary() { + return piperBinary; + } + + public void setPiperBinary(String piperBinary) { + this.piperBinary = piperBinary; + } + + public String getPiperModelPath() { + return piperModelPath; + } + + public void setPiperModelPath(String piperModelPath) { + this.piperModelPath = piperModelPath; + } + + public String getMusicModel() { + return musicModel; + } + + public void setMusicModel(String musicModel) { + this.musicModel = musicModel; + } + + public String getSfxModel() { + return sfxModel; + } + + public void setSfxModel(String sfxModel) { + this.sfxModel = sfxModel; + } + } + public static class LocalDirector { private boolean enabled = true; diff --git a/src/main/java/org/example/videoclips/editing/AssetGenerationItem.java b/src/main/java/org/example/videoclips/editing/AssetGenerationItem.java new file mode 100644 index 0000000..b942db9 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/AssetGenerationItem.java @@ -0,0 +1,13 @@ +package org.example.videoclips.editing; + +public record AssetGenerationItem( + String type, + String assetKey, + String cacheKey, + boolean reused, + String sourcePath, + String targetPath, + String requestPath, + String notes +) { +} diff --git a/src/main/java/org/example/videoclips/editing/AssetGenerationResult.java b/src/main/java/org/example/videoclips/editing/AssetGenerationResult.java new file mode 100644 index 0000000..d6964a8 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/AssetGenerationResult.java @@ -0,0 +1,15 @@ +package org.example.videoclips.editing; + +import java.time.Instant; +import java.util.List; + +public record AssetGenerationResult( + String projectId, + boolean readyForRender, + List items, + Instant createdAt +) { + public AssetGenerationResult { + items = items == null ? List.of() : List.copyOf(items); + } +} diff --git a/src/main/java/org/example/videoclips/editing/AssetGenerationStage.java b/src/main/java/org/example/videoclips/editing/AssetGenerationStage.java new file mode 100644 index 0000000..1419611 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/AssetGenerationStage.java @@ -0,0 +1,6 @@ +package org.example.videoclips.editing; + +public interface AssetGenerationStage { + + AssetGenerationResult prepare(String projectId, EditPlan plan); +} diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index 1146552..bfe373c 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -105,7 +105,7 @@ public class FfmpegEditRenderer implements EditRenderer { } private void renderInternal(String projectId, long startedAt) { - EditPlan plan = validator.validate(projectId, store.readJson(projectId, "edit-plan.json", EditPlan.class)); + EditPlan plan = validator.validate(projectId, readEditPlan(projectId)); if (observability != null) { observability.renderStarted(projectId, plan.decisions().size()); } @@ -167,6 +167,23 @@ public class FfmpegEditRenderer implements EditRenderer { } } + private EditPlan readEditPlan(String projectId) { + Path editPlan = store.editPlanFile(projectId); + if (Files.isRegularFile(editPlan)) { + return store.readJson(projectId, "edit-plan.json", EditPlan.class); + } + Path inboxPlan = store.inboxDirectory(projectId).resolve("edit-plan.json"); + if (Files.isRegularFile(inboxPlan)) { + try { + return new com.fasterxml.jackson.databind.ObjectMapper().findAndRegisterModules() + .readValue(inboxPlan.toFile(), EditPlan.class); + } catch (IOException ex) { + throw new IllegalStateException("Unable to read edit plan from inbox: " + inboxPlan, ex); + } + } + throw new IllegalStateException("Unable to locate edit plan for project: " + projectId); + } + List segmentCommand(String source, EditDecision decision, Path output) { double outputDuration = (decision.sourceEndSeconds() - decision.sourceStartSeconds()) / decision.playbackSpeed(); StringBuilder filter = new StringBuilder(); diff --git a/src/main/java/org/example/videoclips/editing/HighlightAssetPreparationService.java b/src/main/java/org/example/videoclips/editing/HighlightAssetPreparationService.java new file mode 100644 index 0000000..e21e0e0 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightAssetPreparationService.java @@ -0,0 +1,190 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HighlightAssetPreparationService { + + private static final Logger log = LoggerFactory.getLogger(HighlightAssetPreparationService.class); + + private final HighlightProjectStore store; + private final EditAssetProvider assetProvider; + private final EditAssetLibrary assetLibrary; + private final VideoClippingProperties.Editing.Assets assets; + private final ObjectMapper objectMapper; + + public HighlightAssetPreparationService(VideoClippingProperties properties, HighlightProjectStore store, + EditAssetProvider assetProvider, EditAssetLibrary assetLibrary, + ObjectMapper objectMapper) { + this.store = store; + this.assetProvider = assetProvider; + this.assetLibrary = assetLibrary; + this.assets = properties.getEditing().getAssets(); + this.objectMapper = objectMapper; + } + + public HighlightAssetPreparationResult prepare(String projectId, HighlightProject project, + HighlightDirectorPlan.HighlightItem highlight, + ContentCategory category) { + log.info("event=highlight_asset_preparation_started project_id={} highlight_id={} category={} target_duration_seconds={}", + projectId, highlight.highlightId(), category, highlight.targetDurationSeconds()); + Path highlightDirectory = store.highlightsDirectory(projectId).resolve(highlight.highlightId()); + Path assetsDirectory = highlightDirectory.resolve("assets"); + Path musicDirectory = assetsDirectory.resolve("music"); + Path sfxDirectory = assetsDirectory.resolve("sfx"); + Path voiceoverDirectory = assetsDirectory.resolve("voiceover"); + Path requestDirectory = assetsDirectory.resolve("requests"); + createDirectories(musicDirectory, sfxDirectory, voiceoverDirectory, requestDirectory); + + List requestFiles = new ArrayList<>(); + List resolvedAssets = new ArrayList<>(); + + Optional music = assetLibrary.select(new EditAssetSelectionRequest(EditAssetType.MUSIC, + category, highlight.musicDirection(), highlight.targetDurationSeconds())); + if (music.isPresent()) { + Path target = musicDirectory.resolve("music.wav"); + copy(Path.of(music.get().path()), target); + resolvedAssets.add(target.toString()); + } else { + requestFiles.add(writeRequest(requestDirectory, projectId, highlight.highlightId(), "music", + "music_bed", highlight.musicDirection(), musicDirectory.resolve("music.wav"), + highlight.targetDurationSeconds(), false)); + } + + if (!highlight.voiceover().isEmpty()) { + Path script = voiceoverDirectory.resolve("voiceover-script.txt"); + writeString(script, String.join(System.lineSeparator(), highlight.voiceover())); + Optional voiceover = assetProvider.list(EditAssetType.VOICEOVER, null).stream() + .findFirst(); + if (voiceover.isPresent()) { + Path target = voiceoverDirectory.resolve("voiceover.wav"); + copy(Path.of(voiceover.get().path()), target); + resolvedAssets.add(target.toString()); + } else { + requestFiles.add(writeRequest(requestDirectory, projectId, highlight.highlightId(), "voiceover", + "voiceover_script", String.join(System.lineSeparator(), highlight.voiceover()), + voiceoverDirectory.resolve("voiceover.wav"), highlight.targetDurationSeconds(), false)); + } + } + + List sfxCues = sfxCues(highlight); + for (AudioCue cue : sfxCues) { + Optional resolved = assetProvider.resolve(new EditAssetRequest(EditAssetType.SFX, + cue.assetKey(), category, cue.timelineEndSeconds() - cue.timelineStartSeconds(), cue.notes())); + Path target = sfxDirectory.resolve(cue.assetKey() + ".wav"); + if (resolved.isPresent()) { + copy(Path.of(resolved.get().path()), target); + resolvedAssets.add(target.toString()); + } else { + requestFiles.add(writeRequest(requestDirectory, projectId, highlight.highlightId(), "sfx", + cue.assetKey(), cue.notes(), target, cue.timelineEndSeconds() - cue.timelineStartSeconds(), + true)); + } + } + + log.info("event=highlight_asset_preparation_completed project_id={} highlight_id={} resolved={} requests={}", + projectId, highlight.highlightId(), resolvedAssets.size(), requestFiles.size()); + return new HighlightAssetPreparationResult(highlightDirectory, assetsDirectory, requestFiles, resolvedAssets); + } + + private List sfxCues(HighlightDirectorPlan.HighlightItem highlight) { + double duration = Math.max(1.0, highlight.targetDurationSeconds()); + List cues = new ArrayList<>(); + if (highlight.storyPurpose() != null && highlight.storyPurpose().contains("opening")) { + cues.add(new AudioCue("sfx", "whoosh_soft", 0.0, Math.min(0.8, duration), -6.0, + highlight.sfxDirection())); + } + cues.add(new AudioCue("sfx", "impact_hit", Math.max(0.0, duration - 0.8), duration, -3.0, + highlight.sfxDirection())); + return cues; + } + + private String writeRequest(Path requestDirectory, String projectId, String highlightId, String type, + String assetKey, String notes, Path target, double durationSeconds, boolean blocking) { + Path requestFile = requestDirectory.resolve(type + "-" + assetKey + ".md"); + Path requestJson = requestDirectory.resolve(type + "-" + assetKey + ".json"); + String content = """ + # Highlight Asset Request + + Project: `%s` + Highlight: `%s` + Type: `%s` + Asset key: `%s` + Project target: `%s` + Blocking for render: `%s` + Duration seconds: %.3f + + Notes: + %s + """.formatted(projectId, highlightId, type, assetKey, target, blocking, durationSeconds, + notes == null ? "" : notes); + writeString(requestFile, content); + writeRequestJson(requestJson, new HighlightAssetRequest(projectId, highlightId, type, assetKey, + target.toString(), requestFile.toString(), notes, durationSeconds, blocking)); + return requestFile.toString(); + } + + private void writeRequestJson(Path path, HighlightAssetRequest request) { + try { + Files.createDirectories(path.getParent()); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), request); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write highlight asset request JSON: " + path, ex); + } + } + + private void writeString(Path path, String value) { + try { + Files.createDirectories(path.getParent()); + Files.writeString(path, value, StandardCharsets.UTF_8); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write highlight asset file: " + path, ex); + } + } + + private void copy(Path source, Path target) { + try { + Files.createDirectories(target.getParent()); + Files.copy(source, target, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ex) { + throw new IllegalStateException("Unable to copy highlight asset: " + source, ex); + } + } + + private void createDirectories(Path... directories) { + for (Path directory : directories) { + try { + Files.createDirectories(directory); + } catch (IOException ex) { + throw new IllegalStateException("Unable to create highlight asset directory: " + directory, ex); + } + } + } + + public record HighlightAssetPreparationResult( + Path highlightDirectory, + Path assetsDirectory, + List requestFiles, + List resolvedAssets + ) { + public HighlightAssetPreparationResult { + requestFiles = requestFiles == null ? List.of() : List.copyOf(requestFiles); + resolvedAssets = resolvedAssets == null ? List.of() : List.copyOf(resolvedAssets); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java b/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java index 2ea6ab7..043fb3f 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java +++ b/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java @@ -65,7 +65,6 @@ public class HighlightDirectorFlowService { markFailed(project, "No highlights were provided by the director plan"); throw new IllegalStateException("Director plan contains no highlights"); } - markStatus(project, HighlightProjectStatus.RENDERING, null); log.info("event=highlight_flow_started flow_id={} scan_id={} project_id={} source_file={} highlights={}", flowId, scanId, projectId, project.sourceVideoFileName(), plan.highlights().size()); List finalOutputs = new ArrayList<>(); @@ -89,6 +88,12 @@ public class HighlightDirectorFlowService { log.info("event=highlight_asset_worker_completed flow_id={} project_id={} highlight_id={} resolved={} pending={}", flowId, projectId, highlight.highlightId(), assetWorkerResult.resolvedAssets().size(), assetWorkerResult.pendingRequests().size()); + if (!assetWorkerResult.pendingRequests().isEmpty()) { + log.warn("event=highlight_flow_waiting_for_assets flow_id={} project_id={} highlight_id={} pending_requests={}", + flowId, projectId, highlight.highlightId(), assetWorkerResult.pendingRequests()); + return HighlightFlowResult.skipped(projectId, flowId, "assets_pending"); + } + markStatus(project, HighlightProjectStatus.RENDERING, null); HighlightFfmpegRenderer.HighlightRenderResult result = renderer.render(projectId, highlight, editPlan); finalOutputs.add(result.finalOutput()); renderedHighlightIds.add(result.highlightId()); diff --git a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java index 73481c1..343530a 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java +++ b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java @@ -2,6 +2,7 @@ package org.example.videoclips.editing; import org.example.videoclips.config.VideoClippingProperties; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import java.io.IOException; @@ -27,6 +28,7 @@ public class HighlightFfmpegRenderer { private final EditObservability observability; private final ProcessExecutor executor; + @Autowired public HighlightFfmpegRenderer(VideoClippingProperties properties, HighlightProjectStore store, EditAssetProvider assetProvider, EditObservability observability) { this(properties, store, assetProvider, observability, HighlightFfmpegRenderer::execute); @@ -119,12 +121,26 @@ public class HighlightFfmpegRenderer { "sfx", Integer.toString(sfx.size()), "voiceover", Boolean.toString(Files.isRegularFile(voiceover))); if (Files.isRegularFile(music) || Files.isRegularFile(voiceover) || !sfx.isEmpty()) { - run(audioMixCommand(postTimeline, Files.isRegularFile(music) ? music : null, - Files.isRegularFile(voiceover) ? voiceover : null, sfx, output), commands); + try { + run(audioMixCommand(postTimeline, Files.isRegularFile(music) ? music : null, + Files.isRegularFile(voiceover) ? voiceover : null, sfx, output), commands); + } catch (IllegalStateException ex) { + log(projectId, highlight.highlightId(), "highlight_render_audio_failed", + "error", summarizeMessage(ex.getMessage()), + "fallback", "copy_post_timeline"); + copy(postTimeline, output); + } } else { copy(postTimeline, output); } - run(previewCommand(output, preview), commands); + try { + run(previewCommand(output, preview), commands); + } catch (IllegalStateException ex) { + log(projectId, highlight.highlightId(), "highlight_render_preview_failed", + "error", summarizeMessage(ex.getMessage()), + "fallback", "copy_final_output"); + copy(output, preview); + } double duration = plan.decisions().isEmpty() ? 0 : plan.decisions().get(plan.decisions().size() - 1) .timelineEndSeconds(); RenderManifest manifest = new RenderManifest(projectId, @@ -403,10 +419,11 @@ public class HighlightFfmpegRenderer { private void run(List command, List> commands) { try { - ProcessResult result = executor.execute(command); commands.add(command); + ProcessResult result = executor.execute(command); if (result.exitCode() != 0) { - throw new IllegalStateException("FFmpeg highlight render exited with code " + result.exitCode()); + throw new IllegalStateException("FFmpeg highlight render exited with code " + result.exitCode() + + messageSuffix(result.output())); } } catch (InterruptedException ex) { Thread.currentThread().interrupt(); @@ -416,6 +433,20 @@ public class HighlightFfmpegRenderer { } } + private String summarizeMessage(String message) { + if (message == null || message.isBlank()) { + return "n/a"; + } + return message.length() <= 500 ? message : message.substring(0, 500) + "..."; + } + + private String messageSuffix(String output) { + if (output == null || output.isBlank()) { + return ""; + } + return ": " + summarizeMessage(output); + } + private void log(String projectId, String highlightId, String event, String... kvPairs) { StringBuilder builder = new StringBuilder("event=").append(event).append(" project_id=").append(projectId) .append(" highlight_id=").append(highlightId); diff --git a/src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java b/src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java index 4807a31..80eaa84 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java +++ b/src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java @@ -26,13 +26,16 @@ public class HighlightLocalAssetWorker { private final HighlightProjectStore store; private final EditAssetProvider assetProvider; private final EditAssetLibrary assetLibrary; + private final LocalAssetSynthesizer synthesizer; private final ObjectMapper objectMapper; public HighlightLocalAssetWorker(HighlightProjectStore store, EditAssetProvider assetProvider, - EditAssetLibrary assetLibrary, ObjectMapper objectMapper) { + EditAssetLibrary assetLibrary, LocalAssetSynthesizer synthesizer, + ObjectMapper objectMapper) { this.store = store; this.assetProvider = assetProvider; this.assetLibrary = assetLibrary; + this.synthesizer = synthesizer; this.objectMapper = objectMapper; } @@ -44,9 +47,14 @@ public class HighlightLocalAssetWorker { return new HighlightAssetWorkerResult(projectId, highlight.highlightId(), List.of(), List.of()); } List requests = readRequests(requestsDirectory); + log.info("event=highlight_asset_worker_started project_id={} highlight_id={} request_count={} request_directory={} category={}", + projectId, highlight.highlightId(), requests.size(), requestsDirectory, category); List resolved = new ArrayList<>(); List pending = new ArrayList<>(); for (HighlightAssetRequest request : requests) { + log.info("event=highlight_asset_generation_requested project_id={} highlight_id={} type={} asset_key={} target={} blocking={} duration_seconds={} request={}", + projectId, highlight.highlightId(), request.type(), request.assetKey(), request.targetPath(), + request.blocking(), request.durationSeconds(), request.requestPath()); Optional materialized = materialize(request, category); if (materialized.isPresent()) { resolved.add(materialized.get().toString()); @@ -58,6 +66,8 @@ public class HighlightLocalAssetWorker { projectId, highlight.highlightId(), request.type(), request.requestPath()); } } + log.info("event=highlight_asset_worker_completed project_id={} highlight_id={} resolved={} pending={}", + projectId, highlight.highlightId(), resolved.size(), pending.size()); return new HighlightAssetWorkerResult(projectId, highlight.highlightId(), resolved, pending); } @@ -101,7 +111,7 @@ public class HighlightLocalAssetWorker { copy(Path.of(asset.get().path()), target); return Optional.of(target); } - return generateToneBed(target, request.durationSeconds()); + return synthesizer.synthesizeMusic(request.projectId(), request.notes(), request.durationSeconds(), target); } private Optional resolveOrGenerateSfx(HighlightAssetRequest request, ContentCategory category) { @@ -118,7 +128,7 @@ public class HighlightLocalAssetWorker { copy(Path.of(fallback.get().path()), target); return Optional.of(target); } - return generateImpactTone(target, request.durationSeconds()); + return synthesizer.synthesizeSfx(request.projectId(), request.notes(), request.durationSeconds(), target); } private Optional resolveOrGenerateVoiceover(HighlightAssetRequest request) { @@ -127,70 +137,8 @@ public class HighlightLocalAssetWorker { return Optional.empty(); } Path target = Path.of(request.targetPath()); - if (runSay(text, target)) { - return Optional.of(target); - } - if (runEspeak(text, target)) { - return Optional.of(target); - } - return Optional.empty(); - } - - private Optional generateToneBed(Path target, double durationSeconds) { - return generateAudio(target, durationSeconds, "sine=frequency=110:sample_rate=48000", 0.02); - } - - private Optional generateImpactTone(Path target, double durationSeconds) { - return generateAudio(target, Math.max(0.5, Math.min(1.0, durationSeconds)), - "sine=frequency=880:sample_rate=48000", 0.12); - } - - private Optional generateAudio(Path target, double durationSeconds, String source, double volume) { - try { - Files.createDirectories(target.getParent()); - List command = List.of("ffmpeg", "-hide_banner", "-loglevel", "error", "-y", - "-f", "lavfi", "-i", source, "-t", Double.toString(durationSeconds), - "-af", "volume=" + volume, "-c:a", "aac", target.toString()); - Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); - String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); - if (process.waitFor() == 0) { - return Optional.of(target); - } - log.warn("event=highlight_asset_generation_failed target={} message={}", target, output); - return Optional.empty(); - } catch (IOException | InterruptedException ex) { - Thread.currentThread().interrupt(); - log.warn("event=highlight_asset_generation_failed target={} error_type={} message={}", - target, ex.getClass().getSimpleName(), ex.getMessage()); - return Optional.empty(); - } - } - - private boolean runSay(String text, Path target) { - return runSpeechCommand(List.of("say", "-o", target.toString(), text), target); - } - - private boolean runEspeak(String text, Path target) { - return runSpeechCommand(List.of("espeak", "-w", target.toString(), text), target); - } - - private boolean runSpeechCommand(List command, Path target) { - try { - Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); - String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); - if (process.waitFor() == 0 && Files.isRegularFile(target)) { - return true; - } - if (!output.isBlank()) { - log.warn("event=highlight_voiceover_generation_failed target={} output={}", target, output); - } - return false; - } catch (IOException | InterruptedException ex) { - Thread.currentThread().interrupt(); - log.warn("event=highlight_voiceover_generation_failed target={} error_type={} message={}", target, - ex.getClass().getSimpleName(), ex.getMessage()); - return false; - } + return synthesizer.synthesizeVoiceover(request.projectId(), List.of(new VoiceoverLine(text, 0.0, + Math.max(1.0, request.durationSeconds()), "cinematic_narration")), target); } private void copy(Path source, Path target) { diff --git a/src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java b/src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java index 04a5945..ce9b471 100644 --- a/src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java +++ b/src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java @@ -28,15 +28,17 @@ public class LocalAssetGenerationStage implements AssetGenerationStage { private final EditProjectStore store; private final EditAssetProvider provider; private final EditAssetLibrary library; + private final LocalAssetSynthesizer synthesizer; private final Path voiceoverCacheRoot; private final ObjectMapper objectMapper; public LocalAssetGenerationStage(VideoClippingProperties properties, EditProjectStore store, EditAssetProvider provider, EditAssetLibrary library, - ObjectMapper objectMapper) { + LocalAssetSynthesizer synthesizer, ObjectMapper objectMapper) { this.store = store; this.provider = provider; this.library = library; + this.synthesizer = synthesizer; this.voiceoverCacheRoot = Path.of(properties.getEditing().getAssets().getVoiceoverFolder()).normalize(); this.objectMapper = objectMapper; } @@ -151,6 +153,13 @@ public class LocalAssetGenerationStage implements AssetGenerationStage { objectMapper.writerWithDefaultPrettyPrinter().writeValue(requestDirectory.resolve("request.json").toFile(), new AssetGenerationItem(type, assetKey, cacheKey, false, null, projectTarget.toString(), requestFile.toString(), notes)); + Optional generated = generate(type, projectId, notes, durationSeconds, projectTarget); + if (generated.isPresent()) { + log.info("event=asset_materialized project_id={} type={} asset_key={} cache_key={} source={} target={} strategy=local-worker", + projectId, type, assetKey, cacheKey, generated.get(), projectTarget); + return new AssetGenerationItem(type, assetKey, cacheKey, true, generated.get().toString(), + projectTarget.toString(), requestFile.toString(), notes); + } log.info("event=asset_generation_requested project_id={} type={} asset_key={} cache_key={} request={}", projectId, type, assetKey, cacheKey, requestFile); return new AssetGenerationItem(type, assetKey, cacheKey, false, null, projectTarget.toString(), @@ -179,6 +188,23 @@ public class LocalAssetGenerationStage implements AssetGenerationStage { return Optional.empty(); } + private Optional generate(String type, String projectId, String notes, double durationSeconds, + Path target) { + if ("music".equals(type)) { + return synthesizer.synthesizeMusic(projectId, notes, durationSeconds, target); + } + if ("voiceover".equals(type)) { + List lines = notes == null || notes.isBlank() + ? List.of() + : List.of(new VoiceoverLine(notes, 0.0, Math.max(1.0, durationSeconds), "cinematic_narration")); + return synthesizer.synthesizeVoiceover(projectId, lines, target); + } + if ("sfx".equals(type)) { + return synthesizer.synthesizeSfx(projectId, notes, durationSeconds, target); + } + return Optional.empty(); + } + private String request(String projectId, String type, String assetKey, String cacheKey, Path cachePath, Path projectTarget, String notes, double durationSeconds, boolean blocking) { return """ diff --git a/src/main/java/org/example/videoclips/editing/LocalAssetRuntimeVerifier.java b/src/main/java/org/example/videoclips/editing/LocalAssetRuntimeVerifier.java new file mode 100644 index 0000000..3d6d9c9 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/LocalAssetRuntimeVerifier.java @@ -0,0 +1,205 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class LocalAssetRuntimeVerifier implements ApplicationRunner { + + private static final Logger log = LoggerFactory.getLogger(LocalAssetRuntimeVerifier.class); + + private final VideoClippingProperties.Editing.LocalAssetWorker worker; + + public LocalAssetRuntimeVerifier(VideoClippingProperties properties) { + this.worker = properties.getEditing().getLocalAssetWorker(); + } + + @Override + public void run(ApplicationArguments args) { + log.info("event=local_asset_runtime_check_started auto_start={} strict_runtime={} script={} python={} " + + "bootstrap_script={} piper_binary={} piper_model_path={} music_model={} sfx_model={}", + worker.isAutoStart(), worker.isStrictRuntime(), worker.getScript(), worker.getPythonBinary(), + worker.getBootstrapScript(), + worker.getPiperBinary(), worker.getPiperModelPath(), worker.getMusicModel(), worker.getSfxModel()); + if (!worker.isAutoStart()) { + log.info("event=local_asset_runtime_check_skipped reason=auto_start_disabled"); + return; + } + + bootstrapEnvironment(); + + RuntimeReadiness readiness = verifyRuntime(); + if (!readiness.ready()) { + log.error("event=local_asset_runtime_check_failed missing={} voiceover_ready={} music_ready={} sfx_ready={}", + readiness.missing(), readiness.voiceoverReady(), readiness.musicReady(), + readiness.sfxReady()); + if (worker.isStrictRuntime()) { + log.warn("event=local_asset_runtime_degraded strict_runtime_enabled=true missing={}", + readiness.missing()); + } + return; + } + + log.info("event=local_asset_runtime_check_completed status=ready voiceover_ready=true music_ready=true " + + "sfx_ready=true python={} piper_binary={} piper_model_path={}", + worker.getPythonBinary(), worker.getPiperBinary(), worker.getPiperModelPath()); + } + + private void bootstrapEnvironment() { + Path bootstrapScript = Path.of(worker.getBootstrapScript()).toAbsolutePath().normalize(); + if (!Files.isRegularFile(bootstrapScript)) { + throw new IllegalStateException("Local asset bootstrap script does not exist: " + bootstrapScript); + } + log.info("event=local_asset_bootstrap_started script={} bootstrap_mode=auto preload_only=true python={} " + + "piper_binary={} piper_model_path={}", + bootstrapScript, worker.getPythonBinary(), worker.getPiperBinary(), worker.getPiperModelPath()); + ProcessBuilder builder = new ProcessBuilder("bash", bootstrapScript.toString()) + .redirectErrorStream(true); + Map environment = builder.environment(); + environment.put("LOCAL_ASSET_BOOTSTRAP_MODE", "auto"); + environment.put("LOCAL_ASSET_PRELOAD_ONLY", "true"); + environment.put("LOCAL_ASSET_PIPER_BINARY", worker.getPiperBinary()); + if (worker.getPiperModelPath() != null && !worker.getPiperModelPath().isBlank()) { + environment.put("LOCAL_ASSET_PIPER_MODEL_PATH", worker.getPiperModelPath()); + } + long startedAt = System.nanoTime(); + try { + Process process = builder.start(); + try (BufferedReader reader = new BufferedReader( + new InputStreamReader(process.getInputStream(), StandardCharsets.UTF_8))) { + String line; + while ((line = reader.readLine()) != null) { + log.info("event=local_asset_bootstrap_output script={} message={}", bootstrapScript, line); + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + log.error("event=local_asset_bootstrap_failed script={} exit_code={} reason=non_blocking", + bootstrapScript, exitCode); + return; + } + log.info("event=local_asset_bootstrap_completed script={} elapsed_ms={}", bootstrapScript, + elapsedMillis(startedAt)); + } catch (IOException ex) { + log.error("event=local_asset_bootstrap_failed script={} error_type={} message={}", bootstrapScript, + ex.getClass().getSimpleName(), ex.getMessage()); + throw new IllegalStateException("Unable to bootstrap local asset runtime", ex); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + log.error("event=local_asset_bootstrap_failed script={} error_type={} message={}", bootstrapScript, + ex.getClass().getSimpleName(), ex.getMessage()); + throw new IllegalStateException("Interrupted while bootstrapping local asset runtime", ex); + } + } + + private RuntimeReadiness verifyRuntime() { + List missing = new ArrayList<>(); + boolean pythonReady = isPythonAudioStackReady(); + if (!Files.isRegularFile(Path.of(worker.getPythonBinary()))) { + missing.add("python_binary:" + worker.getPythonBinary()); + } + if (!Files.isRegularFile(Path.of(worker.getBootstrapScript()))) { + missing.add("bootstrap_script:" + worker.getBootstrapScript()); + } + if (!Files.isRegularFile(Path.of(worker.getScript()))) { + missing.add("worker_script:" + worker.getScript()); + } + if (!isExecutablePresent(worker.getPiperBinary())) { + missing.add("piper_binary:" + worker.getPiperBinary()); + } + if (worker.getPiperModelPath() == null || worker.getPiperModelPath().isBlank()) { + missing.add("piper_model_path:missing"); + } else if (!Files.isRegularFile(Path.of(worker.getPiperModelPath()))) { + missing.add("piper_model_path:" + worker.getPiperModelPath()); + } + if (!pythonReady) { + missing.add("python_audio_stack:torch,audiocraft,soundfile,numpy"); + } + return new RuntimeReadiness( + missing.isEmpty(), + isVoiceoverReady(pythonReady), + pythonReady, + pythonReady, + List.copyOf(missing) + ); + } + + private boolean isVoiceoverReady(boolean pythonReady) { + if (!pythonReady) { + return false; + } + if (worker.getPiperModelPath() == null || worker.getPiperModelPath().isBlank()) { + return false; + } + return isExecutablePresent(worker.getPiperBinary()); + } + + private boolean isPythonAudioStackReady() { + List command = List.of(worker.getPythonBinary(), "-c", + "import torch, audiocraft, soundfile, numpy"); + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + int exitCode = process.waitFor(); + if (exitCode != 0 && !output.isBlank()) { + log.warn("event=local_asset_python_stack_check_failed python={} exit_code={} output={}", + worker.getPythonBinary(), exitCode, output); + } + return exitCode == 0; + } catch (IOException ex) { + log.warn("event=local_asset_python_stack_check_failed python={} error_type={} message={}", + worker.getPythonBinary(), ex.getClass().getSimpleName(), ex.getMessage()); + return false; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + log.warn("event=local_asset_python_stack_check_failed python={} error_type={} message={}", + worker.getPythonBinary(), ex.getClass().getSimpleName(), ex.getMessage()); + return false; + } + } + + private boolean isExecutablePresent(String executable) { + try { + Process process = new ProcessBuilder(executable, "--help").redirectErrorStream(true).start(); + process.waitFor(); + return true; + } catch (IOException ex) { + return false; + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return false; + } + } + + private long elapsedMillis(long startedAt) { + return (System.nanoTime() - startedAt) / 1_000_000; + } + + private record RuntimeReadiness( + boolean ready, + boolean voiceoverReady, + boolean musicReady, + boolean sfxReady, + List missing + ) { + private RuntimeReadiness { + missing = missing == null ? List.of() : List.copyOf(missing); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/LocalAssetSynthesizer.java b/src/main/java/org/example/videoclips/editing/LocalAssetSynthesizer.java new file mode 100644 index 0000000..b58f925 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/LocalAssetSynthesizer.java @@ -0,0 +1,316 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Optional; +import javax.sound.sampled.AudioInputStream; +import javax.sound.sampled.AudioSystem; +import javax.sound.sampled.UnsupportedAudioFileException; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class LocalAssetSynthesizer { + + private static final Logger log = LoggerFactory.getLogger(LocalAssetSynthesizer.class); + + private final VideoClippingProperties.Editing.LocalAssetWorker worker; + private final ProcessExecutor processExecutor; + + @Autowired + public LocalAssetSynthesizer(VideoClippingProperties properties) { + this(properties, command -> new ProcessBuilder(command).redirectErrorStream(true).start()); + } + + LocalAssetSynthesizer(VideoClippingProperties properties, ProcessExecutor processExecutor) { + this.worker = properties.getEditing().getLocalAssetWorker(); + this.processExecutor = processExecutor; + } + + public Optional synthesizeVoiceover(String projectId, List lines, Path target) { + if (lines == null || lines.isEmpty()) { + return Optional.empty(); + } + String prompt = String.join(System.lineSeparator(), + lines.stream().map(VoiceoverLine::text).toList()); + return synthesize(projectId, "voiceover", prompt, target, Math.max(1.0, lines.size() * 2.5), true); + } + + public Optional synthesizeMusic(String projectId, String prompt, double durationSeconds, Path target) { + return synthesize(projectId, "music", prompt, target, durationSeconds, false); + } + + public Optional synthesizeSfx(String projectId, String prompt, double durationSeconds, Path target) { + return synthesize(projectId, "sfx", prompt, target, durationSeconds, false); + } + + private Optional synthesize(String projectId, String kind, String prompt, Path target, + double durationSeconds, boolean allowSpeechFallback) { + if (prompt == null || prompt.isBlank()) { + return Optional.empty(); + } + try { + String model = selectedModel(kind); + log.info("event=local_asset_generation_started project_id={} type={} model={} target={} duration_seconds={} strategy=local-worker", + projectId, kind, model, target, durationSeconds); + Files.createDirectories(target.getParent()); + Path promptFile = Files.createTempFile(target.getParent(), + sanitize(kind) + "-prompt-", ".txt"); + Files.writeString(promptFile, prompt, StandardCharsets.UTF_8); + List command = new ArrayList<>(); + command.add(worker.getPythonBinary()); + command.add(Path.of(worker.getScript()).toAbsolutePath().normalize().toString()); + command.add(kind); + command.add("--prompt-file"); + command.add(promptFile.toString()); + command.add("--output"); + command.add(target.toString()); + command.add("--duration"); + command.add(Double.toString(durationSeconds)); + if ("music".equals(kind)) { + command.add("--model"); + command.add(worker.getMusicModel()); + } else if ("sfx".equals(kind)) { + command.add("--model"); + command.add(worker.getSfxModel()); + } else if ("voiceover".equals(kind) && !worker.getPiperModelPath().isBlank()) { + command.add("--model"); + command.add(worker.getPiperModelPath()); + command.add("--piper-binary"); + command.add(worker.getPiperBinary()); + } + + if (runWorker(command, target)) { + log.info("event=local_asset_generation_completed project_id={} type={} model={} target={} strategy=local-worker", + projectId, kind, model, target); + return Optional.of(target); + } + + if ("voiceover".equals(kind) && allowSpeechFallback) { + log.info("event=local_asset_generation_started project_id={} type={} model={} target={} duration_seconds={} strategy=speech-fallback", + projectId, kind, selectedVoiceoverFallback(), target, durationSeconds); + if (runSpeechFallback(prompt, target)) { + if (!hasAudibleSignal(target)) { + log.warn("event=local_asset_voiceover_silence_detected project_id={} target={} strategy=speech-fallback", + projectId, target); + } else { + log.info("event=local_asset_generation_completed project_id={} type={} model={} target={} strategy=speech-fallback", + projectId, kind, selectedVoiceoverFallback(), target); + return Optional.of(target); + } + } + } + + if ("voiceover".equals(kind)) { + log.error("event=local_asset_voiceover_unavailable project_id={} target={} reason=no_audible_speech_generated", + projectId, target); + return Optional.empty(); + } + + if ("music".equals(kind)) { + if (worker.isStrictRuntime()) { + log.error("event=local_asset_generation_failed project_id={} type={} model={} target={} reason=strict_runtime_required", + projectId, kind, model, target); + return Optional.empty(); + } + return fallbackTone(target, durationSeconds, 110.0, 0.02, projectId, kind); + } + if ("sfx".equals(kind)) { + if (worker.isStrictRuntime()) { + log.error("event=local_asset_generation_failed project_id={} type={} model={} target={} reason=strict_runtime_required", + projectId, kind, model, target); + return Optional.empty(); + } + return fallbackTone(target, Math.max(0.5, Math.min(1.0, durationSeconds)), 880.0, 0.12, + projectId, kind); + } + if (worker.isStrictRuntime()) { + log.error("event=local_asset_generation_failed project_id={} type={} model={} target={} reason=strict_runtime_required", + projectId, kind, model, target); + return Optional.empty(); + } + writeSilence(target, Math.max(1.0, durationSeconds)); + log.warn("event=local_asset_generation_completed project_id={} type={} model=silence target={} strategy=silence", + projectId, kind, target); + return Optional.of(target); + } catch (IOException ex) { + log.warn("event=local_asset_generation_failed project_id={} type={} target={} error_type={} message={}", + projectId, kind, target, ex.getClass().getSimpleName(), ex.getMessage()); + return Optional.empty(); + } + } + + private boolean runWorker(List command, Path target) { + try { + log.info("event=local_asset_worker_invoked script={} target={}", worker.getScript(), target); + Process process = processExecutor.execute(command); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + int exitCode = process.waitFor(); + if (exitCode == 0 && Files.isRegularFile(target)) { + log.info("event=local_asset_worker_completed script={} target={} exit_code={}", + worker.getScript(), target, exitCode); + return true; + } + if (!output.isBlank()) { + log.warn("event=local_asset_worker_failed target={} output={}", target, output); + } + return false; + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + log.warn("event=local_asset_worker_failed target={} error_type={} message={}", + target, ex.getClass().getSimpleName(), ex.getMessage()); + return false; + } + } + + private boolean runSpeechFallback(String text, Path target) { + List> candidates = List.of( + List.of("say", "-o", target.toString(), text), + List.of("espeak", "-w", target.toString(), text) + ); + for (List command : candidates) { + if (command.isEmpty() || command.get(0).isBlank()) { + continue; + } + if (!isAvailable(command.get(0))) { + continue; + } + try { + log.info("event=local_asset_voiceover_fallback_started tool={} target={}", command.get(0), target); + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + if (process.waitFor() == 0 && Files.isRegularFile(target)) { + log.info("event=local_asset_voiceover_fallback_completed tool={} target={}", + command.get(0), target); + return true; + } + if (!output.isBlank()) { + log.warn("event=local_asset_voiceover_fallback_failed target={} output={}", target, output); + } + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + log.warn("event=local_asset_voiceover_fallback_failed target={} error_type={} message={}", + target, ex.getClass().getSimpleName(), ex.getMessage()); + } + } + return false; + } + + private Optional fallbackTone(Path target, double durationSeconds, double frequency, double volume, + String projectId, String kind) { + try { + log.info("event=local_asset_generation_started project_id={} type={} model=ffmpeg-sine target={} duration_seconds={} strategy=fallback-tone", + projectId, kind, target, durationSeconds); + Files.createDirectories(target.getParent()); + List command = List.of("ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "sine=frequency=" + (int) frequency + ":sample_rate=48000", + "-t", Double.toString(durationSeconds), "-af", "volume=" + volume, "-c:a", "pcm_s16le", + target.toString()); + Process process = processExecutor.execute(command); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + if (process.waitFor() == 0 && Files.isRegularFile(target)) { + log.info("event=local_asset_generation_completed project_id={} type={} model=ffmpeg-sine target={} strategy=fallback-tone", + projectId, kind, target); + return Optional.of(target); + } + if (!output.isBlank()) { + log.warn("event=local_asset_generation_failed project_id={} type={} target={} output={}", + projectId, kind, target, output); + } + return Optional.empty(); + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + log.warn("event=local_asset_generation_failed project_id={} type={} target={} error_type={} message={}", + projectId, kind, target, ex.getClass().getSimpleName(), ex.getMessage()); + return Optional.empty(); + } + } + + private void writeSilence(Path target, double durationSeconds) { + try { + log.info("event=local_asset_generation_started type=voiceover model=silence target={} duration_seconds={} strategy=silence", + target, durationSeconds); + Files.createDirectories(target.getParent()); + List command = List.of("ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "anullsrc=channel_layout=mono:sample_rate=24000", + "-t", Double.toString(durationSeconds), "-c:a", "pcm_s16le", target.toString()); + Process process = processExecutor.execute(command); + process.waitFor(); + log.info("event=local_asset_generation_completed type=voiceover model=silence target={} strategy=silence", + target); + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + } + } + + private String selectedModel(String kind) { + if ("music".equals(kind)) { + return worker.getMusicModel(); + } + if ("sfx".equals(kind)) { + return worker.getSfxModel(); + } + if ("voiceover".equals(kind)) { + return worker.getPiperModelPath().isBlank() ? selectedVoiceoverFallback() : worker.getPiperModelPath(); + } + return "unknown"; + } + + private String selectedVoiceoverFallback() { + return "piper-fallback"; + } + + private boolean hasAudibleSignal(Path audioFile) { + if (!Files.isRegularFile(audioFile)) { + return false; + } + try (AudioInputStream input = AudioSystem.getAudioInputStream(audioFile.toFile())) { + byte[] buffer = new byte[8192]; + int read; + while ((read = input.read(buffer)) != -1) { + for (int index = 0; index < read; index++) { + if (buffer[index] != 0) { + return true; + } + } + } + return false; + } catch (UnsupportedAudioFileException | IOException ex) { + log.warn("event=local_asset_voiceover_audibility_check_failed target={} error_type={} message={}", + audioFile, ex.getClass().getSimpleName(), ex.getMessage()); + return false; + } + } + + private boolean isAvailable(String executable) { + try { + Process process = new ProcessBuilder(executable, "--help").redirectErrorStream(true).start(); + process.waitFor(); + return true; + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + return false; + } + } + + private String sanitize(String value) { + return value == null ? "asset" : value.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9._-]+", "-"); + } + + @FunctionalInterface + interface ProcessExecutor { + Process execute(List command) throws IOException; + } +} diff --git a/src/main/java/org/example/videoclips/editing/LocalVoiceoverGenerator.java b/src/main/java/org/example/videoclips/editing/LocalVoiceoverGenerator.java new file mode 100644 index 0000000..848113e --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/LocalVoiceoverGenerator.java @@ -0,0 +1,43 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.stereotype.Component; + +import java.nio.file.Path; +import java.util.List; + +@Component +@ConditionalOnExpression("${video-clipping.editing.enabled:true} and " + + "'${video-clipping.editing.voiceover-provider:local}' == 'local'") +public class LocalVoiceoverGenerator implements VoiceoverGenerator { + + private final EditProjectStore store; + private final LocalAssetSynthesizer synthesizer; + + public LocalVoiceoverGenerator(EditProjectStore store, LocalAssetSynthesizer synthesizer) { + this.store = store; + this.synthesizer = synthesizer; + } + + @Override + public Path generateVoiceover(String projectId, List lines) { + Path script = store.projectDirectory(projectId).resolve("voiceover-script.txt"); + Path target = store.projectDirectory(projectId).resolve("audio").resolve("voiceover.wav"); + StringBuilder content = new StringBuilder("Voiceover script for project ").append(projectId).append('\n'); + for (VoiceoverLine line : lines) { + content.append("[%.3f-%.3f] %s%nDelivery: %s%n%n".formatted( + line.timelineStartSeconds(), line.timelineEndSeconds(), line.text(), line.delivery())); + } + try { + java.nio.file.Files.createDirectories(script.getParent()); + java.nio.file.Files.writeString(script, content.toString()); + } catch (java.io.IOException ex) { + throw new IllegalStateException("Unable to write voiceover script for project: " + projectId, ex); + } + synthesizer.synthesizeVoiceover(projectId, lines, target) + .orElseThrow(() -> new IllegalStateException("Unable to synthesize voiceover for project: " + + projectId)); + return target; + } +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3c50a4b..e449239 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -34,7 +34,7 @@ video-clipping: audio-sample-rate: ${VIDEO_EDITING_AUDIO_SAMPLE_RATE:48000} video-bitrate: ${VIDEO_EDITING_VIDEO_BITRATE:12000k} audio-bitrate: ${VIDEO_EDITING_AUDIO_BITRATE:192k} - voiceover-provider: ${VIDEO_EDITING_VOICEOVER_PROVIDER:noop} + voiceover-provider: ${VIDEO_EDITING_VOICEOVER_PROVIDER:local} loudness-target-i: ${VIDEO_EDITING_LOUDNESS_TARGET_I:-16.0} loudness-true-peak: ${VIDEO_EDITING_LOUDNESS_TRUE_PEAK:-1.5} loudness-range: ${VIDEO_EDITING_LOUDNESS_RANGE:11.0} @@ -56,9 +56,22 @@ video-clipping: local-cv-worker: auto-start: ${VIDEO_EDITING_LOCAL_CV_WORKER_AUTO_START:true} script: ${VIDEO_EDITING_LOCAL_CV_WORKER_SCRIPT:./tools/run_local_cv_worker.sh} - startup-wait-ms: ${VIDEO_EDITING_LOCAL_CV_WORKER_STARTUP_WAIT_MS:120000} + startup-wait-ms: ${VIDEO_EDITING_LOCAL_CV_WORKER_STARTUP_WAIT_MS:0} health-path: ${VIDEO_EDITING_LOCAL_CV_WORKER_HEALTH_PATH:/health} health-check-interval-ms: ${VIDEO_EDITING_LOCAL_CV_WORKER_HEALTH_CHECK_INTERVAL_MS:1000} + local-asset-worker: + auto-start: ${VIDEO_EDITING_LOCAL_ASSET_WORKER_AUTO_START:true} + strict-runtime: ${VIDEO_EDITING_LOCAL_ASSET_WORKER_STRICT_RUNTIME:true} + bootstrap-script: ${VIDEO_EDITING_LOCAL_ASSET_BOOTSTRAP_SCRIPT:./tools/run_local_asset_worker.sh} + script: ${VIDEO_EDITING_LOCAL_ASSET_WORKER_SCRIPT:./tools/local_asset_worker.py} + startup-wait-ms: ${VIDEO_EDITING_LOCAL_ASSET_WORKER_STARTUP_WAIT_MS:0} + health-path: ${VIDEO_EDITING_LOCAL_ASSET_WORKER_HEALTH_PATH:/health} + health-check-interval-ms: ${VIDEO_EDITING_LOCAL_ASSET_WORKER_HEALTH_CHECK_INTERVAL_MS:1000} + python-binary: ${VIDEO_EDITING_LOCAL_ASSET_PYTHON_BINARY:./.venv-local-asset/bin/python} + piper-binary: ${VIDEO_EDITING_LOCAL_ASSET_PIPER_BINARY:piper} + piper-model-path: ${VIDEO_EDITING_LOCAL_ASSET_PIPER_MODEL_PATH:} + music-model: ${VIDEO_EDITING_LOCAL_ASSET_MUSIC_MODEL:musicgen-small} + sfx-model: ${VIDEO_EDITING_LOCAL_ASSET_SFX_MODEL:audiogen-medium} local-director: enabled: ${VIDEO_EDITING_LOCAL_DIRECTOR_ENABLED:true} source-directory: ${VIDEO_EDITING_LOCAL_DIRECTOR_SOURCE_DIRECTORY:./input/editing/source} diff --git a/src/test/java/org/example/videoclips/api/EditProjectControllerTest.java b/src/test/java/org/example/videoclips/api/EditProjectControllerTest.java index 3040a5a..db34fb3 100644 --- a/src/test/java/org/example/videoclips/api/EditProjectControllerTest.java +++ b/src/test/java/org/example/videoclips/api/EditProjectControllerTest.java @@ -123,11 +123,15 @@ class EditProjectControllerTest { mockMvc.perform(put("/v1/edit-projects/{projectId}/plan", projectId) .contentType(MediaType.APPLICATION_JSON) .content(""" - {"style":"cinematic-porsche-promo","targetDurationSeconds":45, + {"style":"cinematic-porsche-promo","targetDurationSeconds":4, "decisions":[{"clipId":"clip-1","sourceStartSeconds":0,"sourceEndSeconds":4, "timelineStartSeconds":0,"timelineEndSeconds":4,"transitionIn":"cut", "transitionOut":"cut","playbackSpeed":1,"visualTreatment":"grade","reason":"hero"}], - "audioCues":[],"voiceover":[],"renderProfile":"mp4-h264-aac-1080p"} + "audioCues":[{"type":"music","assetKey":"cinematic-bed","timelineStartSeconds":0, + "timelineEndSeconds":4,"gainDb":-12,"notes":"bed"}],"voiceover":[], + "renderProfile":"mp4-h264-aac-1080p","overlays":[{"text":"Hero shot", + "timelineStartSeconds":0,"timelineEndSeconds":4,"placement":"lower_center_safe", + "animation":"fade_in","reason":"title"}]} """)) .andExpect(status().isOk()) .andExpect(jsonPath("$.projectId").value(projectId)); diff --git a/src/test/java/org/example/videoclips/api/VideoAssetControllerTest.java b/src/test/java/org/example/videoclips/api/VideoAssetControllerTest.java index b332377..975e861 100644 --- a/src/test/java/org/example/videoclips/api/VideoAssetControllerTest.java +++ b/src/test/java/org/example/videoclips/api/VideoAssetControllerTest.java @@ -10,7 +10,8 @@ import org.springframework.test.web.servlet.MvcResult; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.hamcrest.Matchers.isOneOf; +import static org.hamcrest.Matchers.anyOf; +import static org.hamcrest.Matchers.is; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.delete; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; @@ -106,7 +107,7 @@ class VideoAssetControllerTest { mockMvc.perform(get("/v1/clip-jobs/{jobId}", jobId)) .andExpect(status().isOk()) .andExpect(jsonPath("$.assetId").value(assetId)) - .andExpect(jsonPath("$.status", isOneOf("QUEUED", "RUNNING", "SUCCEEDED"))); + .andExpect(jsonPath("$.status", anyOf(is("QUEUED"), is("RUNNING"), is("SUCCEEDED")))); for (int attempt = 0; attempt < 20; attempt++) { String statusBody = mockMvc.perform(get("/v1/clip-jobs/{jobId}", jobId)) diff --git a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java index 3ba886c..df12297 100644 --- a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java +++ b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java @@ -38,7 +38,7 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().getAudioSampleRate()).isEqualTo(48000); assertThat(properties.getEditing().getVideoBitrate()).isEqualTo("12000k"); assertThat(properties.getEditing().getAudioBitrate()).isEqualTo("192k"); - assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("noop"); + assertThat(properties.getEditing().getVoiceoverProvider()).isEqualTo("local"); assertThat(properties.getEditing().getLoudnessTargetI()).isEqualTo(-16.0); assertThat(properties.getEditing().getLoudnessTruePeak()).isEqualTo(-1.5); assertThat(properties.getEditing().getLoudnessRange()).isEqualTo(11.0); @@ -67,6 +67,21 @@ class VideoClippingPropertiesTest { .isEqualTo("/health"); assertThat(properties.getEditing().getVisualAnalysis().getLocalCvWorker().getHealthCheckIntervalMs()) .isEqualTo(1000); + assertThat(properties.getEditing().getLocalAssetWorker().isAutoStart()).isFalse(); + assertThat(properties.getEditing().getLocalAssetWorker().getBootstrapScript()) + .isEqualTo("./tools/run_local_asset_worker.sh"); + assertThat(properties.getEditing().getLocalAssetWorker().getScript()) + .isEqualTo("./tools/local_asset_worker.py"); + assertThat(properties.getEditing().getLocalAssetWorker().getStartupWaitMs()).isZero(); + assertThat(properties.getEditing().getLocalAssetWorker().getHealthPath()).isEqualTo("/health"); + assertThat(properties.getEditing().getLocalAssetWorker().getHealthCheckIntervalMs()).isEqualTo(1000); + assertThat(properties.getEditing().getLocalAssetWorker().isStrictRuntime()).isFalse(); + assertThat(properties.getEditing().getLocalAssetWorker().getPythonBinary()) + .isEqualTo("./.venv-local-asset/bin/python"); + assertThat(properties.getEditing().getLocalAssetWorker().getPiperBinary()).isEqualTo("piper"); + assertThat(properties.getEditing().getLocalAssetWorker().getPiperModelPath()).isEmpty(); + assertThat(properties.getEditing().getLocalAssetWorker().getMusicModel()).isEqualTo("musicgen-small"); + assertThat(properties.getEditing().getLocalAssetWorker().getSfxModel()).isEqualTo("audiogen-medium"); assertThat(properties.getEditing().getLocalDirector().isEnabled()).isTrue(); assertThat(properties.getEditing().getLocalDirector().getSourceDirectory()) .isEqualTo("./input/editing/source"); @@ -114,6 +129,17 @@ class VideoClippingPropertiesTest { "video-clipping.editing.visual-analysis.local-cv-worker.startup-wait-ms=250", "video-clipping.editing.visual-analysis.local-cv-worker.health-path=/ready", "video-clipping.editing.visual-analysis.local-cv-worker.health-check-interval-ms=25", + "video-clipping.editing.local-asset-worker.auto-start=true", + "video-clipping.editing.local-asset-worker.bootstrap-script=/tmp/local-asset-bootstrap.sh", + "video-clipping.editing.local-asset-worker.script=/tmp/local-asset.py", + "video-clipping.editing.local-asset-worker.startup-wait-ms=250", + "video-clipping.editing.local-asset-worker.health-path=/asset-health", + "video-clipping.editing.local-asset-worker.health-check-interval-ms=50", + "video-clipping.editing.local-asset-worker.python-binary=/tmp/python", + "video-clipping.editing.local-asset-worker.piper-binary=/tmp/piper", + "video-clipping.editing.local-asset-worker.piper-model-path=/tmp/voice.onnx", + "video-clipping.editing.local-asset-worker.music-model=musicgen-large", + "video-clipping.editing.local-asset-worker.sfx-model=audiogen-large", "video-clipping.editing.local-director.enabled=false", "video-clipping.editing.local-director.source-directory=/tmp/source", "video-clipping.editing.local-director.poll-interval-ms=9000", @@ -162,6 +188,22 @@ class VideoClippingPropertiesTest { .isEqualTo("/ready"); assertThat(properties.getEditing().getVisualAnalysis().getLocalCvWorker().getHealthCheckIntervalMs()) .isEqualTo(25); + assertThat(properties.getEditing().getLocalAssetWorker().isAutoStart()).isTrue(); + assertThat(properties.getEditing().getLocalAssetWorker().getBootstrapScript()) + .isEqualTo("/tmp/local-asset-bootstrap.sh"); + assertThat(properties.getEditing().getLocalAssetWorker().getScript()).isEqualTo("/tmp/local-asset.py"); + assertThat(properties.getEditing().getLocalAssetWorker().getStartupWaitMs()).isEqualTo(250); + assertThat(properties.getEditing().getLocalAssetWorker().getHealthPath()).isEqualTo("/asset-health"); + assertThat(properties.getEditing().getLocalAssetWorker().getHealthCheckIntervalMs()).isEqualTo(50); + assertThat(properties.getEditing().getLocalAssetWorker().isStrictRuntime()).isFalse(); + assertThat(properties.getEditing().getLocalAssetWorker().getPythonBinary()).isEqualTo("/tmp/python"); + assertThat(properties.getEditing().getLocalAssetWorker().getPiperBinary()).isEqualTo("/tmp/piper"); + assertThat(properties.getEditing().getLocalAssetWorker().getPiperModelPath()) + .isEqualTo("/tmp/voice.onnx"); + assertThat(properties.getEditing().getLocalAssetWorker().getMusicModel()) + .isEqualTo("musicgen-large"); + assertThat(properties.getEditing().getLocalAssetWorker().getSfxModel()) + .isEqualTo("audiogen-large"); assertThat(properties.getEditing().getLocalDirector().isEnabled()).isFalse(); assertThat(properties.getEditing().getLocalDirector().getSourceDirectory()).isEqualTo("/tmp/source"); assertThat(properties.getEditing().getLocalDirector().getPollIntervalMs()).isEqualTo(9000); diff --git a/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java b/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java index 0dedfc2..f5f06e7 100644 --- a/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java +++ b/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java @@ -135,8 +135,13 @@ class CinematicEditingIntegrationTest { index * 2.0, (index + 1) * 2.0, "cut", "cut", 1, "cinematic grade", "integration sequence")) .toList(); + List audioCues = List.of(new AudioCue("music", "cinematic-bed", 0.0, + decisions.get(decisions.size() - 1).timelineEndSeconds(), -12.0, "integration score")); + List overlays = List.of(new TextOverlay("Cinematic highlight", 0.0, + decisions.get(0).timelineEndSeconds(), "lower_center_safe", "fade_in", "integration title")); return new EditPlan(projectId, "category-aware-cinematic-highlights", - decisions.get(decisions.size() - 1).timelineEndSeconds(), decisions, List.of(), List.of(), + decisions.get(decisions.size() - 1).timelineEndSeconds(), decisions, audioCues, List.of(), + overlays, "mp4-h264-aac-320p", "Generated integration edit"); } diff --git a/src/test/java/org/example/videoclips/editing/ContactSheetGeneratorTest.java b/src/test/java/org/example/videoclips/editing/ContactSheetGeneratorTest.java index 1b66653..d257b00 100644 --- a/src/test/java/org/example/videoclips/editing/ContactSheetGeneratorTest.java +++ b/src/test/java/org/example/videoclips/editing/ContactSheetGeneratorTest.java @@ -34,7 +34,7 @@ class ContactSheetGeneratorTest { assertCommandPair(command.get(), "-i", "./clips/clip_00001.mp4"); assertCommandPair(command.get(), "-vf", "fps=1/2,scale=320:-1,tile=4x4"); assertCommandPair(command.get(), "-frames:v", "1"); - assertThat(command.get().getLast()).endsWith("clip_00001.jpg"); + assertThat(command.get().get(command.get().size() - 1)).endsWith("clip_00001.jpg"); } @Test diff --git a/src/test/java/org/example/videoclips/editing/EditPlanServiceTest.java b/src/test/java/org/example/videoclips/editing/EditPlanServiceTest.java index 6cb09ab..4a22c05 100644 --- a/src/test/java/org/example/videoclips/editing/EditPlanServiceTest.java +++ b/src/test/java/org/example/videoclips/editing/EditPlanServiceTest.java @@ -32,9 +32,10 @@ class EditPlanServiceTest { new ClipAnalysis("clip-1", "clip.mp4", 8, "h264", "aac", 1920, 1080, 30, List.of(), null, null, 0, 0, 0)), List.of(), Instant.now())); EditPlanService service = new EditPlanService(store, projects, new EditPlanValidator(store)); - SaveEditPlanRequest request = new SaveEditPlanRequest("cinematic-porsche-promo", 60, + List audioCues = List.of(new AudioCue("music", "cinematic-bed", 0, 4, -12, "bed")); + SaveEditPlanRequest request = new SaveEditPlanRequest("cinematic-porsche-promo", 4, List.of(new EditDecision("clip-1", 0, 4, 0, 4, "cut", "cut", 1, "grade", "hero")), - List.of(), List.of(), "mp4-h264-aac-1080p", "summary"); + audioCues, List.of(), List.of(), "mp4-h264-aac-1080p", "summary"); EditPlan saved = service.save("project", request); diff --git a/src/test/java/org/example/videoclips/editing/EditProjectAnalyzerTest.java b/src/test/java/org/example/videoclips/editing/EditProjectAnalyzerTest.java index 2310b1c..b71fbc1 100644 --- a/src/test/java/org/example/videoclips/editing/EditProjectAnalyzerTest.java +++ b/src/test/java/org/example/videoclips/editing/EditProjectAnalyzerTest.java @@ -53,9 +53,9 @@ class EditProjectAnalyzerTest { assertThat(analysis.createdAt()).isEqualTo(Instant.parse("2026-07-10T10:00:00Z")); assertThat(analysis.errors()).isEmpty(); assertThat(analysis.clips()).hasSize(1); - assertThat(analysis.clips().getFirst().thumbnails()).containsExactly("thumb-1.jpg"); - assertThat(analysis.clips().getFirst().contactSheet()).isEqualTo("contact-sheet.jpg"); - assertThat(analysis.clips().getFirst().proxyPath()).isEqualTo("proxy.mp4"); + assertThat(analysis.clips().get(0).thumbnails()).containsExactly("thumb-1.jpg"); + assertThat(analysis.clips().get(0).contactSheet()).isEqualTo("contact-sheet.jpg"); + assertThat(analysis.clips().get(0).proxyPath()).isEqualTo("proxy.mp4"); assertThat(Files.exists(tempDir.resolve("projects/project-1/analysis.json"))).isTrue(); assertThat(analyzer.readAnalysis("project-1")).isEqualTo(analysis); } diff --git a/src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java b/src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java index fb3c2f7..16eae47 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java @@ -14,7 +14,9 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; import static org.mockito.Mockito.when; +import static org.mockito.Mockito.verify; class HighlightDirectorFlowServiceTest { @@ -66,6 +68,42 @@ class HighlightDirectorFlowServiceTest { .isEqualTo(HighlightProjectStatus.RENDERED); } + @Test + void waitsForAssetsInsteadOfRenderingWhenLocalWorkerLeavesPendingRequests() throws Exception { + VideoClippingProperties properties = properties(); + ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); + FileSystemHighlightProjectStore store = new FileSystemHighlightProjectStore(properties, mapper); + HighlightProject project = project(); + store.createProject(project, manifest()); + store.writeJson(project.id(), "analysis/source-analysis.json", sourceAnalysis()); + store.writeJson(project.id(), "director/edit-plan.json", directorPlan()); + Path source = store.sourceDirectory(project.id()).resolve("source.mp4"); + createDummyFile(source); + HighlightVisualEffectsStage visualEffectsStage = mock(HighlightVisualEffectsStage.class); + HighlightAssetPreparationService assetPreparationService = mock(HighlightAssetPreparationService.class); + HighlightAssetPreparationService.HighlightAssetPreparationResult assetResult = + new HighlightAssetPreparationService.HighlightAssetPreparationResult( + store.highlightsDirectory(project.id()).resolve("highlight_001"), + store.highlightsDirectory(project.id()).resolve("highlight_001").resolve("assets"), + List.of("requests/voiceover.json"), List.of()); + when(assetPreparationService.prepare(anyString(), any(), any(), any())).thenReturn(assetResult); + HighlightLocalAssetWorker assetWorker = mock(HighlightLocalAssetWorker.class); + when(assetWorker.process(anyString(), any(), any())).thenReturn( + new HighlightLocalAssetWorker.HighlightAssetWorkerResult(project.id(), "highlight_001", + List.of(), List.of("requests/voiceover.json"))); + HighlightFfmpegRenderer renderer = mock(HighlightFfmpegRenderer.class); + HighlightDirectorFlowService service = new HighlightDirectorFlowService(properties, mapper, store, + visualEffectsStage, assetPreparationService, assetWorker, renderer); + + HighlightDirectorFlowService.HighlightFlowResult result = service.process(project.id(), 1); + + assertThat(result.reason()).isEqualTo("assets_pending"); + assertThat(result.finalOutputs()).isEmpty(); + verify(renderer, never()).render(anyString(), any(), any()); + assertThat(store.readJson(project.id(), "project.json", HighlightProject.class).status()) + .isEqualTo(HighlightProjectStatus.WAITING_FOR_DIRECTOR); + } + private VideoClippingProperties properties() { VideoClippingProperties properties = new VideoClippingProperties(); properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); diff --git a/src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java b/src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java index 694d7e0..76bbcc9 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java @@ -52,7 +52,8 @@ class HighlightLocalAssetWorkerTest { "premium cinematic grade", "low pulse", "whoosh", List.of("A clean first look."), List.of("Pure presence"), "open with presence"); prep.prepare("project-1", project, highlight, ContentCategory.GENERIC_VLOG); - HighlightLocalAssetWorker worker = new HighlightLocalAssetWorker(store, assetProvider, assetLibrary, mapper); + HighlightLocalAssetWorker worker = new HighlightLocalAssetWorker(store, assetProvider, assetLibrary, + new LocalAssetSynthesizer(properties), mapper); HighlightLocalAssetWorker.HighlightAssetWorkerResult result = worker.process("project-1", highlight, ContentCategory.GENERIC_VLOG); diff --git a/src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java b/src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java index 810212f..edf4cc2 100644 --- a/src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java +++ b/src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java @@ -18,7 +18,7 @@ class LocalAssetGenerationStageTest { Path tempDir; @Test - void reusesExistingSharedAssetsAndWritesRequestsForMissingVoiceover() throws Exception { + void reusesExistingSharedAssetsAndSynthesizesMissingVoiceover() throws Exception { VideoClippingProperties properties = properties(); Files.createDirectories(tempDir.resolve("shared/music")); Files.createDirectories(tempDir.resolve("shared/sfx")); @@ -36,6 +36,7 @@ class LocalAssetGenerationStageTest { LocalAssetGenerationStage stage = new LocalAssetGenerationStage(properties, store, new LocalEditAssetProvider(properties), new LocalEditAssetLibrary(new LocalEditAssetProvider(properties)), + new LocalAssetSynthesizer(properties), new ObjectMapper().findAndRegisterModules()); EditPlan plan = new EditPlan("project", "cinematic-porsche-promo", 4, @@ -55,13 +56,13 @@ class LocalAssetGenerationStageTest { assertThat(result.readyForRender()).isTrue(); assertThat(tempDir.resolve("projects/project/audio/music.wav")).exists(); assertThat(tempDir.resolve("projects/project/audio/sfx/whoosh.wav")).exists(); + assertThat(tempDir.resolve("projects/project/audio/voiceover.wav")).exists(); assertThat(tempDir.resolve("projects/project/assets/asset-generation-manifest.json")).exists(); assertThat(tempDir.resolve("projects/project/assets/generated-assets.json")).exists(); - assertThat(tempDir.resolve("projects/project/assets/requests/voiceover")).exists(); } @Test - void writesBlockingRequestsWhenSfxIsMissing() throws Exception { + void synthesizesMissingSfxAndKeepsProjectReadyForRender() throws Exception { VideoClippingProperties properties = properties(); properties.getEditing().getAssets().setMusicFolder(tempDir.resolve("shared/music").toString()); properties.getEditing().getAssets().setSfxFolder(tempDir.resolve("shared/sfx").toString()); @@ -73,6 +74,7 @@ class LocalAssetGenerationStageTest { LocalAssetGenerationStage stage = new LocalAssetGenerationStage(properties, store, new LocalEditAssetProvider(properties), new LocalEditAssetLibrary(new LocalEditAssetProvider(properties)), + new LocalAssetSynthesizer(properties), new ObjectMapper().findAndRegisterModules()); EditPlan plan = new EditPlan("project", "cinematic-porsche-promo", 4, @@ -82,8 +84,8 @@ class LocalAssetGenerationStageTest { AssetGenerationResult result = stage.prepare("project", plan); - assertThat(result.readyForRender()).isFalse(); - assertThat(tempDir.resolve("projects/project/assets/requests/sfx")).exists(); + assertThat(result.readyForRender()).isTrue(); + assertThat(tempDir.resolve("projects/project/audio/sfx/engine-hit.wav")).exists(); } private VideoClippingProperties properties() { diff --git a/src/test/java/org/example/videoclips/editing/LocalDirectorSchedulerTest.java b/src/test/java/org/example/videoclips/editing/LocalDirectorSchedulerTest.java index 8ac8df4..97e080f 100644 --- a/src/test/java/org/example/videoclips/editing/LocalDirectorSchedulerTest.java +++ b/src/test/java/org/example/videoclips/editing/LocalDirectorSchedulerTest.java @@ -80,7 +80,7 @@ class LocalDirectorSchedulerTest { assertThat(project.status()).isEqualTo(EditProjectStatus.WAITING_FOR_DIRECTOR); assertThat(project.inputDirectory()).isEqualTo(tempDir.resolve("processed/01-porsche").toString()); EditProjectAnalysis analysis = store.readJson("01-porsche", "analysis.json", EditProjectAnalysis.class); - assertThat(analysis.clips().getFirst().sourcePath()) + assertThat(analysis.clips().get(0).sourcePath()) .isEqualTo(tempDir.resolve("processed/01-porsche/clip_00001.mp4").toString()); assertThat(firstClip).doesNotExist(); } diff --git a/src/test/java/org/example/videoclips/editing/ThumbnailExtractorTest.java b/src/test/java/org/example/videoclips/editing/ThumbnailExtractorTest.java index dec8843..753ecff 100644 --- a/src/test/java/org/example/videoclips/editing/ThumbnailExtractorTest.java +++ b/src/test/java/org/example/videoclips/editing/ThumbnailExtractorTest.java @@ -40,7 +40,7 @@ class ThumbnailExtractorTest { assertCommandPair(commands.get(1), "-ss", "4.000"); assertCommandPair(commands.get(2), "-ss", "6.000"); assertCommandPair(commands.get(0), "-i", "./clips/clip_00001.mp4"); - assertThat(commands.get(0).getLast()).endsWith("clip_00001_0001.jpg"); + assertThat(commands.get(0).get(commands.get(0).size() - 1)).endsWith("clip_00001_0001.jpg"); } @Test diff --git a/src/test/java/org/example/videoclips/folder/FolderFfmpegClipperTest.java b/src/test/java/org/example/videoclips/folder/FolderFfmpegClipperTest.java index e75233a..e00c0dd 100644 --- a/src/test/java/org/example/videoclips/folder/FolderFfmpegClipperTest.java +++ b/src/test/java/org/example/videoclips/folder/FolderFfmpegClipperTest.java @@ -24,7 +24,7 @@ class FolderFfmpegClipperTest { AtomicReference> command = new AtomicReference<>(); FolderFfmpegClipper clipper = clipper(arguments -> { command.set(arguments); - Path outputDirectory = Path.of(arguments.getLast()).getParent(); + Path outputDirectory = Path.of(arguments.get(arguments.size() - 1)).getParent(); Files.writeString(outputDirectory.resolve("clip_00000.mp4"), "clip"); Files.writeString(outputDirectory.resolve("clip_00001.mp4"), "clip"); Files.writeString(outputDirectory.resolve("ignored.txt"), "ignored"); @@ -38,7 +38,7 @@ class FolderFfmpegClipperTest { assertEquals(2, result.clipCount()); assertCommandPair(command.get(), "-c", "copy"); assertCommandPair(command.get(), "-segment_time", "8"); - assertTrue(command.get().getLast().endsWith("clip_%05d.mp4")); + assertTrue(command.get().get(command.get().size() - 1).endsWith("clip_%05d.mp4")); } @Test @@ -48,7 +48,7 @@ class FolderFfmpegClipperTest { properties.getFolderScheduler().setPreserveInputQuality(false); FolderFfmpegClipper clipper = new FolderFfmpegClipper(properties, arguments -> { command.set(arguments); - Path outputDirectory = Path.of(arguments.getLast()).getParent(); + Path outputDirectory = Path.of(arguments.get(arguments.size() - 1)).getParent(); Files.writeString(outputDirectory.resolve("clip_00000.mp4"), "clip"); return new FolderFfmpegClipper.ProcessResult(0, "ok"); }); @@ -66,7 +66,7 @@ class FolderFfmpegClipperTest { Files.createDirectory(tempDir.resolve("output/movie")); Files.createDirectory(tempDir.resolve("output/movie-1")); FolderFfmpegClipper clipper = clipper(arguments -> { - Path outputDirectory = Path.of(arguments.getLast()).getParent(); + Path outputDirectory = Path.of(arguments.get(arguments.size() - 1)).getParent(); Files.writeString(outputDirectory.resolve("clip_00000.mp4"), "clip"); return new FolderFfmpegClipper.ProcessResult(0, "ok"); }); @@ -77,7 +77,7 @@ class FolderFfmpegClipperTest { @Test void usesWholeFilenameAsStemWhenThereIsNoExtension() throws Exception { FolderFfmpegClipper clipper = clipper(arguments -> { - Path outputDirectory = Path.of(arguments.getLast()).getParent(); + Path outputDirectory = Path.of(arguments.get(arguments.size() - 1)).getParent(); Files.writeString(outputDirectory.resolve("clip_00000.mp4"), "clip"); return new FolderFfmpegClipper.ProcessResult(0, "ok"); }); @@ -96,7 +96,7 @@ class FolderFfmpegClipperTest { @Test void reportsGeneratedClipListingFailure() { assertThrows(IllegalStateException.class, () -> clipper(arguments -> { - Files.delete(Path.of(arguments.getLast()).getParent()); + Files.delete(Path.of(arguments.get(arguments.size() - 1)).getParent()); return new FolderFfmpegClipper.ProcessResult(0, "ok"); }).clip(Path.of("deleted-output.mp4"))); } diff --git a/src/test/java/org/example/videoclips/folder/FolderVideoValidatorTest.java b/src/test/java/org/example/videoclips/folder/FolderVideoValidatorTest.java index 634c434..99d6ee6 100644 --- a/src/test/java/org/example/videoclips/folder/FolderVideoValidatorTest.java +++ b/src/test/java/org/example/videoclips/folder/FolderVideoValidatorTest.java @@ -32,8 +32,8 @@ class FolderVideoValidatorTest { assertTrue(result.valid()); assertEquals("valid video", result.reason()); - assertEquals("ffprobe", command.get().getFirst()); - assertEquals("movie.MP4", command.get().getLast()); + assertEquals("ffprobe", command.get().get(0)); + assertEquals("movie.MP4", command.get().get(command.get().size() - 1)); } @Test diff --git a/src/test/resources/application.properties b/src/test/resources/application.properties index 71f756d..f83bb4e 100644 --- a/src/test/resources/application.properties +++ b/src/test/resources/application.properties @@ -2,3 +2,5 @@ video-clipping.folder-scheduler.enabled=false video-clipping.editing.local-director.enabled=false video-clipping.editing.highlight-scheduler.enabled=false video-clipping.editing.visual-analysis.local-cv-worker.auto-start=false +video-clipping.editing.local-asset-worker.auto-start=false +video-clipping.editing.local-asset-worker.strict-runtime=false diff --git a/tmp/ffmpeg-input/uploads/upl_5ad28940dd95/source b/tmp/ffmpeg-input/uploads/upl_5ad28940dd95/source deleted file mode 100644 index cb3d12a..0000000 --- a/tmp/ffmpeg-input/uploads/upl_5ad28940dd95/source +++ /dev/null @@ -1 +0,0 @@ -stub-video-content \ No newline at end of file diff --git a/tmp/ffmpeg-input/uploads/upl_b96dfa0dedbe/source b/tmp/ffmpeg-input/uploads/upl_b96dfa0dedbe/source deleted file mode 100644 index cb3d12a..0000000 --- a/tmp/ffmpeg-input/uploads/upl_b96dfa0dedbe/source +++ /dev/null @@ -1 +0,0 @@ -stub-video-content \ No newline at end of file diff --git a/tmp/ffmpeg-input/uploads/upl_d0599ff63186/source b/tmp/ffmpeg-input/uploads/upl_d0599ff63186/source deleted file mode 100644 index cb3d12a..0000000 --- a/tmp/ffmpeg-input/uploads/upl_d0599ff63186/source +++ /dev/null @@ -1 +0,0 @@ -stub-video-content \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_13022b79d037/0.mp4 b/tmp/in-memory-storage/clips/job_13022b79d037/0.mp4 deleted file mode 100644 index d20e5c8..0000000 --- a/tmp/in-memory-storage/clips/job_13022b79d037/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_55da9cbe7120 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_13022b79d037/1.mp4 b/tmp/in-memory-storage/clips/job_13022b79d037/1.mp4 deleted file mode 100644 index 4533c85..0000000 --- a/tmp/in-memory-storage/clips/job_13022b79d037/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_4f379a7d5941 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_13022b79d037/2.mp4 b/tmp/in-memory-storage/clips/job_13022b79d037/2.mp4 deleted file mode 100644 index 7ecd43c..0000000 --- a/tmp/in-memory-storage/clips/job_13022b79d037/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_12f6257786bf \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_30a3a3959622/0.mp4 b/tmp/in-memory-storage/clips/job_30a3a3959622/0.mp4 deleted file mode 100644 index ab74bad..0000000 --- a/tmp/in-memory-storage/clips/job_30a3a3959622/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_3db7ae28dd6e \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_30a3a3959622/1.mp4 b/tmp/in-memory-storage/clips/job_30a3a3959622/1.mp4 deleted file mode 100644 index 5fa6d13..0000000 --- a/tmp/in-memory-storage/clips/job_30a3a3959622/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_a87a659135b7 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_30a3a3959622/2.mp4 b/tmp/in-memory-storage/clips/job_30a3a3959622/2.mp4 deleted file mode 100644 index 0c8fbd2..0000000 --- a/tmp/in-memory-storage/clips/job_30a3a3959622/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_f6d9f025b976 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_5e48ca647258/0.mp4 b/tmp/in-memory-storage/clips/job_5e48ca647258/0.mp4 deleted file mode 100644 index d867dd6..0000000 --- a/tmp/in-memory-storage/clips/job_5e48ca647258/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_4c3db1c00f58 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_5e48ca647258/1.mp4 b/tmp/in-memory-storage/clips/job_5e48ca647258/1.mp4 deleted file mode 100644 index c231063..0000000 --- a/tmp/in-memory-storage/clips/job_5e48ca647258/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_40f5c4bd1569 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_5e48ca647258/2.mp4 b/tmp/in-memory-storage/clips/job_5e48ca647258/2.mp4 deleted file mode 100644 index 1789b11..0000000 --- a/tmp/in-memory-storage/clips/job_5e48ca647258/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_9bc569665c0f \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_92b86e138cb9/0.mp4 b/tmp/in-memory-storage/clips/job_92b86e138cb9/0.mp4 deleted file mode 100644 index 6b36211..0000000 --- a/tmp/in-memory-storage/clips/job_92b86e138cb9/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_10c8dc32bebf \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_92b86e138cb9/1.mp4 b/tmp/in-memory-storage/clips/job_92b86e138cb9/1.mp4 deleted file mode 100644 index 8dcb3f8..0000000 --- a/tmp/in-memory-storage/clips/job_92b86e138cb9/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_04750c742dae \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_92b86e138cb9/2.mp4 b/tmp/in-memory-storage/clips/job_92b86e138cb9/2.mp4 deleted file mode 100644 index c139735..0000000 --- a/tmp/in-memory-storage/clips/job_92b86e138cb9/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_9c0062d6e046 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_af397ecc03e3/0.mp4 b/tmp/in-memory-storage/clips/job_af397ecc03e3/0.mp4 deleted file mode 100644 index d1d19b4..0000000 --- a/tmp/in-memory-storage/clips/job_af397ecc03e3/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_f205e44bd8cb \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_af397ecc03e3/1.mp4 b/tmp/in-memory-storage/clips/job_af397ecc03e3/1.mp4 deleted file mode 100644 index d0ab802..0000000 --- a/tmp/in-memory-storage/clips/job_af397ecc03e3/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_0f9504972838 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_af397ecc03e3/2.mp4 b/tmp/in-memory-storage/clips/job_af397ecc03e3/2.mp4 deleted file mode 100644 index 736651b..0000000 --- a/tmp/in-memory-storage/clips/job_af397ecc03e3/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_26599ec473ec \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_b740f1cb4a37/0.mp4 b/tmp/in-memory-storage/clips/job_b740f1cb4a37/0.mp4 deleted file mode 100644 index e92d80d..0000000 --- a/tmp/in-memory-storage/clips/job_b740f1cb4a37/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_247296e47c86 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_b740f1cb4a37/1.mp4 b/tmp/in-memory-storage/clips/job_b740f1cb4a37/1.mp4 deleted file mode 100644 index 949bcb4..0000000 --- a/tmp/in-memory-storage/clips/job_b740f1cb4a37/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_30378c1a8d2c \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_b740f1cb4a37/2.mp4 b/tmp/in-memory-storage/clips/job_b740f1cb4a37/2.mp4 deleted file mode 100644 index 08a11cf..0000000 --- a/tmp/in-memory-storage/clips/job_b740f1cb4a37/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_f859bc77404b \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_b8dd304c03f9/0.mp4 b/tmp/in-memory-storage/clips/job_b8dd304c03f9/0.mp4 deleted file mode 100644 index 509082f..0000000 --- a/tmp/in-memory-storage/clips/job_b8dd304c03f9/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_8469868182d1 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_b8dd304c03f9/1.mp4 b/tmp/in-memory-storage/clips/job_b8dd304c03f9/1.mp4 deleted file mode 100644 index 094443d..0000000 --- a/tmp/in-memory-storage/clips/job_b8dd304c03f9/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_dedf0bdd8aec \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_b8dd304c03f9/2.mp4 b/tmp/in-memory-storage/clips/job_b8dd304c03f9/2.mp4 deleted file mode 100644 index dad4cd6..0000000 --- a/tmp/in-memory-storage/clips/job_b8dd304c03f9/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_937f3ff7df71 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_c8c81ec6507f/0.mp4 b/tmp/in-memory-storage/clips/job_c8c81ec6507f/0.mp4 deleted file mode 100644 index 314534f..0000000 --- a/tmp/in-memory-storage/clips/job_c8c81ec6507f/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_6610d92757c1 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_c8c81ec6507f/1.mp4 b/tmp/in-memory-storage/clips/job_c8c81ec6507f/1.mp4 deleted file mode 100644 index 22b25d7..0000000 --- a/tmp/in-memory-storage/clips/job_c8c81ec6507f/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_f6612e1119d5 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_c8c81ec6507f/2.mp4 b/tmp/in-memory-storage/clips/job_c8c81ec6507f/2.mp4 deleted file mode 100644 index 9c0d3cc..0000000 --- a/tmp/in-memory-storage/clips/job_c8c81ec6507f/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_b68e31fe0da6 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/0.mp4 b/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/0.mp4 deleted file mode 100644 index fe5a375..0000000 --- a/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_a58ccb424fa0 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/1.mp4 b/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/1.mp4 deleted file mode 100644 index 8210c38..0000000 --- a/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_59a1c0bee8ed \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/2.mp4 b/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/2.mp4 deleted file mode 100644 index fb9dd54..0000000 --- a/tmp/in-memory-storage/clips/job_de0b3d0c1d8a/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_aeaf1155bc22 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_e0204b61aab7/0.mp4 b/tmp/in-memory-storage/clips/job_e0204b61aab7/0.mp4 deleted file mode 100644 index 37dc36c..0000000 --- a/tmp/in-memory-storage/clips/job_e0204b61aab7/0.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_5806cd2fc91e \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_e0204b61aab7/1.mp4 b/tmp/in-memory-storage/clips/job_e0204b61aab7/1.mp4 deleted file mode 100644 index c560785..0000000 --- a/tmp/in-memory-storage/clips/job_e0204b61aab7/1.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_297c6a9ab956 \ No newline at end of file diff --git a/tmp/in-memory-storage/clips/job_e0204b61aab7/2.mp4 b/tmp/in-memory-storage/clips/job_e0204b61aab7/2.mp4 deleted file mode 100644 index 4bfb8c6..0000000 --- a/tmp/in-memory-storage/clips/job_e0204b61aab7/2.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_8a000ebfe018 \ No newline at end of file diff --git a/tmp/stub-output/job_5e48ca647258/clip_00000.mp4 b/tmp/stub-output/job_5e48ca647258/clip_00000.mp4 deleted file mode 100644 index d867dd6..0000000 --- a/tmp/stub-output/job_5e48ca647258/clip_00000.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_4c3db1c00f58 \ No newline at end of file diff --git a/tmp/stub-output/job_5e48ca647258/clip_00001.mp4 b/tmp/stub-output/job_5e48ca647258/clip_00001.mp4 deleted file mode 100644 index c231063..0000000 --- a/tmp/stub-output/job_5e48ca647258/clip_00001.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_40f5c4bd1569 \ No newline at end of file diff --git a/tmp/stub-output/job_5e48ca647258/clip_00002.mp4 b/tmp/stub-output/job_5e48ca647258/clip_00002.mp4 deleted file mode 100644 index 1789b11..0000000 --- a/tmp/stub-output/job_5e48ca647258/clip_00002.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_9bc569665c0f \ No newline at end of file diff --git a/tmp/stub-output/job_b740f1cb4a37/clip_00000.mp4 b/tmp/stub-output/job_b740f1cb4a37/clip_00000.mp4 deleted file mode 100644 index e92d80d..0000000 --- a/tmp/stub-output/job_b740f1cb4a37/clip_00000.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_247296e47c86 \ No newline at end of file diff --git a/tmp/stub-output/job_b740f1cb4a37/clip_00001.mp4 b/tmp/stub-output/job_b740f1cb4a37/clip_00001.mp4 deleted file mode 100644 index 949bcb4..0000000 --- a/tmp/stub-output/job_b740f1cb4a37/clip_00001.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_30378c1a8d2c \ No newline at end of file diff --git a/tmp/stub-output/job_b740f1cb4a37/clip_00002.mp4 b/tmp/stub-output/job_b740f1cb4a37/clip_00002.mp4 deleted file mode 100644 index 08a11cf..0000000 --- a/tmp/stub-output/job_b740f1cb4a37/clip_00002.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_f859bc77404b \ No newline at end of file diff --git a/tmp/stub-output/job_b8dd304c03f9/clip_00000.mp4 b/tmp/stub-output/job_b8dd304c03f9/clip_00000.mp4 deleted file mode 100644 index 509082f..0000000 --- a/tmp/stub-output/job_b8dd304c03f9/clip_00000.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_8469868182d1 \ No newline at end of file diff --git a/tmp/stub-output/job_b8dd304c03f9/clip_00001.mp4 b/tmp/stub-output/job_b8dd304c03f9/clip_00001.mp4 deleted file mode 100644 index 094443d..0000000 --- a/tmp/stub-output/job_b8dd304c03f9/clip_00001.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_dedf0bdd8aec \ No newline at end of file diff --git a/tmp/stub-output/job_b8dd304c03f9/clip_00002.mp4 b/tmp/stub-output/job_b8dd304c03f9/clip_00002.mp4 deleted file mode 100644 index dad4cd6..0000000 --- a/tmp/stub-output/job_b8dd304c03f9/clip_00002.mp4 +++ /dev/null @@ -1 +0,0 @@ -stub-clip-content-clip_937f3ff7df71 \ No newline at end of file diff --git a/tools/local_asset_requirements.txt b/tools/local_asset_requirements.txt new file mode 100644 index 0000000..8eae89e --- /dev/null +++ b/tools/local_asset_requirements.txt @@ -0,0 +1,4 @@ +torch +audiocraft +soundfile +numpy diff --git a/tools/local_asset_worker.py b/tools/local_asset_worker.py new file mode 100644 index 0000000..fd46fe6 --- /dev/null +++ b/tools/local_asset_worker.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import math +import os +import shutil +import subprocess +import wave +from array import array +from pathlib import Path + +try: + import soundfile as sf # type: ignore +except Exception: # pragma: no cover - optional runtime dependency + sf = None + +try: + import torch # type: ignore +except Exception: # pragma: no cover - optional runtime dependency + torch = None + +try: + import numpy as np # type: ignore +except Exception: # pragma: no cover - optional runtime dependency + np = None + + +def main() -> int: + parser = argparse.ArgumentParser(description="Local cinematic asset synthesizer") + parser.add_argument("kind", choices=["voiceover", "music", "sfx"]) + parser.add_argument("--prompt-file", required=True) + parser.add_argument("--output", required=True) + parser.add_argument("--duration", type=float, default=8.0) + parser.add_argument("--model", default="") + parser.add_argument("--piper-binary", default=os.getenv("LOCAL_ASSET_PIPER_BINARY", "piper")) + args = parser.parse_args() + + prompt = Path(args.prompt_file).read_text(encoding="utf-8").strip() + output = Path(args.output) + output.parent.mkdir(parents=True, exist_ok=True) + + if args.kind == "voiceover": + if synthesize_voiceover(prompt, output, args.piper_binary, args.model): + return 0 + write_silence(output, max(1.0, args.duration)) + return 0 + + if args.kind == "music": + if synthesize_audiocraft("music", prompt, output, args.duration, args.model): + return 0 + write_fallback_tone(output, max(4.0, args.duration), 110.0, 0.02) + return 0 + + if synthesize_audiocraft("sfx", prompt, output, args.duration, args.model): + return 0 + write_fallback_tone(output, max(0.5, min(1.0, args.duration)), 880.0, 0.12) + return 0 + + +def synthesize_voiceover(prompt: str, output: Path, piper_binary: str, model: str) -> bool: + model_path = model.strip() + if model_path and shutil.which(piper_binary): + command = [piper_binary, "--model", model_path, "--output_file", str(output)] + try: + process = subprocess.run( + command, + input=prompt.encode("utf-8"), + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=False, + ) + return process.returncode == 0 and output.is_file() + except Exception: + return False + + say = shutil.which("say") + if say: + process = subprocess.run([say, "-o", str(output), prompt], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return process.returncode == 0 and output.is_file() + + espeak = shutil.which("espeak") + if espeak: + process = subprocess.run([espeak, "-w", str(output), prompt], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return process.returncode == 0 and output.is_file() + + return False + + +def synthesize_audiocraft(kind: str, prompt: str, output: Path, duration: float, model_name: str) -> bool: + if torch is None: + return False + try: + if kind == "music": + from audiocraft.models import MusicGen # type: ignore + + model = MusicGen.get_pretrained(normalize_model_name(model_name, "musicgen-small")) + model.set_generation_params(duration=max(1.0, float(duration))) + audio = model.generate([prompt])[0] + write_tensor_audio(audio, output, getattr(model, "sample_rate", 32000)) + return output.is_file() + + from audiocraft.models import AudioGen # type: ignore + + model = AudioGen.get_pretrained(normalize_model_name(model_name, "audiogen-medium")) + model.set_generation_params(duration=max(0.5, float(duration))) + audio = model.generate([prompt])[0] + write_tensor_audio(audio, output, getattr(model, "sample_rate", 32000)) + return output.is_file() + except Exception: + return False + + +def normalize_model_name(model_name: str, default_name: str) -> str: + value = model_name.strip() if model_name else default_name + if "/" in value: + return value + return f"facebook/{value}" + + +def write_tensor_audio(audio, output: Path, sample_rate: int) -> None: + if np is None: + raise RuntimeError("numpy is required to write model-generated audio") + if hasattr(audio, "detach"): + audio = audio.detach().cpu().numpy() + elif torch is not None and isinstance(audio, torch.Tensor): + audio = audio.cpu().numpy() + audio = np.asarray(audio) + if audio.ndim == 3: + audio = audio[0] + if audio.ndim == 2 and audio.shape[0] < audio.shape[1]: + audio = audio.transpose(1, 0) + if sf is not None: + sf.write(str(output), audio, sample_rate) + return + write_wave_file(output, audio, sample_rate) + + +def write_fallback_tone(output: Path, duration: float, frequency: float, volume: float) -> None: + sample_rate = 48000 + total_samples = int(sample_rate * duration) + frames = array( + "h", + [ + int(max(-1.0, min(1.0, math.sin(2.0 * math.pi * frequency * (i / sample_rate))) * volume) * 32767.0) + for i in range(total_samples) + ], + ) + write_wave_file(output, frames, sample_rate, channels=1) + + +def write_silence(output: Path, duration: float) -> None: + sample_rate = 24000 + total_samples = int(sample_rate * duration) + frames = array("h", [0] * total_samples) + write_wave_file(output, frames, sample_rate, channels=1) + + +def write_wave_file(output: Path, waveform, sample_rate: int, channels: int | None = None) -> None: + if hasattr(waveform, "ndim") and np is not None: + waveform = np.asarray(waveform) + if waveform.ndim == 1: + waveform = waveform[:, None] + waveform = np.clip(waveform, -1.0, 1.0) + pcm = (waveform * 32767.0).astype(np.int16) + channels = pcm.shape[1] + frames = array("h", pcm.reshape(-1).tolist()) + else: + frames = waveform + if channels is None: + channels = 1 + with wave.open(str(output), "wb") as wav: + wav.setnchannels(channels) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + wav.writeframes(frames.tobytes()) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/run_local_asset_worker.sh b/tools/run_local_asset_worker.sh new file mode 100644 index 0000000..df81556 --- /dev/null +++ b/tools/run_local_asset_worker.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +set -euo pipefail + +HOST="${LOCAL_ASSET_HOST:-127.0.0.1}" +PORT="${LOCAL_ASSET_PORT:-8092}" +BOOTSTRAP_MODE="${LOCAL_ASSET_BOOTSTRAP_MODE:-auto}" + +if [ ! -d ".venv-local-asset" ]; then + python3 -m venv .venv-local-asset +fi + +. .venv-local-asset/bin/activate +SENTINEL=".venv-local-asset/.bootstrap-complete" + +if [ "${BOOTSTRAP_MODE}" = "always" ]; then + python -m pip install --upgrade pip + python -m pip install -r tools/local_asset_requirements.txt + touch "${SENTINEL}" +elif [ "${BOOTSTRAP_MODE}" = "auto" ] && [ ! -f "${SENTINEL}" ]; then + python -m pip install --upgrade pip + python -m pip install -r tools/local_asset_requirements.txt + touch "${SENTINEL}" +else + echo "Local asset bootstrap skipped; using existing virtualenv." +fi + +if [ "${LOCAL_ASSET_PRELOAD_ONLY:-false}" = "true" ]; then + echo "Local asset dependencies are ready." + exit 0 +fi + +exec python tools/local_asset_worker.py "$@" diff --git a/tools/run_local_cv_worker.sh b/tools/run_local_cv_worker.sh index 2cd4fd8..e8e2c6a 100755 --- a/tools/run_local_cv_worker.sh +++ b/tools/run_local_cv_worker.sh @@ -4,14 +4,23 @@ set -euo pipefail HOST="${LOCAL_CV_HOST:-127.0.0.1}" PORT="${LOCAL_CV_PORT:-8091}" MODEL="${LOCAL_CV_YOLO_MODEL:-yolov8n.pt}" +BOOTSTRAP_MODE="${LOCAL_CV_BOOTSTRAP_MODE:-auto}" if [ ! -d ".venv-local-cv" ]; then python3 -m venv .venv-local-cv fi . .venv-local-cv/bin/activate -python -m pip install --upgrade pip -python -m pip install -r tools/local_cv_requirements.txt + +if [ "${BOOTSTRAP_MODE}" = "always" ]; then + python -m pip install --upgrade pip + python -m pip install -r tools/local_cv_requirements.txt +elif [ "${BOOTSTRAP_MODE}" = "auto" ] && [ ! -x ".venv-local-cv/bin/uvicorn" ]; then + python -m pip install --upgrade pip + python -m pip install -r tools/local_cv_requirements.txt +else + echo "Local CV bootstrap skipped; using existing virtualenv." +fi if [ "${LOCAL_CV_DISABLE_YOLO:-false}" != "true" ]; then LOCAL_CV_YOLO_MODEL="$MODEL" python -c "import os; from ultralytics import YOLO; YOLO(os.environ['LOCAL_CV_YOLO_MODEL'])"