From 618ba9af493b2f38ca4346c2a1de84fe741b2ed0 Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Sun, 12 Jul 2026 01:20:18 +0200 Subject: [PATCH] - Implement end-to-end cinematic highlight rendering pipeline - Wire visual effects and local asset generation into highlight flow - Add director plan rendering with local asset worker --- docs/cinematic-editing-runbook.md | 28 +- ...production-cinematic-ai-director-prompt.md | 310 ++++++++++++ ...uction-cinematic-highlight-editing-plan.md | 4 + .../edit-projects/source-clips/project.json | 15 + .../source-clips/render-manifest.json | 8 + .../config/VideoClippingProperties.java | 65 ++- .../editing/EditPlanInboxScanner.java | 28 +- .../videoclips/editing/EditPlanValidator.java | 46 +- .../editing/FfmpegEditRenderer.java | 30 +- .../editing/HighlightAssetRequest.java | 14 + .../editing/HighlightDirectorFlowService.java | 338 +++++++++++++ .../editing/HighlightDirectorPlan.java | 36 ++ .../editing/HighlightDirectorPlanScanner.java | 87 ++++ .../HighlightDirectorPromptBackfill.java | 70 +++ .../HighlightDirectorPromptGenerator.java | 246 ++++++++++ .../editing/HighlightFfmpegRenderer.java | 460 ++++++++++++++++++ .../editing/HighlightLocalAssetWorker.java | 216 ++++++++ .../editing/HighlightSourceScheduler.java | 12 +- .../editing/HighlightVisualEffectsStage.java | 63 +++ .../editing/LocalAssetGenerationStage.java | 251 ++++++++++ .../videoclips/editing/RenderManifest.java | 16 +- .../editing/StoryboardPromptGenerator.java | 12 + src/main/resources/application.yml | 8 +- .../config/VideoClippingPropertiesTest.java | 2 +- .../CinematicEditingIntegrationTest.java | 10 +- .../editing/EditPlanInboxScannerTest.java | 5 +- .../editing/EditPlanValidatorTest.java | 41 +- .../HighlightAssetPreparationServiceTest.java | 75 +++ .../HighlightDirectorFlowServiceTest.java | 109 +++++ .../HighlightDirectorPromptBackfillTest.java | 83 ++++ .../HighlightDirectorPromptGeneratorTest.java | 90 ++++ .../HighlightLocalAssetWorkerTest.java | 64 +++ .../editing/HighlightSourceSchedulerTest.java | 5 +- .../HighlightVisualEffectsStageTest.java | 41 ++ .../LocalAssetGenerationStageTest.java | 98 ++++ .../StoryboardPromptGeneratorTest.java | 1 + 36 files changed, 2965 insertions(+), 22 deletions(-) create mode 100644 docs/production-cinematic-ai-director-prompt.md create mode 100644 output/edit-projects/source-clips/project.json create mode 100644 output/edit-projects/source-clips/render-manifest.json create mode 100644 src/main/java/org/example/videoclips/editing/HighlightAssetRequest.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightDirectorPlan.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightDirectorPlanScanner.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightDirectorPromptBackfill.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightDirectorPromptGenerator.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java create mode 100644 src/main/java/org/example/videoclips/editing/HighlightVisualEffectsStage.java create mode 100644 src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java create mode 100644 src/test/java/org/example/videoclips/editing/HighlightAssetPreparationServiceTest.java create mode 100644 src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java create mode 100644 src/test/java/org/example/videoclips/editing/HighlightDirectorPromptBackfillTest.java create mode 100644 src/test/java/org/example/videoclips/editing/HighlightDirectorPromptGeneratorTest.java create mode 100644 src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java create mode 100644 src/test/java/org/example/videoclips/editing/HighlightVisualEffectsStageTest.java create mode 100644 src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java diff --git a/docs/cinematic-editing-runbook.md b/docs/cinematic-editing-runbook.md index 2c7e441..7a335f3 100644 --- a/docs/cinematic-editing-runbook.md +++ b/docs/cinematic-editing-runbook.md @@ -50,7 +50,7 @@ video-clipping: project-directory: ./output/edit-projects thumbnail-count-per-clip: 5 proxy-enabled: true - target-duration-seconds: 60 + target-duration-seconds: 600 output-width: 1920 output-height: 1080 output-frame-rate: 30 @@ -186,6 +186,13 @@ The AI director should inspect: - `thumbnails/*.jpg` - `proxies/*.mp4` if more visual context is needed +After the plan is imported, the service also runs an asset materialization stage: + +- It reuses matching generated music, SFX, and voiceover assets from the shared cache folders. +- It copies reusable assets into the project-local `audio/` folder so the renderer can use them. +- It writes reusable asset requests under `assets/requests/` when a needed asset is missing. +- It waits to render if required SFX assets are still missing. + The AI director must write this file: ```text @@ -239,6 +246,14 @@ Rules the AI director must follow: The renderer can mix optional WAV files into the final output. +Shared cache locations: + +- Music: `input/highlights/assets/music/` +- SFX: `input/highlights/assets/sfx/` +- Voiceover cache: `output/highlight-projects/_voiceover-cache/` + +If the service or an AI worker generates an asset there, later projects can reuse it automatically. + Place optional assets here before rendering: ```text @@ -308,6 +323,14 @@ After rendering completes, the final output is: output/edit-projects/porsche-session-001/final.mp4 ``` +The renderer also publishes each final rendered segment clip that was concatenated into the full video: + +```text +output/edit-projects/porsche-session-001/rendered-clips/clip_0001.mp4 +output/edit-projects/porsche-session-001/rendered-clips/clip_0002.mp4 +output/edit-projects/porsche-session-001/rendered-clips/clip_0003.mp4 +``` + The render manifest is: ```text @@ -320,11 +343,12 @@ The QA report is: output/edit-projects/porsche-session-001/qa-report.json ``` -The manifest is useful for debugging because it records the selected clips, output path, duration, and FFmpeg command summaries. +The manifest is useful for debugging because it records the selected clips, rendered segment clip paths, output path, duration, and FFmpeg command summaries. The QA report is useful for acceptance because it records: - final output existence +- rendered segment clip existence - duration consistency with the edit timeline - required asset resolution - text overlay timeline and placement safety diff --git a/docs/production-cinematic-ai-director-prompt.md b/docs/production-cinematic-ai-director-prompt.md new file mode 100644 index 0000000..e8ea2e5 --- /dev/null +++ b/docs/production-cinematic-ai-director-prompt.md @@ -0,0 +1,310 @@ +# Production Cinematic AI Director Prompt + +Use this prompt with a filesystem-capable AI agent such as Codex or Claude when a highlight project has already been analyzed by the service. + +Expected project folder: + +```text +output/highlight-projects// +``` + +The AI director must inspect the project artifacts, choose all worthwhile highlight moments, and write a strict renderable edit plan. The goal is a platform-ready cinematic highlight video with strong pacing, fitting sound design, fitting voiceover, tasteful text overlays, and visual treatments that match the actual content. + +## Prompt + +```text +You are a senior commercial video editor, creative director, sound designer, and post-production supervisor. + +Your assignment is to create a production-ready cinematic highlight edit plan from the analyzed project folder. The final rendered video must feel strong enough for a public video streaming platform: clear hook, strong visual selection, coherent micro-story, professional pacing, fitting sound design, tasteful overlays, and a voiceover that matches the actual footage. Do not promise revenue or make unsupported claims; optimize for viewer retention and professional presentation. + +PROJECT FOLDER +- Work only inside this folder: output/highlight-projects// +- Read all available project files before deciding: + - project.json + - manifest.json + - analysis/source-analysis.json + - analysis/ffprobe.json + - analysis/scene-segments.json + - analysis/audio-analysis.json + - analysis/visual-analysis.json + - analysis/contact-sheets/* + - analysis/frames/* + - analysis/proxies/* when motion needs closer inspection + - assets/music/*, assets/sfx/*, assets/voiceover/*, assets/overlays/* if present + +OUTPUT FILE +- Write the final edit plan JSON to: + director/edit-plan.json +- Return JSON only in that file. +- Do not wrap JSON in Markdown fences. +- Do not render video. +- Do not modify source video, analysis files, thumbnails, contact sheets, or proxies. + +PRIMARY GOAL +Create one cinematic highlight video that is specific to the provided footage and content type. The highlight length must be content-driven, not fixed. Use all strong highlights from the source, but do not pad with weak or repetitive footage just to reach a duration. + +The edit must have: +- A 1-3 second opening hook that immediately shows the most visually interesting or emotionally compelling moment. +- A clear middle section with rising energy, variety, and no repetitive shot choices. +- A satisfying final hero/payoff shot. +- A final duration that fits the source material and keeps audience retention high. +- Beat-aware pacing and deliberate shot lengths. +- Specific visual treatment notes that the renderer can execute: cinematic grade, punch-in crop, subtle speed ramp, vignette, sharpness, contrast, saturation, slow-motion emphasis, or clean natural grade. +- Fitting music direction and sound effects timing. +- Voiceover text that comments on what is actually visible or strongly implied by the footage. +- Short text overlays only where they improve retention or clarity. + +DELIVERABLE EXPECTATION +The service must ultimately render both: +- the full final highlight video +- each rendered timeline segment clip that was used to assemble the full final highlight video + +Expected output for the current edit-project renderer: +- final.mp4 +- rendered-clips/clip_0001.mp4 +- rendered-clips/clip_0002.mp4 +- rendered-clips/clip_0003.mp4 +- render-manifest.json with renderedClipPaths + +Expected output for the highlight-project renderer when implemented: +- highlights//final.mp4 +- highlights//rendered-clips/clip_0001.mp4 +- highlights//rendered-clips/clip_0002.mp4 +- highlights//rendered-clips/clip_0003.mp4 +- highlights//render-manifest.json with renderedClipPaths + +CONTENT CATEGORY DECISION +Classify the project into exactly one content category: +- car_vlog +- food_vlog +- family_vlog +- generic_vlog + +Use the category to direct the whole edit. + +For car_vlog: +- Make it premium, powerful, precise, and aspirational. +- Prioritize hero angles, reflections, wheels, badges, road movement, cockpit details, acceleration, reveal shots, light, shadow, and motion. +- Use punchy cuts, speed ramps, whooshes, impact hits, engine/exhaust SFX only when appropriate, cinematic contrast, and confident short overlays. +- Voiceover should praise what is actually visible: stance, motion, design, presence, craft, road feel. Do not invent specifications, price, horsepower, rarity, or model facts unless visible in the footage. + +For food_vlog: +- Make it sensory, warm, rhythmic, and appetizing. +- Prioritize texture, slicing, pouring, steam, sizzling, plating, table reveals, reactions, and macro-like detail. +- Use tactile SFX, warm grade, gentle groove, ingredient/payoff overlays, and voiceover focused on craft, freshness, texture, and taste. + +For family_vlog: +- Make it warm, emotional, human, and memory-focused. +- Prioritize faces, reactions, laughter, hugs, children, travel reveals, meaningful moments, and natural dialogue. +- Use gentle music, soft grade, light grain, slower pacing, and minimal overlays. +- Voiceover should feel personal and reflective, not promotional. + +For generic_vlog: +- Build a clear mini-story from novelty, clean composition, motion, reaction, place, process, and payoff. +- Use restrained cinematic effects and explain only what the footage supports. + +SHOT SELECTION RULES +Before writing JSON, inspect the contact sheets and representative frames. + +Create a private selection table for yourself: +- candidate source range +- visual appeal +- motion/energy +- uniqueness +- story role +- audio opportunity +- risk: blur, darkness, repetitive angle, dead time + +Then use only the best ranges. Avoid: +- repeated near-identical shots unless repetition is intentional +- long boring segments +- badly blurred or unusably dark footage unless it is the only important moment +- shots with no story purpose +- unsupported narration + +TIMELINE AND PACING RULES +- Do not force a strict 60 second edit unless the footage actually supports it. +- Derive the final duration from the input source length and number of strong moments. +- Include every strong highlight moment that improves the final video. +- Exclude weak, repetitive, technically poor, or dead-time footage even if the source is long. +- Maximum final duration is 600 seconds, which is 10 minutes. +- Most videos should be shorter than the maximum. Use 10 minutes only when the source has enough distinct, high-quality highlights to justify it. +- Recommended duration guide: + - Source under 30 seconds: final highlight usually 8-20 seconds. + - Source 30-90 seconds: final highlight usually 15-45 seconds. + - Source 90 seconds-5 minutes: final highlight usually 30-120 seconds. + - Source 5-20 minutes: final highlight usually 60-180 seconds. + - Source over 20 minutes: final highlight can be longer, but only if the footage has enough distinct high-quality moments. +- The JSON field targetDurationSeconds must equal the actual planned final timeline duration, not a fixed configured default. +- The final decision timelineEndSeconds must equal targetDurationSeconds. +- Timeline must start at 0.0. +- Every decision timelineStartSeconds must equal the previous decision timelineEndSeconds. +- No overlaps. No gaps. +- For each decision: + rendered duration = (sourceEndSeconds - sourceStartSeconds) / playbackSpeed + timelineEndSeconds - timelineStartSeconds must equal rendered duration within 0.05 seconds. +- playbackSpeed must be between 0.25 and 4.0. +- Keep most shots between 1.0 and 4.0 seconds. +- Use longer shots only for emotional moments, reveals, or beautiful hero motion. +- Use faster shots for montages, details, impacts, or beat drops. + +ALLOWED TRANSITIONS +Use only: +- cut +- crossfade +- fade-in +- fade-out +- none + +Prefer mostly cuts. Use crossfade/fades sparingly for emotional, elegant, or time-passing moments. + +VISUAL TREATMENT RULES +visualTreatment must be concise but specific. Good examples: +- cinematic grade with subtle punch-in +- high-contrast car hero grade +- warm food closeup grade +- soft family memory grade +- speed-ramp detail accent +- clean natural grade +- none + +Use visual effects to support the moment. Do not overuse effects. + +AUDIO RULES +Use audioCues for music and SFX direction. + +Music: +- A music cue may use a descriptive assetKey such as premium_cinematic_drive, warm_acoustic_memory, tactile_food_groove, urban_pulse_intro. +- Music should cover the full timeline unless silence is intentionally used. +- Set gainDb conservatively, usually between -22 and -10. +- Describe the music mood and why it fits. + +SFX: +- Only reference SFX assetKey values that exist as WAV files in the project audio/sfx folder or assets/sfx folder if the renderer maps them. +- If no SFX files exist, do not invent file-backed SFX cues. Instead, put desired SFX ideas in the cue notes only if the service can later generate assets. +- SFX should be precise: whoosh on transition, bass hit on reveal, camera shutter on detail, subtle riser before hero shot, engine accent for car footage, sizzle/chop/pour for food, gentle ambience for family. + +VOICEOVER RULES +- Voiceover is optional only if the project disables it. If enabled, include concise lines. +- Each line must be 240 characters or fewer. +- Use natural, premium, non-cheesy language. +- Voiceover must fit the category and visible content. +- Do not use generic filler like "this is amazing" or "an unforgettable journey" unless the footage clearly supports it. +- Do not invent facts. +- Give each line a delivery direction, for example: + - calm premium narrator + - confident cinematic narrator + - warm reflective narrator + - energetic food host + +TEXT OVERLAY RULES +- Overlays must be short: 80 characters or fewer. +- Use only these placements: + - lower_left_safe + - lower_center_safe + - center_safe + - upper_left_safe + - upper_right_safe +- Use only these animations: + - fade_slide_up + - fade_in + - none +- Use overlays sparingly. 1-5 overlays is usually enough. +- Overlays should feel premium and specific, not clickbait. + +QUALITY BAR +Reject your own first draft if it has any of these problems: +- The first 3 seconds are not compelling. +- The edit is just chronological trimming without a story. +- The voiceover is generic or not grounded in visible footage. +- Music or SFX are placeholders with no timing purpose. +- Text overlays repeat what the voiceover already says. +- The final shot is weak. +- Any JSON field uses invalid values. +- Any clip ID or timestamp is invented. +- Any timeline math is invalid. + +REQUIRED JSON SHAPE +Write exactly this JSON shape: + +{ + "projectId": "", + "style": "", + "targetDurationSeconds": , + "decisions": [ + { + "clipId": "", + "sourceStartSeconds": 0.0, + "sourceEndSeconds": 2.5, + "timelineStartSeconds": 0.0, + "timelineEndSeconds": 2.5, + "transitionIn": "cut", + "transitionOut": "cut", + "playbackSpeed": 1.0, + "visualTreatment": "cinematic grade with subtle punch-in", + "reason": "Opening hook: strongest hero angle and immediate viewer retention." + } + ], + "audioCues": [ + { + "type": "music", + "assetKey": "premium_cinematic_drive", + "timelineStartSeconds": 0.0, + "timelineEndSeconds": , + "gainDb": -16.0, + "notes": "Low cinematic pulse that builds through detail shots and lifts into the final hero moment." + } + ], + "voiceover": [ + { + "text": "A concise, specific line grounded in what is visible in the footage.", + "timelineStartSeconds": 1.0, + "timelineEndSeconds": 4.0, + "delivery": "confident cinematic narrator" + } + ], + "overlays": [ + { + "text": "Short premium overlay", + "timelineStartSeconds": 0.5, + "timelineEndSeconds": 2.0, + "placement": "lower_left_safe", + "animation": "fade_slide_up", + "reason": "Adds context without duplicating the narration." + } + ], + "renderProfile": "mp4-h264-aac-1080p", + "summary": "A short explanation of the creative strategy, category choice, hook, pacing, sound design, and final payoff." +} + +FINAL SELF-CHECK BEFORE WRITING director/edit-plan.json +Verify: +- projectId matches project.json. +- style matches project.json. +- targetDurationSeconds equals the actual final timeline duration selected from the source highlights. +- decisions is not empty. +- audioCues, voiceover, and overlays are arrays even when empty. +- All clip IDs exist. +- All source ranges are inside clip duration. +- Timeline starts at 0.0 and is contiguous. +- Final decision timelineEndSeconds equals targetDurationSeconds. +- Duration math matches playbackSpeed. +- transitionIn and transitionOut use only allowed values. +- playbackSpeed is between 0.25 and 4.0. +- overlay placements and animations use only allowed values. +- overlay text is 80 characters or fewer. +- voiceover lines are 240 characters or fewer. +- SFX asset keys reference real files if SFX cues are included. +- JSON is valid and contains no comments or Markdown. +``` + +## Notes For The Service + +This prompt is intentionally stricter than the current lightweight storyboard prompt. It is designed to prevent the common failure mode where the AI produces a generic, boring, or invalid edit plan. + +Current renderer constraints still matter: + +- `FfmpegEditRenderer` can render the `EditPlan` schema used by `output/edit-projects`. +- The highlight project flow still needs an importer/renderer bridge before `output/highlight-projects//highlights//final.mp4` can be produced automatically. +- If SFX are referenced, the current validator expects matching WAV files for SFX asset keys. Do not ask the AI to reference non-existent SFX files unless the asset generation step is implemented first. +- The current noop voiceover provider writes a script, not a real spoken voice. Production-ready voice requires a real TTS provider or recorded voiceover asset. diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index 8c69544..9c91de0 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -204,6 +204,7 @@ The system needs explicit providers instead of pretending a prompt can create fi - Font provider: local fonts with brand/category mapping. - LUT provider: local LUT files or named FFmpeg grade presets. - Asset cache: deterministic reuse by prompt, category, duration, and provider settings. +- Asset generation stage: when a needed asset is missing, write a reusable request pack and keep the target cache path stable so future AI-generated assets can be reused. Licensing must be tracked in `render-manifest.json` for any music, SFX, voice, font, or LUT asset used in a final export. @@ -552,6 +553,9 @@ Expected renderer output: ```text output/highlight-projects//highlights//final.mp4 +output/highlight-projects//highlights//rendered-clips/clip_0001.mp4 +output/highlight-projects//highlights//rendered-clips/clip_0002.mp4 +output/highlight-projects//highlights//rendered-clips/clip_0003.mp4 output/highlight-projects//highlights//preview.mp4 output/highlight-projects//highlights//render-manifest.json output/highlight-projects//highlights//qa-report.json diff --git a/output/edit-projects/source-clips/project.json b/output/edit-projects/source-clips/project.json new file mode 100644 index 0000000..97f8fc4 --- /dev/null +++ b/output/edit-projects/source-clips/project.json @@ -0,0 +1,15 @@ +{ + "id" : "source-clips", + "name" : "source-clips", + "status" : "RENDERED", + "inputDirectory" : "./input/editing/processed/source-clips", + "outputDirectory" : "output/edit-projects/source-clips", + "targetDurationSeconds" : 60, + "style" : "cinematic-porsche-promo", + "voiceoverEnabled" : true, + "musicEnabled" : true, + "soundEffectsEnabled" : true, + "createdAt" : "2026-07-10T17:25:59.366353Z", + "updatedAt" : "2026-07-10T21:58:05.307273Z", + "failureReason" : null +} \ No newline at end of file diff --git a/output/edit-projects/source-clips/render-manifest.json b/output/edit-projects/source-clips/render-manifest.json new file mode 100644 index 0000000..bdfc9c3 --- /dev/null +++ b/output/edit-projects/source-clips/render-manifest.json @@ -0,0 +1,8 @@ +{ + "projectId" : "source-clips", + "clipIds" : [ "clip_00000", "clip_00005", "clip_00008", "clip_00001", "clip_00002", "clip_00003", "clip_00004", "clip_00010", "clip_00032", "clip_00012", "clip_00013", "clip_00020", "clip_00018", "clip_00024", "clip_00029", "clip_00028", "clip_00033", "clip_00006", "clip_00002", "clip_00034" ], + "outputPath" : "output/edit-projects/source-clips/final.mp4", + "durationSeconds" : 60.0, + "commands" : [ [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00000.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0001.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "2.0", "-to", "5.0", "-i", "./input/editing/processed/source-clips/clip_00005.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0002.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "0.0", "-to", "3.0", "-i", "./input/editing/processed/source-clips/clip_00008.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0003.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00001.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0004.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00002.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0005.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "2.0", "-to", "5.0", "-i", "./input/editing/processed/source-clips/clip_00003.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0006.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00004.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0007.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00010.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0008.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00032.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0009.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "2.0", "-to", "5.0", "-i", "./input/editing/processed/source-clips/clip_00012.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0010.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "0.0", "-to", "3.0", "-i", "./input/editing/processed/source-clips/clip_00013.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0011.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "0.0", "-to", "3.0", "-i", "./input/editing/processed/source-clips/clip_00020.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0012.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00018.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0013.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "2.0", "-to", "5.0", "-i", "./input/editing/processed/source-clips/clip_00024.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0014.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "0.0", "-to", "3.0", "-i", "./input/editing/processed/source-clips/clip_00029.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0015.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "0.5", "-to", "3.5", "-i", "./input/editing/processed/source-clips/clip_00028.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0016.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "1.0", "-to", "4.0", "-i", "./input/editing/processed/source-clips/clip_00033.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0017.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "0.0", "-to", "3.0", "-i", "./input/editing/processed/source-clips/clip_00006.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0018.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "4.0", "-to", "7.0", "-i", "./input/editing/processed/source-clips/clip_00002.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=in:st=0:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0019.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-ss", "0.0", "-to", "3.0", "-i", "./input/editing/processed/source-clips/clip_00034.mp4", "-vf", "scale=1920:1080:force_original_aspect_ratio=decrease,pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0,fade=t=out:st=2.5:d=0.5", "-af", "atempo=1.0", "-r", "30", "-map", "0:v:0", "-map", "0:a?", "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/render-work/segment_0020.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-f", "concat", "-safe", "0", "-i", "output/edit-projects/source-clips/render-work/concat.txt", "-c", "copy", "output/edit-projects/source-clips/render-work/timeline-video.mp4" ], [ "ffmpeg", "-hide_banner", "-y", "-i", "output/edit-projects/source-clips/render-work/timeline-video.mp4", "-i", "output/edit-projects/source-clips/audio/music.wav", "-i", "output/edit-projects/source-clips/audio/voiceover.wav", "-i", "output/edit-projects/source-clips/audio/sfx/impact.wav", "-i", "output/edit-projects/source-clips/audio/sfx/whoosh.wav", "-i", "output/edit-projects/source-clips/audio/sfx/engine-pulse.wav", "-i", "output/edit-projects/source-clips/audio/sfx/whoosh.wav", "-i", "output/edit-projects/source-clips/audio/sfx/impact.wav", "-filter_complex", "[1:a]volume=0.25[music];[2:a]volume=1.0[voice];[3:a]atrim=duration=0.8,volume=-6.0dB,adelay=0|0[sfx0];[4:a]atrim=duration=1.2000000000000002,volume=-10.0dB,adelay=6000|6000[sfx1];[5:a]atrim=duration=1.3999999999999986,volume=-9.0dB,adelay=21000|21000[sfx2];[6:a]atrim=duration=1.1999999999999993,volume=-11.0dB,adelay=27000|27000[sfx3];[7:a]atrim=duration=0.7999999999999972,volume=-8.0dB,adelay=57000|57000[sfx4];[0:a][music][voice][sfx0][sfx1][sfx2][sfx3][sfx4]amix=inputs=8:duration=first:dropout_transition=2[a]", "-map", "0:v:0", "-map", "[a]", "-c:v", "copy", "-c:a", "aac", "-b:a", "192k", "-ar", "48000", "output/edit-projects/source-clips/final.mp4" ] ], + "completedAt" : "2026-07-10T21:58:05.304722Z" +} \ No newline at end of file diff --git a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java index 849a6bf..3e9808d 100644 --- a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java +++ b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java @@ -512,7 +512,7 @@ public class VideoClippingProperties { private double silenceMinimumDurationSeconds = 0.5; @Min(1) - private int targetDurationSeconds = 60; + private int targetDurationSeconds = 600; @Min(1) private int outputWidth = 1920; @@ -1078,6 +1078,21 @@ public class VideoClippingProperties { @Min(1000) private long pollIntervalMs = 5000; + private boolean renderEnabled = true; + + private boolean requireDirectorApproval = false; + + private String approvalFileName = "approved.flag"; + + @Min(1) + private int maxHighlightsPerSource = 3; + + @Min(1) + private double highlightMinDurationSeconds = 8.0; + + @Min(1) + private double highlightMaxDurationSeconds = 35.0; + public boolean isEnabled() { return enabled; } @@ -1125,6 +1140,54 @@ public class VideoClippingProperties { public void setPollIntervalMs(long pollIntervalMs) { this.pollIntervalMs = pollIntervalMs; } + + public boolean isRenderEnabled() { + return renderEnabled; + } + + public void setRenderEnabled(boolean renderEnabled) { + this.renderEnabled = renderEnabled; + } + + public boolean isRequireDirectorApproval() { + return requireDirectorApproval; + } + + public void setRequireDirectorApproval(boolean requireDirectorApproval) { + this.requireDirectorApproval = requireDirectorApproval; + } + + public String getApprovalFileName() { + return approvalFileName; + } + + public void setApprovalFileName(String approvalFileName) { + this.approvalFileName = approvalFileName; + } + + public int getMaxHighlightsPerSource() { + return maxHighlightsPerSource; + } + + public void setMaxHighlightsPerSource(int maxHighlightsPerSource) { + this.maxHighlightsPerSource = maxHighlightsPerSource; + } + + public double getHighlightMinDurationSeconds() { + return highlightMinDurationSeconds; + } + + public void setHighlightMinDurationSeconds(double highlightMinDurationSeconds) { + this.highlightMinDurationSeconds = highlightMinDurationSeconds; + } + + public double getHighlightMaxDurationSeconds() { + return highlightMaxDurationSeconds; + } + + public void setHighlightMaxDurationSeconds(double highlightMaxDurationSeconds) { + this.highlightMaxDurationSeconds = highlightMaxDurationSeconds; + } } } diff --git a/src/main/java/org/example/videoclips/editing/EditPlanInboxScanner.java b/src/main/java/org/example/videoclips/editing/EditPlanInboxScanner.java index 4aaf7b3..f430223 100644 --- a/src/main/java/org/example/videoclips/editing/EditPlanInboxScanner.java +++ b/src/main/java/org/example/videoclips/editing/EditPlanInboxScanner.java @@ -37,6 +37,7 @@ public class EditPlanInboxScanner { private final EditProjectStore store; private final EditProjectService projectService; private final EditPlanValidator validator; + private final Optional assetGenerationStage; private final Optional renderer; private final VoiceoverGenerator voiceoverGenerator; private final AtomicBoolean scanning = new AtomicBoolean(); @@ -49,9 +50,11 @@ public class EditPlanInboxScanner { EditProjectService projectService, EditPlanValidator validator, VoiceoverGenerator voiceoverGenerator, + ObjectProvider assetGenerationStageProvider, ObjectProvider rendererProvider ) { this(properties, objectMapper, store, projectService, validator, voiceoverGenerator, + assetGenerationStageProvider.getIfAvailable(), rendererProvider.getIfAvailable()); } @@ -63,13 +66,13 @@ public class EditPlanInboxScanner { EditPlanValidator validator, EditRenderer renderer ) { - this(properties, objectMapper, store, projectService, validator, null, renderer); + this(properties, objectMapper, store, projectService, validator, null, null, renderer); } EditPlanInboxScanner( VideoClippingProperties properties, ObjectMapper objectMapper, EditProjectStore store, EditProjectService projectService, EditPlanValidator validator, - VoiceoverGenerator voiceoverGenerator, EditRenderer renderer + VoiceoverGenerator voiceoverGenerator, AssetGenerationStage assetGenerationStage, EditRenderer renderer ) { this.projectRoot = Path.of(properties.getEditing().getProjectDirectory()); this.expectedPlanFileName = properties.getEditing().getLocalDirector().getExpectedPlanFileName(); @@ -82,6 +85,7 @@ public class EditPlanInboxScanner { this.validator = validator; this.renderer = Optional.ofNullable(renderer); this.voiceoverGenerator = voiceoverGenerator; + this.assetGenerationStage = Optional.ofNullable(assetGenerationStage); } @Scheduled( @@ -154,6 +158,11 @@ public class EditPlanInboxScanner { } store.writeJson(projectId, "edit-plan.json", normalized(plan)); + AssetGenerationResult assetGenerationResult = assetGenerationStage + .map(stage -> stage.prepare(projectId, plan)) + .orElse(new AssetGenerationResult(projectId, true, List.of(), java.time.Instant.now())); + log.info("event=edit_asset_generation_completed project_id={} item_count={} ready_for_render={}", + projectId, assetGenerationResult.items().size(), assetGenerationResult.readyForRender()); if (voiceoverGenerator != null && !plan.voiceover().isEmpty()) { voiceoverGenerator.generateVoiceover(projectId, plan.voiceover()); } @@ -164,8 +173,11 @@ public class EditPlanInboxScanner { projectId, plan.decisions().size(), autoRender, requireApprovalBeforeRender); log.info("event=edit_plan_saved project_id={} decision_count={} source=inbox", projectId, plan.decisions().size()); - if (autoRender) { + if (autoRender && assetGenerationResult.readyForRender()) { renderIfApproved(projectId, store.inboxDirectory(projectId)); + } else if (autoRender) { + log.info("event=edit_render_waiting_for_assets project_id={} blocking_assets_pending=true", + projectId); } } @@ -175,6 +187,16 @@ public class EditPlanInboxScanner { projectId, inbox.resolve(approvalFileName)); return; } + AssetGenerationStage stage = assetGenerationStage.orElse(null); + if (stage != null) { + EditPlan plan = store.readJson(projectId, "edit-plan.json", EditPlan.class); + AssetGenerationResult result = stage.prepare(projectId, plan); + if (!result.readyForRender()) { + log.info("event=edit_render_waiting_for_assets project_id={} blocking_assets_pending=true", + projectId); + return; + } + } renderer.orElseThrow(() -> new IllegalStateException( "Automatic rendering is enabled but no EditRenderer is available")).render(projectId); } diff --git a/src/main/java/org/example/videoclips/editing/EditPlanValidator.java b/src/main/java/org/example/videoclips/editing/EditPlanValidator.java index 108120f..39d68db 100644 --- a/src/main/java/org/example/videoclips/editing/EditPlanValidator.java +++ b/src/main/java/org/example/videoclips/editing/EditPlanValidator.java @@ -80,9 +80,13 @@ public class EditPlanValidator { validateAudioCues(projectId, plan); validateVoiceover(plan); validateOverlays(plan); - if (previousEnd > project.targetDurationSeconds() + TARGET_TOLERANCE_SECONDS - || Math.abs(plan.targetDurationSeconds() - project.targetDurationSeconds()) > EPSILON) { - reject("Edit plan duration exceeds or does not match the project target duration"); + validateCinematicRichness(project, plan); + if (plan.targetDurationSeconds() <= 0 + || Math.abs(previousEnd - plan.targetDurationSeconds()) > TARGET_TOLERANCE_SECONDS) { + reject("Edit plan target duration must match the final timeline duration"); + } + if (plan.targetDurationSeconds() > project.targetDurationSeconds() + TARGET_TOLERANCE_SECONDS) { + reject("Edit plan duration exceeds the project maximum target duration"); } return plan; } @@ -152,6 +156,42 @@ public class EditPlanValidator { } } + private void validateCinematicRichness(EditProject project, EditPlan plan) { + int cinematicDimensions = 0; + if (hasVisualTreatment(plan)) { + cinematicDimensions++; + } + if (project.musicEnabled() && hasAudioCueType(plan, "music")) { + cinematicDimensions++; + } + if (project.voiceoverEnabled() && !plan.voiceover().isEmpty()) { + cinematicDimensions++; + } + if (project.soundEffectsEnabled() && hasAudioCueType(plan, "sfx")) { + cinematicDimensions++; + } + if (!plan.overlays().isEmpty()) { + cinematicDimensions++; + } + + if (cinematicDimensions < 2) { + reject("Edit plan is too bare; it must use at least two cinematic elements such as visual treatment, " + + "music, voiceover, sound effects, or text overlays"); + } + } + + private boolean hasVisualTreatment(EditPlan plan) { + return plan.decisions().stream() + .map(EditDecision::visualTreatment) + .filter(value -> value != null && !value.isBlank() && !"none".equalsIgnoreCase(value)) + .findFirst() + .isPresent(); + } + + private boolean hasAudioCueType(EditPlan plan, String type) { + return plan.audioCues().stream().anyMatch(cue -> type.equals(cue.type())); + } + private void validateTransition(String transition) { if (transition == null || !TRANSITIONS.contains(transition)) { reject("Unsupported transition: " + transition); diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index e08c1b9..1146552 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -127,6 +127,7 @@ public class FfmpegEditRenderer implements EditRenderer { runAndRecord(segmentCommand(clips.get(decision.clipId()).sourcePath(), decision, output), commands); segments.add(output); } + List renderedClips = publishRenderedClips(projectId, segments); Path concatFile = work.resolve("concat.txt"); writeConcatFile(concatFile, segments); Path timeline = work.resolve("timeline-video.mp4"); @@ -152,10 +153,11 @@ public class FfmpegEditRenderer implements EditRenderer { copy(videoTimeline, output); } double duration = plan.decisions().get(plan.decisions().size() - 1).timelineEndSeconds(); - RenderQaReport qaReport = buildQaReport(projectId, plan, output, duration, commands, assets); + RenderQaReport qaReport = buildQaReport(projectId, plan, output, duration, renderedClips, commands, assets); RenderManifest manifest = new RenderManifest(projectId, - plan.decisions().stream().map(EditDecision::clipId).toList(), output.toString(), duration, - List.copyOf(commands), assets, Instant.now()); + plan.decisions().stream().map(EditDecision::clipId).toList(), + renderedClips.stream().map(Path::toString).toList(), output.toString(), duration, List.copyOf(commands), + assets, Instant.now()); store.writeJson(projectId, "qa-report.json", qaReport); store.writeJson(projectId, "render-manifest.json", manifest); projectService.updateProject(projectId, EditProjectStatus.RENDERED, Path.of(project.inputDirectory()), null); @@ -361,9 +363,11 @@ public class FfmpegEditRenderer implements EditRenderer { } RenderQaReport buildQaReport(String projectId, EditPlan plan, Path output, double duration, + List renderedClips, List> commands, List assets) { List checks = new ArrayList<>(); checks.add(outputExistsCheck(output)); + checks.add(renderedClipsExistCheck(renderedClips)); checks.add(durationMatchesTimelineCheck(plan, duration)); checks.add(requiredAssetsResolvedCheck(plan, assets)); checks.add(overlaysSafeCheck(plan, duration)); @@ -388,6 +392,14 @@ public class FfmpegEditRenderer implements EditRenderer { exists ? "final output exists" : "final output is missing: " + output); } + private RenderQaCheck renderedClipsExistCheck(List renderedClips) { + List missing = renderedClips.stream() + .filter(path -> !Files.isRegularFile(path)) + .toList(); + return new RenderQaCheck("rendered_clips_exist", missing.isEmpty(), "ERROR", + missing.isEmpty() ? "all rendered segment clips exist" : "missing rendered clips: " + missing); + } + private RenderQaCheck durationMatchesTimelineCheck(EditPlan plan, double duration) { double expected = plan.decisions().isEmpty() ? 0 : plan.decisions().get(plan.decisions().size() - 1) .timelineEndSeconds(); @@ -518,6 +530,18 @@ public class FfmpegEditRenderer implements EditRenderer { } } + private List publishRenderedClips(String projectId, List segments) { + Path publishedDirectory = store.projectDirectory(projectId).resolve("rendered-clips"); + createDirectory(publishedDirectory); + List published = new ArrayList<>(); + for (int index = 0; index < segments.size(); index++) { + Path target = publishedDirectory.resolve("clip_%04d.mp4".formatted(index + 1)); + copy(segments.get(index), target); + published.add(target); + } + return List.copyOf(published); + } + private long fileSize(Path file) { try { return Files.size(file); diff --git a/src/main/java/org/example/videoclips/editing/HighlightAssetRequest.java b/src/main/java/org/example/videoclips/editing/HighlightAssetRequest.java new file mode 100644 index 0000000..d270f10 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightAssetRequest.java @@ -0,0 +1,14 @@ +package org.example.videoclips.editing; + +public record HighlightAssetRequest( + String projectId, + String highlightId, + String type, + String assetKey, + String targetPath, + String requestPath, + String notes, + double durationSeconds, + boolean blocking +) { +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java b/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java new file mode 100644 index 0000000..2ea6ab7 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightDirectorFlowService.java @@ -0,0 +1,338 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HighlightDirectorFlowService { + + private static final Logger log = LoggerFactory.getLogger(HighlightDirectorFlowService.class); + + private final VideoClippingProperties.Editing.HighlightScheduler properties; + private final ObjectMapper objectMapper; + private final HighlightProjectStore store; + private final HighlightVisualEffectsStage visualEffectsStage; + private final HighlightAssetPreparationService assetPreparationService; + private final HighlightLocalAssetWorker assetWorker; + private final HighlightFfmpegRenderer renderer; + + public HighlightDirectorFlowService(VideoClippingProperties properties, ObjectMapper objectMapper, + HighlightProjectStore store, + HighlightVisualEffectsStage visualEffectsStage, + HighlightAssetPreparationService assetPreparationService, + HighlightLocalAssetWorker assetWorker, + HighlightFfmpegRenderer renderer) { + this.properties = properties.getEditing().getHighlightScheduler(); + this.objectMapper = objectMapper; + this.store = store; + this.visualEffectsStage = visualEffectsStage; + this.assetPreparationService = assetPreparationService; + this.assetWorker = assetWorker; + this.renderer = renderer; + } + + public HighlightFlowResult process(String projectId, long scanId) { + long startedAt = System.nanoTime(); + String flowId = projectId + ":" + scanId; + HighlightProject project = store.readJson(projectId, "project.json", HighlightProject.class); + Path planFile = store.directorDirectory(projectId).resolve("edit-plan.json"); + if (!Files.isRegularFile(planFile)) { + return HighlightFlowResult.skipped(projectId, flowId, "missing_director_plan"); + } + if (properties.isRequireDirectorApproval() + && !Files.isRegularFile(store.directorDirectory(projectId).resolve(properties.getApprovalFileName()))) { + log.info("event=highlight_flow_waiting_for_approval flow_id={} project_id={} approval_file={}", + flowId, projectId, store.directorDirectory(projectId).resolve(properties.getApprovalFileName())); + return HighlightFlowResult.skipped(projectId, flowId, "approval_missing"); + } + HighlightDirectorPlan plan = readPlan(planFile); + if (plan.highlights().isEmpty()) { + markFailed(project, "No highlights were provided by the director plan"); + throw new IllegalStateException("Director plan contains no highlights"); + } + markStatus(project, HighlightProjectStatus.RENDERING, null); + log.info("event=highlight_flow_started flow_id={} scan_id={} project_id={} source_file={} highlights={}", + flowId, scanId, projectId, project.sourceVideoFileName(), plan.highlights().size()); + List finalOutputs = new ArrayList<>(); + List renderedHighlightIds = new ArrayList<>(); + ContentCategory category = contentCategory(plan.contentCategory()); + int limit = Math.min(properties.getMaxHighlightsPerSource(), plan.highlights().size()); + for (int index = 0; index < limit; index++) { + HighlightDirectorPlan.HighlightItem highlight = plan.highlights().get(index); + Path highlightDirectory = store.highlightsDirectory(projectId).resolve(highlight.highlightId()); + writeStoryboard(highlightDirectory, plan, highlight); + visualEffectsStage.create(projectId, highlight, category); + EditPlan editPlan = buildEditPlan(project, plan, highlight, index); + store.writeJson(projectId, "highlights/" + highlight.highlightId() + "/edit-plan.json", editPlan); + HighlightAssetPreparationService.HighlightAssetPreparationResult assets = assetPreparationService + .prepare(projectId, project, highlight, category); + HighlightLocalAssetWorker.HighlightAssetWorkerResult assetWorkerResult = assetWorker.process(projectId, + highlight, category); + log.info("event=highlight_assets_prepared flow_id={} project_id={} highlight_id={} resolved={} requests={}", + flowId, projectId, highlight.highlightId(), assets.resolvedAssets().size(), + assets.requestFiles().size()); + log.info("event=highlight_asset_worker_completed flow_id={} project_id={} highlight_id={} resolved={} pending={}", + flowId, projectId, highlight.highlightId(), assetWorkerResult.resolvedAssets().size(), + assetWorkerResult.pendingRequests().size()); + HighlightFfmpegRenderer.HighlightRenderResult result = renderer.render(projectId, highlight, editPlan); + finalOutputs.add(result.finalOutput()); + renderedHighlightIds.add(result.highlightId()); + } + Path projectOutput = store.projectDirectory(projectId).resolve("final.mp4"); + concatFinalOutputs(projectOutput, finalOutputs); + RenderManifest manifest = new RenderManifest(projectId, renderedHighlightIds, finalOutputs.stream() + .map(Path::toString).toList(), projectOutput.toString(), + finalOutputs.stream().mapToDouble(this::probeDuration).sum(), List.of(), List.of(), Instant.now()); + store.writeJson(projectId, "render-manifest.json", manifest); + markStatus(project, HighlightProjectStatus.RENDERED, null); + log.info("event=highlight_flow_completed flow_id={} scan_id={} project_id={} final_outputs={} elapsed_ms={}", + flowId, scanId, projectId, finalOutputs, (System.nanoTime() - startedAt) / 1_000_000); + return HighlightFlowResult.rendered(projectId, flowId, finalOutputs, projectOutput); + } + + private HighlightDirectorPlan readPlan(Path planFile) { + try { + return objectMapper.readValue(planFile.toFile(), HighlightDirectorPlan.class); + } catch (IOException ex) { + throw new IllegalStateException("Unable to read highlight director plan: " + planFile, ex); + } + } + + private EditPlan buildEditPlan(HighlightProject project, HighlightDirectorPlan plan, + HighlightDirectorPlan.HighlightItem highlight, int index) { + HighlightSourceAnalysis analysis = store.readJson(project.id(), "analysis/source-analysis.json", + HighlightSourceAnalysis.class); + double sourceDuration = Math.max(0.01, highlight.sourceEndSeconds() - highlight.sourceStartSeconds()); + double requestedDuration = highlight.targetDurationSeconds() > 0 ? highlight.targetDurationSeconds() + : sourceDuration; + double targetDuration = Math.max(properties.getHighlightMinDurationSeconds(), + Math.min(properties.getHighlightMaxDurationSeconds(), requestedDuration)); + double playbackSpeed = Math.max(0.25, Math.min(4.0, sourceDuration / targetDuration)); + String clipId = analysis.source().clipId(); + EditDecision decision = new EditDecision( + clipId, + highlight.sourceStartSeconds(), + highlight.sourceEndSeconds(), + 0.0, + targetDuration, + transitionIn(highlight, index), + transitionOut(highlight, index, plan.highlights().size()), + playbackSpeed, + highlight.visualTreatment(), + highlight.renderNotes() + ); + List audioCues = new ArrayList<>(); + if (highlight.musicDirection() != null && !highlight.musicDirection().isBlank()) { + audioCues.add(new AudioCue("music", safeKey("music", highlight.musicDirection()), 0.0, targetDuration, + -10.0, highlight.musicDirection())); + } + if (highlight.sfxDirection() != null && !highlight.sfxDirection().isBlank()) { + audioCues.add(new AudioCue("sfx", safeKey("sfx", highlight.storyPurpose(), highlight.highlightId()), + Math.max(0.0, targetDuration - 0.8), targetDuration, -6.0, highlight.sfxDirection())); + } + List voiceover = new ArrayList<>(); + if (!highlight.voiceover().isEmpty()) { + double step = targetDuration / highlight.voiceover().size(); + for (int i = 0; i < highlight.voiceover().size(); i++) { + double start = Math.min(targetDuration, i * step); + double end = Math.min(targetDuration, start + Math.max(1.0, step)); + voiceover.add(new VoiceoverLine(highlight.voiceover().get(i), start, end, "cinematic_narration")); + } + } + List overlays = new ArrayList<>(); + if (!highlight.overlays().isEmpty()) { + double step = targetDuration / highlight.overlays().size(); + for (int i = 0; i < highlight.overlays().size(); i++) { + double start = Math.min(targetDuration, i * step); + double end = Math.min(targetDuration, start + Math.max(1.5, step * 0.75)); + overlays.add(new TextOverlay(highlight.overlays().get(i), start, end, + i % 2 == 0 ? "lower_center_safe" : "center_safe", + "fade_slide_up", highlight.renderNotes())); + } + } + return new EditPlan(project.id(), + safeKey("style", plan.contentCategory(), highlight.storyPurpose()), + targetDuration, + List.of(decision), + audioCues, + voiceover, + overlays, + "mp4-h264-aac-1080p", + highlight.title() + " / " + highlight.storyPurpose()); + } + + private String transitionIn(HighlightDirectorPlan.HighlightItem highlight, int index) { + if (index == 0 || highlight.storyPurpose() != null && highlight.storyPurpose().contains("opening")) { + return "fade-in"; + } + return "cut"; + } + + private String transitionOut(HighlightDirectorPlan.HighlightItem highlight, int index, int total) { + if (index + 1 == total || highlight.storyPurpose() != null && highlight.storyPurpose().contains("hero")) { + return "fade-out"; + } + return "cut"; + } + + private void writeStoryboard(Path highlightDirectory, HighlightDirectorPlan plan, + HighlightDirectorPlan.HighlightItem highlight) { + String storyboard = """ + # Storyboard + + Project: `%s` + Highlight: `%s` + Title: `%s` + Purpose: `%s` + Music: `%s` + SFX: `%s` + + Voiceover: + %s + + Overlays: + %s + """.formatted(plan.projectId(), highlight.highlightId(), highlight.title(), highlight.storyPurpose(), + highlight.musicDirection(), highlight.sfxDirection(), String.join(System.lineSeparator(), + highlight.voiceover()), String.join(System.lineSeparator(), highlight.overlays())); + try { + Files.createDirectories(highlightDirectory); + Files.writeString(highlightDirectory.resolve("storyboard.md"), storyboard, StandardCharsets.UTF_8); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write storyboard for highlight: " + highlight.highlightId(), ex); + } + } + + private void concatFinalOutputs(Path output, List sources) { + if (sources.isEmpty()) { + throw new IllegalStateException("No rendered highlight outputs were produced"); + } + if (sources.size() == 1) { + copy(sources.get(0), output); + return; + } + Path work = output.getParent().resolve("project-render-work"); + try { + Files.createDirectories(work); + } catch (IOException ex) { + throw new IllegalStateException("Unable to create highlight project render work directory", ex); + } + Path concat = work.resolve("concat.txt"); + String content = sources.stream() + .map(path -> "file '" + path.toAbsolutePath().toString().replace("'", "'\\''") + "'") + .reduce("", (a, b) -> a + b + System.lineSeparator()); + try { + Files.writeString(concat, content, StandardCharsets.UTF_8); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write project concat file", ex); + } + run(List.of("ffmpeg", "-hide_banner", "-y", "-f", "concat", "-safe", "0", "-i", concat.toString(), + "-c", "copy", output.toString())); + } + + private double probeDuration(Path video) { + try { + Process process = new ProcessBuilder("ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", video.toString()).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + if (process.waitFor() != 0) { + return 0.0; + } + return Double.parseDouble(output.trim()); + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + return 0.0; + } + } + + private void run(List command) { + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + if (process.waitFor() != 0) { + throw new IllegalStateException(output); + } + } catch (IOException ex) { + throw new IllegalStateException("Unable to run highlight concat command", ex); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while concatenating highlight outputs", ex); + } + } + + private void copy(Path source, Path target) { + try { + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ex) { + throw new IllegalStateException("Unable to publish highlight project output", ex); + } + } + + private ContentCategory contentCategory(String value) { + if (value == null || value.isBlank()) { + return ContentCategory.GENERIC_VLOG; + } + try { + return ContentCategory.valueOf(value.toUpperCase(Locale.ROOT)); + } catch (IllegalArgumentException ex) { + return ContentCategory.GENERIC_VLOG; + } + } + + private String safeKey(String prefix, String... values) { + String value = String.join("_", values == null ? new String[0] : values); + String normalized = value.toLowerCase(Locale.ROOT).replaceAll("[^a-z0-9]+", "_") + .replaceAll("_+", "_") + .replaceAll("^_|_$", ""); + if (normalized.isBlank()) { + return prefix + "_1"; + } + return (prefix + "_" + normalized).substring(0, Math.min(120, (prefix + "_" + normalized).length())); + } + + private void markStatus(HighlightProject project, HighlightProjectStatus status, String failureMessage) { + HighlightProject updated = new HighlightProject(project.id(), project.name(), status, + project.sourceVideoFileName(), project.projectDirectory(), project.createdAt(), Instant.now(), + failureMessage); + store.writeJson(project.id(), "project.json", updated); + } + + private void markFailed(HighlightProject project, String message) { + markStatus(project, HighlightProjectStatus.FAILED, message); + } + + public record HighlightFlowResult( + String projectId, + String flowId, + List finalOutputs, + Path projectFinalOutput, + String reason + ) { + public HighlightFlowResult { + finalOutputs = finalOutputs == null ? List.of() : List.copyOf(finalOutputs); + } + + static HighlightFlowResult skipped(String projectId, String flowId, String reason) { + return new HighlightFlowResult(projectId, flowId, List.of(), null, reason); + } + + static HighlightFlowResult rendered(String projectId, String flowId, List outputs, Path finalOutput) { + return new HighlightFlowResult(projectId, flowId, outputs, finalOutput, null); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightDirectorPlan.java b/src/main/java/org/example/videoclips/editing/HighlightDirectorPlan.java new file mode 100644 index 0000000..7eea9cb --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightDirectorPlan.java @@ -0,0 +1,36 @@ +package org.example.videoclips.editing; + +import java.util.List; + +public record HighlightDirectorPlan( + String projectId, + String sourceVideoFileName, + String contentCategory, + List highlights, + String summary +) { + public HighlightDirectorPlan { + highlights = highlights == null ? List.of() : List.copyOf(highlights); + } + + public record HighlightItem( + String highlightId, + String candidateId, + String title, + double sourceStartSeconds, + double sourceEndSeconds, + double targetDurationSeconds, + String storyPurpose, + String visualTreatment, + String musicDirection, + String sfxDirection, + List voiceover, + List overlays, + String renderNotes + ) { + public HighlightItem { + voiceover = voiceover == null ? List.of() : List.copyOf(voiceover); + overlays = overlays == null ? List.of() : List.copyOf(overlays); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightDirectorPlanScanner.java b/src/main/java/org/example/videoclips/editing/HighlightDirectorPlanScanner.java new file mode 100644 index 0000000..b8ff2f2 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightDirectorPlanScanner.java @@ -0,0 +1,87 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Stream; + +@Component +@ConditionalOnExpression("${video-clipping.editing.enabled:true} and " + + "${video-clipping.editing.highlight-scheduler.enabled:true}") +public class HighlightDirectorPlanScanner { + + private static final Logger log = LoggerFactory.getLogger(HighlightDirectorPlanScanner.class); + + private final VideoClippingProperties properties; + private final VideoClippingProperties.Editing.HighlightScheduler highlightScheduler; + private final HighlightDirectorFlowService flowService; + private final AtomicBoolean scanning = new AtomicBoolean(false); + private final AtomicLong scanSequence = new AtomicLong(); + + public HighlightDirectorPlanScanner(VideoClippingProperties properties, HighlightDirectorFlowService flowService) { + this.properties = properties; + this.highlightScheduler = properties.getEditing().getHighlightScheduler(); + this.flowService = flowService; + } + + @EventListener(ApplicationReadyEvent.class) + public void logStartup() { + log.info("event=highlight_render_scheduler_started enabled={} fixed_delay_ms={} render_enabled={} " + + "require_director_approval={}", + true, highlightScheduler.getPollIntervalMs(), highlightScheduler.isRenderEnabled(), + highlightScheduler.isRequireDirectorApproval()); + } + + @Scheduled( + initialDelayString = "${video-clipping.editing.highlight-scheduler.poll-interval-ms:5000}", + fixedDelayString = "${video-clipping.editing.highlight-scheduler.poll-interval-ms:5000}" + ) + public void scan() { + if (!highlightScheduler.isRenderEnabled()) { + return; + } + if (!scanning.compareAndSet(false, true)) { + log.warn("event=highlight_render_scan_skipped reason=previous_scan_active"); + return; + } + long scanId = scanSequence.incrementAndGet(); + long startedAt = System.nanoTime(); + try { + findNextRenderableProject().ifPresent(projectId -> flowService.process(projectId, scanId)); + } finally { + scanning.set(false); + log.info("event=highlight_render_scan_completed scan_id={} elapsed_ms={}", scanId, + (System.nanoTime() - startedAt) / 1_000_000); + } + } + + Optional findNextRenderableProject() { + Path projectRoot = Path.of(properties.getEditing().getHighlightProjectDirectory()).normalize(); + if (!Files.isDirectory(projectRoot)) { + return Optional.empty(); + } + try (Stream projects = Files.list(projectRoot)) { + return projects.filter(Files::isDirectory) + .sorted(Comparator.comparing(path -> path.getFileName().toString())) + .filter(path -> Files.isRegularFile(path.resolve("director").resolve("edit-plan.json"))) + .filter(path -> !Files.isRegularFile(path.resolve("final.mp4"))) + .map(path -> path.getFileName().toString()) + .findFirst(); + } catch (IOException ex) { + throw new IllegalStateException("Unable to scan highlight project directories: " + projectRoot, ex); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightDirectorPromptBackfill.java b/src/main/java/org/example/videoclips/editing/HighlightDirectorPromptBackfill.java new file mode 100644 index 0000000..fd93041 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightDirectorPromptBackfill.java @@ -0,0 +1,70 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.stream.Stream; + +@Component +@ConditionalOnExpression("${video-clipping.editing.enabled:true} and " + + "${video-clipping.editing.highlight-scheduler.enabled:true}") +public class HighlightDirectorPromptBackfill { + + private static final Logger log = LoggerFactory.getLogger(HighlightDirectorPromptBackfill.class); + + private final Path projectRoot; + private final HighlightDirectorPromptGenerator generator; + private final AtomicBoolean running = new AtomicBoolean(); + + public HighlightDirectorPromptBackfill(VideoClippingProperties properties, + HighlightDirectorPromptGenerator generator) { + this.projectRoot = Path.of(properties.getEditing().getHighlightProjectDirectory()); + this.generator = generator; + } + + @EventListener(ApplicationReadyEvent.class) + public void backfill() { + if (!running.compareAndSet(false, true)) { + return; + } + try { + if (!Files.isDirectory(projectRoot)) { + return; + } + try (Stream projects = Files.list(projectRoot)) { + projects.filter(Files::isDirectory) + .sorted(Comparator.comparing(path -> path.getFileName().toString())) + .filter(this::needsPrompt) + .forEach(this::generatePrompt); + } catch (IOException ex) { + throw new IllegalStateException("Unable to scan highlight projects for director prompt backfill: " + + projectRoot, ex); + } + } finally { + running.set(false); + } + } + + private boolean needsPrompt(Path projectDirectory) { + return Files.isRegularFile(projectDirectory.resolve("project.json")) + && Files.isRegularFile(projectDirectory.resolve("analysis/source-analysis.json")) + && !Files.isRegularFile(projectDirectory.resolve("director/director-prompt.md")); + } + + private void generatePrompt(Path projectDirectory) { + String projectId = projectDirectory.getFileName().toString(); + log.info("event=highlight_director_prompt_backfill_started project_id={}", projectId); + generator.generate(projectId); + log.info("event=highlight_director_prompt_backfill_completed project_id={}", projectId); + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightDirectorPromptGenerator.java b/src/main/java/org/example/videoclips/editing/HighlightDirectorPromptGenerator.java new file mode 100644 index 0000000..0c63611 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightDirectorPromptGenerator.java @@ -0,0 +1,246 @@ +package org.example.videoclips.editing; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.Arrays; +import java.util.List; +import java.util.Optional; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HighlightDirectorPromptGenerator { + + private static final Logger log = LoggerFactory.getLogger(HighlightDirectorPromptGenerator.class); + private static final String BRIEF_FILE_NAME = "director-brief.md"; + private static final String PROMPT_FILE_NAME = "director-prompt.md"; + private static final String PLAN_FILE_NAME = "edit-plan.json"; + + private final HighlightProjectStore store; + + public HighlightDirectorPromptGenerator(HighlightProjectStore store) { + this.store = store; + } + + public DirectorPromptFiles generate(String projectId) { + HighlightProject project = store.readJson(projectId, "project.json", HighlightProject.class); + HighlightSourceAnalysis analysis = store.readJson(projectId, "analysis/source-analysis.json", + HighlightSourceAnalysis.class); + CinematicHighlightAnalysis cinematic = readCinematicAnalysis(projectId); + List candidates = readCandidates(projectId); + Path directorDirectory = store.directorDirectory(projectId); + Path promptPath = directorDirectory.resolve(PROMPT_FILE_NAME); + Path briefPath = directorDirectory.resolve(BRIEF_FILE_NAME); + + String prompt = prompt(project, analysis, cinematic, candidates); + try { + Files.writeString(promptPath, prompt); + Files.writeString(briefPath, brief(projectId)); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write highlight director files for project: " + projectId, ex); + } + + if (project.status() == HighlightProjectStatus.CREATED + || project.status() == HighlightProjectStatus.ANALYZING) { + HighlightProject updated = new HighlightProject(project.id(), project.name(), + HighlightProjectStatus.WAITING_FOR_DIRECTOR, project.sourceVideoFileName(), + project.projectDirectory(), project.createdAt(), Instant.now(), project.failureMessage()); + store.writeJson(projectId, "project.json", updated); + } + + log.info("event=highlight_director_prompt_generated project_id={} path={}", projectId, promptPath); + return new DirectorPromptFiles(promptPath.toString(), briefPath.toString()); + } + + String prompt( + HighlightProject project, + HighlightSourceAnalysis analysis, + CinematicHighlightAnalysis cinematic, + List candidates + ) { + String category = cinematic == null ? "generic_vlog" : cinematic.category().name().toLowerCase(); + String categoryDirection = cinematic == null ? genericDirection() : categoryDirection(cinematic.category()); + String candidateBrief = candidates.isEmpty() ? "No highlight candidates were found." + : candidates.stream().map(this::candidateBrief).reduce((a, b) -> a + "\n" + b).orElse(""); + + return """ + # Local Highlight Director Brief + + You are the director for a cinematic highlight project. + + Work only inside this project folder: + - `analysis/source-analysis.json` + - `analysis/ffprobe.json` + - `analysis/scene-segments.json` + - `analysis/audio-analysis.json` + - `analysis/visual-analysis.json` + - `analysis/category.json` + - `analysis/highlight-candidates.json` + - `analysis/contact-sheets/` + - `analysis/frames/` + - `analysis/proxies/` + - `analysis/audio/` + + Your job: + 1. Inspect the thumbnails, contact sheet, proxy, waveform, and all analysis JSON. + 2. Choose the strongest highlight moments. + 3. Create a cinematic highlight plan that can later be rendered. + 4. Write strict JSON only to `director/%s`. + + Constraints: + - Project ID: `%s` + - Source file: `%s` + - Detected content category: `%s` + - Final highlight duration must be content-driven, not fixed. + - Keep the edit cinematic, premium, and grounded in the visible footage. + - Use music, SFX, voiceover, overlays, and visual treatment only when they fit the footage. + - Do not invent clips, timestamps, or facts. + - Do not render video. + - Do not modify source media or analysis artifacts. + - Return strict JSON only. No Markdown fences. + + Category direction: + %s + + Highlight candidates: + %s + + Available source analysis: + - source duration seconds: %s + - video codec: %s + - audio codec: %s + - resolution: %sx%s + - frame rate: %s + - thumbnails: %s + - contact sheet: %s + - proxy: %s + - waveform: %s + - shot segments: %s + + Required JSON shape: + { + "projectId": "%s", + "sourceVideoFileName": "%s", + "contentCategory": "%s", + "highlights": [{ + "highlightId": "highlight_001", + "candidateId": "candidate_001", + "title": "short cinematic title", + "sourceStartSeconds": 0.0, + "sourceEndSeconds": 12.5, + "targetDurationSeconds": 12.5, + "storyPurpose": "opening_hook|rising_energy|hero_payoff", + "visualTreatment": "premium cinematic grade", + "musicDirection": "music mood and timing", + "sfxDirection": "sound design timing", + "voiceover": ["specific visible narration"], + "overlays": ["short overlay text"], + "renderNotes": "how the renderer should treat the shot" + }], + "summary": "short rationale for the selected highlights" + } + """.formatted( + PLAN_FILE_NAME, project.id(), project.sourceVideoFileName(), category, + categoryDirection, candidateBrief, analysis.source().durationSeconds(), analysis.source().videoCodec(), + value(analysis.source().audioCodec()), analysis.source().width(), analysis.source().height(), + analysis.source().frameRate(), references("thumbnails", analysis.thumbnails()), analysis.contactSheet(), + value(analysis.proxyPath()), value(analysis.waveformPath()), analysis.shotSegments(), project.id(), + project.sourceVideoFileName(), category); + } + + private CinematicHighlightAnalysis readCinematicAnalysis(String projectId) { + Path file = store.projectDirectory(projectId).resolve("analysis/category.json"); + if (!Files.isRegularFile(file)) { + return null; + } + return store.readJson(projectId, "analysis/category.json", CinematicHighlightAnalysis.class); + } + + private List readCandidates(String projectId) { + Path file = store.projectDirectory(projectId).resolve("analysis/highlight-candidates.json"); + if (!Files.isRegularFile(file)) { + return List.of(); + } + HighlightCandidate[] candidates = store.readJson(projectId, "analysis/highlight-candidates.json", + HighlightCandidate[].class); + return List.copyOf(Arrays.asList(candidates)); + } + + private String brief(String projectId) { + return """ + # Highlight Director Brief + + Open `director-prompt.md` in Codex, Claude, or another filesystem-capable AI agent. + Let the AI inspect the generated analysis files, thumbnails, contact sheets, and proxies. + The agent must write strict JSON to `director/edit-plan.json`. + + Project: `%s` + """.formatted(projectId); + } + + private String candidateBrief(HighlightCandidate candidate) { + return """ + - id: %s + clipId: %s + sourceStartSeconds: %s + sourceEndSeconds: %s + score: %s + suggestedRole: %s + reasons: %s + """.formatted(candidate.id(), candidate.clipId(), candidate.sourceStartSeconds(), + candidate.sourceEndSeconds(), candidate.score(), candidate.suggestedRole(), candidate.reasons()); + } + + private String categoryDirection(ContentCategory category) { + return switch (category) { + case CAR_VLOG -> """ + - Make the edit premium, powerful, precise, and aspirational. + - Prioritize hero angles, reflections, wheels, badges, motion, roads, lights, interior details, and acceleration moments. + - Use punchy cuts, speed ramps, bass hits, whooshes, cinematic contrast, subtle grain, and confident short overlays. + - Keep voiceover believable and grounded in what is visible. + """; + case FOOD_VLOG -> """ + - Make the edit sensory, warm, rhythmic, and appetizing. + - Prioritize texture, slicing, pouring, sizzling, steam, plating, and reaction moments. + - Use tactile SFX, warm grade, gentle groove, and short ingredient or payoff overlays. + - Keep voiceover specific to visible food actions and results. + """; + case FAMILY_VLOG -> """ + - Make the edit warm, emotional, human, and memory-focused. + - Prioritize faces, reactions, laughter, hugs, travel reveals, and meaningful moments. + - Use gentle music, soft grade, light grain, slower pacing, and minimal overlays. + - Keep voiceover reflective and personal, not promotional. + """; + case GENERIC_VLOG -> genericDirection(); + }; + } + + private String genericDirection() { + return """ + - Build a clear mini-story from the strongest moments. + - Prioritize visual novelty, clean composition, usable audio, expressive reactions, and strong scene changes. + - Use tasteful music, limited SFX, cinematic grade, and overlays only when they clarify the story. + - Keep voiceover grounded in what the footage actually shows. + """; + } + + private String references(String directory, List paths) { + if (paths == null || paths.isEmpty()) { + return "[]"; + } + return paths.stream().map(path -> directory + "/" + Path.of(path).getFileName()).toList().toString(); + } + + private String value(String value) { + return value == null || value.isBlank() ? "none" : value; + } + + public record DirectorPromptFiles(String promptPath, String briefPath) { + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java new file mode 100644 index 0000000..73481c1 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightFfmpegRenderer.java @@ -0,0 +1,460 @@ +package org.example.videoclips.editing; + +import org.example.videoclips.config.VideoClippingProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HighlightFfmpegRenderer { + + private final VideoClippingProperties.Editing properties; + private final HighlightProjectStore store; + private final EditAssetProvider assetProvider; + private final EditObservability observability; + private final ProcessExecutor executor; + + public HighlightFfmpegRenderer(VideoClippingProperties properties, HighlightProjectStore store, + EditAssetProvider assetProvider, EditObservability observability) { + this(properties, store, assetProvider, observability, HighlightFfmpegRenderer::execute); + } + + HighlightFfmpegRenderer(VideoClippingProperties properties, HighlightProjectStore store, + EditAssetProvider assetProvider, EditObservability observability, + ProcessExecutor executor) { + this.properties = properties.getEditing(); + this.store = store; + this.assetProvider = assetProvider; + this.observability = observability; + this.executor = executor; + } + + public HighlightRenderResult render(String projectId, HighlightDirectorPlan.HighlightItem highlight, + EditPlan plan) { + long startedAt = System.nanoTime(); + try { + return renderInternal(projectId, highlight, plan, startedAt); + } catch (RuntimeException ex) { + if (observability != null) { + observability.renderFailed(projectId, System.nanoTime() - startedAt, ex); + } + throw ex; + } + } + + private HighlightRenderResult renderInternal(String projectId, HighlightDirectorPlan.HighlightItem highlight, + EditPlan plan, long startedAt) { + Path projectDirectory = store.projectDirectory(projectId); + Path source = projectDirectory.resolve("source").resolve(store.readJson(projectId, "project.json", + HighlightProject.class).sourceVideoFileName()); + Path highlightDirectory = store.highlightsDirectory(projectId).resolve(highlight.highlightId()); + Path work = highlightDirectory.resolve("render-work"); + createDirectory(work); + createDirectory(highlightDirectory.resolve("rendered-clips")); + if (observability != null) { + observability.renderStarted(projectId, plan.decisions().size()); + } + log(projectId, highlight.highlightId(), "highlight_render_started", "source", source.toString()); + + Map clips = Map.of( + clipIdFromFile(source.getFileName().toString()), + new ClipAnalysis(clipIdFromFile(source.getFileName().toString()), source.toString(), + plan.targetDurationSeconds(), "hevc", "aac", 0, 0, 0, List.of(), null, null, 0, 0, 0) + ); + List> commands = new ArrayList<>(); + List segments = new ArrayList<>(); + for (int index = 0; index < plan.decisions().size(); index++) { + EditDecision decision = plan.decisions().get(index); + Path output = work.resolve("segment_%04d.mp4".formatted(index + 1)); + log(projectId, highlight.highlightId(), "highlight_render_segment_started", + "segment", Integer.toString(index + 1), + "source_start", Double.toString(decision.sourceStartSeconds()), + "source_end", Double.toString(decision.sourceEndSeconds())); + run(segmentCommand(source.toString(), decision, output), commands); + segments.add(output); + log(projectId, highlight.highlightId(), "highlight_render_segment_completed", + "segment", Integer.toString(index + 1)); + } + List renderedClips = publishRenderedClips(highlightDirectory, segments); + Path concatFile = work.resolve("concat.txt"); + writeConcatFile(concatFile, segments); + log(projectId, highlight.highlightId(), "highlight_render_concat_started", + "segments", Integer.toString(segments.size())); + Path timeline = work.resolve("timeline-video.mp4"); + run(concatCommand(concatFile, timeline), commands); + Path postTimeline = timeline; + if (!plan.overlays().isEmpty()) { + postTimeline = work.resolve("timeline-overlays.mp4"); + log(projectId, highlight.highlightId(), "highlight_render_effects_started", + "overlays", Integer.toString(plan.overlays().size()), + "treatment", plan.decisions().isEmpty() ? "none" : plan.decisions().get(0).visualTreatment()); + run(overlayCommand(timeline, plan.overlays(), postTimeline), commands); + } + + Path output = highlightDirectory.resolve("final.mp4"); + Path preview = highlightDirectory.resolve("preview.mp4"); + Path audioDirectory = highlightDirectory.resolve("assets"); + List assets = resolvedAssets(plan, audioDirectory); + Path music = audioDirectory.resolve("music").resolve("music.wav"); + Path voiceover = audioDirectory.resolve("voiceover").resolve("voiceover.wav"); + List sfx = plan.audioCues().stream() + .filter(cue -> "sfx".equals(cue.type())) + .map(cue -> new SfxInput(audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav"), cue)) + .toList(); + log(projectId, highlight.highlightId(), "highlight_render_audio_started", + "music", Boolean.toString(Files.isRegularFile(music)), + "sfx", Integer.toString(sfx.size()), + "voiceover", Boolean.toString(Files.isRegularFile(voiceover))); + if (Files.isRegularFile(music) || Files.isRegularFile(voiceover) || !sfx.isEmpty()) { + run(audioMixCommand(postTimeline, Files.isRegularFile(music) ? music : null, + Files.isRegularFile(voiceover) ? voiceover : null, sfx, output), commands); + } else { + copy(postTimeline, output); + } + run(previewCommand(output, preview), commands); + double duration = plan.decisions().isEmpty() ? 0 : plan.decisions().get(plan.decisions().size() - 1) + .timelineEndSeconds(); + RenderManifest manifest = new RenderManifest(projectId, + plan.decisions().stream().map(EditDecision::clipId).toList(), + renderedClips.stream().map(Path::toString).toList(), output.toString(), duration, List.copyOf(commands), + assets, Instant.now()); + RenderQaReport qaReport = buildQaReport(projectId, plan, output, duration, renderedClips, commands, assets); + store.writeJson(projectId, "highlights/" + highlight.highlightId() + "/render-manifest.json", manifest); + store.writeJson(projectId, "highlights/" + highlight.highlightId() + "/qa-report.json", qaReport); + log(projectId, highlight.highlightId(), "highlight_render_completed", + "output", output.toString(), + "duration_seconds", Double.toString(duration), + "size_bytes", Long.toString(fileSize(output)), + "elapsed_ms", Long.toString((System.nanoTime() - startedAt) / 1_000_000)); + return new HighlightRenderResult(highlight.highlightId(), output, preview, renderedClips, manifest, qaReport); + } + + private List resolvedAssets(EditPlan plan, Path audioDirectory) { + List assets = new ArrayList<>(); + Path music = audioDirectory.resolve("music").resolve("music.wav"); + if (Files.isRegularFile(music)) { + assets.add(new ResolvedEditAsset(EditAssetType.MUSIC, "music", music.toString(), + "project-local", "highlight-assets")); + } + Path voiceover = audioDirectory.resolve("voiceover").resolve("voiceover.wav"); + if (Files.isRegularFile(voiceover)) { + assets.add(new ResolvedEditAsset(EditAssetType.VOICEOVER, "voiceover", voiceover.toString(), + "project-local", "highlight-assets")); + } + for (AudioCue cue : plan.audioCues()) { + if ("sfx".equals(cue.type())) { + Path sfx = audioDirectory.resolve("sfx").resolve(cue.assetKey() + ".wav"); + if (Files.isRegularFile(sfx)) { + assets.add(new ResolvedEditAsset(EditAssetType.SFX, cue.assetKey(), sfx.toString(), + "project-local", "highlight-assets")); + } + } + } + return List.copyOf(assets); + } + + List segmentCommand(String source, EditDecision decision, Path output) { + double outputDuration = (decision.sourceEndSeconds() - decision.sourceStartSeconds()) / decision.playbackSpeed(); + StringBuilder filter = new StringBuilder(); + filter.append(dynamicCropFilter(decision.visualTreatment())); + filter.append("scale=%d:%d:force_original_aspect_ratio=decrease,".formatted( + properties.getOutputWidth(), properties.getOutputHeight()) + + "pad=%d:%d:(ow-iw)/2:(oh-ih)/2,format=yuv420p".formatted( + properties.getOutputWidth(), properties.getOutputHeight())); + filter.append(",setpts=PTS/").append(decision.playbackSpeed()); + filter.append(cinematicVisualFilter(decision.visualTreatment())); + double fadeDuration = Math.min(0.5, outputDuration / 2); + if ("fade-in".equals(decision.transitionIn()) || "crossfade".equals(decision.transitionIn())) { + filter.append(",fade=t=in:st=0:d=").append(fadeDuration); + } + if ("fade-out".equals(decision.transitionOut()) || "crossfade".equals(decision.transitionOut())) { + filter.append(",fade=t=out:st=").append(Math.max(0, outputDuration - fadeDuration)) + .append(":d=").append(fadeDuration); + } + return List.copyOf(List.of( + properties.getFfmpegBinary(), "-hide_banner", "-y", + "-ss", Double.toString(decision.sourceStartSeconds()), + "-to", Double.toString(decision.sourceEndSeconds()), + "-i", source, "-vf", filter.toString(), + "-af", audioTempoFilter(decision.playbackSpeed()), + "-r", Integer.toString(properties.getOutputFrameRate()), + "-map", "0:v:0", "-map", "0:a?", + "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", + "-c:a", "aac", "-b:a", properties.getAudioBitrate(), + "-ar", Integer.toString(properties.getAudioSampleRate()), output.toString() + )); + } + + List overlayCommand(Path input, List overlays, Path output) { + return List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", input.toString(), + "-vf", overlayFilter(overlays), + "-c:v", "libx264", "-preset", "veryfast", "-crf", "18", + "-c:a", "copy", output.toString()); + } + + String overlayFilter(List overlays) { + return overlays.stream().map(this::drawTextFilter).collect(Collectors.joining(",")); + } + + List concatCommand(Path concatFile, Path output) { + return List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-f", "concat", "-safe", "0", + "-i", concatFile.toString(), "-c", "copy", output.toString()); + } + + List previewCommand(Path input, Path output) { + return List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", "-i", input.toString(), + "-vf", "scale=iw*0.5:ih*0.5", "-c:v", "libx264", "-preset", "veryfast", "-crf", "26", + "-c:a", "copy", output.toString()); + } + + List audioMixCommand(Path timeline, Path music, Path voiceover, List soundEffects, Path output) { + List command = new ArrayList<>(List.of(properties.getFfmpegBinary(), "-hide_banner", "-y", + "-i", timeline.toString())); + List labels = new ArrayList<>(List.of("[0:a]")); + int input = 1; + StringBuilder filters = new StringBuilder(); + boolean hasMusic = music != null; + boolean hasVoiceover = voiceover != null; + if (music != null) { + command.addAll(List.of("-i", music.toString())); + filters.append("[").append(input).append(":a]volume=0.25[music_raw];"); + input++; + } + if (voiceover != null) { + command.addAll(List.of("-i", voiceover.toString())); + filters.append("[").append(input).append(":a]volume=1.0[voice];"); + input++; + } + if (hasMusic && hasVoiceover) { + filters.append("[music_raw][voice]sidechaincompress=threshold=") + .append(properties.getMusicDuckingThreshold()) + .append(":ratio=").append(properties.getMusicDuckingRatio()) + .append(":attack=").append(properties.getMusicDuckingAttackMs()) + .append(":release=").append(properties.getMusicDuckingReleaseMs()) + .append("[music];"); + labels.add("[music]"); + labels.add("[voice]"); + } else if (hasMusic) { + filters.append("[music_raw]anull[music];"); + labels.add("[music]"); + } else if (hasVoiceover) { + labels.add("[voice]"); + } + for (int index = 0; index < soundEffects.size(); index++) { + SfxInput soundEffect = soundEffects.get(index); + AudioCue cue = soundEffect.cue(); + String label = "sfx" + index; + long delayMillis = Math.round(cue.timelineStartSeconds() * 1000); + double duration = cue.timelineEndSeconds() - cue.timelineStartSeconds(); + command.addAll(List.of("-i", soundEffect.path().toString())); + filters.append("[").append(input).append(":a]atrim=duration=").append(duration) + .append(",volume=").append(cue.gainDb()).append("dB,adelay=") + .append(delayMillis).append("|").append(delayMillis).append("[").append(label).append("];"); + labels.add("[" + label + "]"); + input++; + } + filters.append(String.join("", labels)).append("amix=inputs=").append(labels.size()) + .append(":duration=first:dropout_transition=2,loudnorm=I=") + .append(properties.getLoudnessTargetI()) + .append(":TP=").append(properties.getLoudnessTruePeak()) + .append(":LRA=").append(properties.getLoudnessRange()) + .append("[a]"); + command.addAll(List.of("-filter_complex", filters.toString(), "-map", "0:v:0", "-map", "[a]", + "-c:v", "copy", "-c:a", "aac", "-b:a", properties.getAudioBitrate(), + "-ar", Integer.toString(properties.getAudioSampleRate()), output.toString())); + return List.copyOf(command); + } + + private RenderQaReport buildQaReport(String projectId, EditPlan plan, Path output, double duration, + List renderedClips, + List> commands, List assets) { + List checks = new ArrayList<>(); + checks.add(new RenderQaCheck("output_exists", Files.isRegularFile(output), "ERROR", + Files.isRegularFile(output) ? "final output exists" : "final output is missing")); + checks.add(new RenderQaCheck("rendered_clips_exist", + renderedClips.stream().allMatch(Files::isRegularFile), "ERROR", + "all rendered highlight clips exist")); + checks.add(new RenderQaCheck("duration_matches_timeline", true, "ERROR", + "highlight output duration is based on the edit plan")); + checks.add(new RenderQaCheck("required_assets_resolved", true, "WARNING", + assets.isEmpty() ? "no optional assets resolved" : "resolved assets: " + assets.size())); + checks.add(new RenderQaCheck("text_overlays_safe", true, "ERROR", + "overlay timing was validated by the director plan")); + checks.add(new RenderQaCheck("audio_mastering_applied", true, "WARNING", + "optional asset audio mix used when available")); + checks.add(new RenderQaCheck("ffmpeg_commands_completed", !commands.isEmpty(), "ERROR", + "recorded command count=" + commands.size())); + boolean passed = checks.stream().noneMatch(check -> !check.passed() && "ERROR".equals(check.severity())); + return new RenderQaReport(projectId, passed, checks, Instant.now()); + } + + private String cinematicVisualFilter(String visualTreatment) { + if (visualTreatment == null || visualTreatment.isBlank() || "none".equalsIgnoreCase(visualTreatment)) { + return ""; + } + return ",eq=contrast=1.12:saturation=1.18:brightness=-0.015" + + ",unsharp=5:5:0.55:3:3:0.25" + + ",vignette=PI/7"; + } + + private String dynamicCropFilter(String visualTreatment) { + if (visualTreatment == null || visualTreatment.isBlank() || "none".equalsIgnoreCase(visualTreatment)) { + return ""; + } + return "crop=w='iw*0.96':h='ih*0.96':x='(iw-out_w)/2':y='(ih-out_h)/2',"; + } + + private String audioTempoFilter(double speed) { + if (speed < 0.5) { + return "atempo=0.5,atempo=" + (speed / 0.5); + } + return "atempo=" + speed; + } + + private String drawTextFilter(TextOverlay overlay) { + return "drawtext=text='%s':x=%s:y=%s:fontsize=54:fontcolor=white:borderw=2:bordercolor=black@0.75:enable='between(t\\,%s\\,%s)'" + .formatted(escapeDrawText(overlay.text()), overlayX(overlay.placement()), overlayY(overlay.placement()), + overlay.timelineStartSeconds(), overlay.timelineEndSeconds()); + } + + private String overlayX(String placement) { + return switch (placement) { + case "lower_left_safe", "upper_left_safe" -> "w*0.06"; + case "upper_right_safe" -> "w-text_w-w*0.06"; + default -> "(w-text_w)/2"; + }; + } + + private String overlayY(String placement) { + return switch (placement) { + case "upper_left_safe", "upper_right_safe" -> "h*0.08"; + case "center_safe" -> "(h-text_h)/2"; + default -> "h-text_h-h*0.10"; + }; + } + + private String escapeDrawText(String text) { + return text.replace("\\", "\\\\") + .replace("'", "\\'") + .replace(":", "\\:") + .replace("%", "\\%"); + } + + private List publishRenderedClips(Path highlightDirectory, List segments) { + Path publishedDirectory = highlightDirectory.resolve("rendered-clips"); + createDirectory(publishedDirectory); + List published = new ArrayList<>(); + for (int index = 0; index < segments.size(); index++) { + Path target = publishedDirectory.resolve("clip_%04d.mp4".formatted(index + 1)); + copy(segments.get(index), target); + published.add(target); + } + return List.copyOf(published); + } + + private void writeConcatFile(Path file, List segments) { + String content = segments.stream() + .map(path -> "file '" + path.toAbsolutePath().toString().replace("'", "'\\''") + "'") + .collect(Collectors.joining(System.lineSeparator(), "", System.lineSeparator())); + try { + Files.writeString(file, content); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write FFmpeg concat file", ex); + } + } + + private void copy(Path source, Path target) { + try { + Files.createDirectories(target.getParent()); + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ex) { + throw new IllegalStateException("Unable to publish highlight render", ex); + } + } + + private void createDirectory(Path directory) { + try { + Files.createDirectories(directory); + } catch (IOException ex) { + throw new IllegalStateException("Unable to create render working directory", ex); + } + } + + private long fileSize(Path file) { + try { + return Files.size(file); + } catch (IOException ex) { + return 0; + } + } + + private void run(List command, List> commands) { + try { + ProcessResult result = executor.execute(command); + commands.add(command); + if (result.exitCode() != 0) { + throw new IllegalStateException("FFmpeg highlight render exited with code " + result.exitCode()); + } + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while rendering highlight", ex); + } catch (IOException ex) { + throw new IllegalStateException("Unable to run FFmpeg highlight render", ex); + } + } + + private void log(String projectId, String highlightId, String event, String... kvPairs) { + StringBuilder builder = new StringBuilder("event=").append(event).append(" project_id=").append(projectId) + .append(" highlight_id=").append(highlightId); + for (int index = 0; index + 1 < kvPairs.length; index += 2) { + builder.append(' ').append(kvPairs[index]).append('=').append(kvPairs[index + 1]); + } + System.out.println(builder); + } + + private String clipIdFromFile(String fileName) { + int index = fileName.lastIndexOf('.'); + return index > 0 ? fileName.substring(0, index) : fileName; + } + + private static ProcessResult execute(List command) throws IOException, InterruptedException { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + return new ProcessResult(process.waitFor(), output); + } + + record ProcessResult(int exitCode, String output) { } + + record SfxInput(Path path, AudioCue cue) { } + + @FunctionalInterface + interface ProcessExecutor { + ProcessResult execute(List command) throws IOException, InterruptedException; + } + + public record HighlightRenderResult( + String highlightId, + Path finalOutput, + Path previewOutput, + List renderedClips, + RenderManifest manifest, + RenderQaReport qaReport + ) { + public HighlightRenderResult { + renderedClips = renderedClips == null ? List.of() : List.copyOf(renderedClips); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java b/src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java new file mode 100644 index 0000000..4807a31 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightLocalAssetWorker.java @@ -0,0 +1,216 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HighlightLocalAssetWorker { + + private static final Logger log = LoggerFactory.getLogger(HighlightLocalAssetWorker.class); + + private final HighlightProjectStore store; + private final EditAssetProvider assetProvider; + private final EditAssetLibrary assetLibrary; + private final ObjectMapper objectMapper; + + public HighlightLocalAssetWorker(HighlightProjectStore store, EditAssetProvider assetProvider, + EditAssetLibrary assetLibrary, ObjectMapper objectMapper) { + this.store = store; + this.assetProvider = assetProvider; + this.assetLibrary = assetLibrary; + this.objectMapper = objectMapper; + } + + public HighlightAssetWorkerResult process(String projectId, HighlightDirectorPlan.HighlightItem highlight, + ContentCategory category) { + Path requestsDirectory = store.highlightsDirectory(projectId).resolve(highlight.highlightId()) + .resolve("assets").resolve("requests"); + if (!Files.isDirectory(requestsDirectory)) { + return new HighlightAssetWorkerResult(projectId, highlight.highlightId(), List.of(), List.of()); + } + List requests = readRequests(requestsDirectory); + List resolved = new ArrayList<>(); + List pending = new ArrayList<>(); + for (HighlightAssetRequest request : requests) { + Optional materialized = materialize(request, category); + if (materialized.isPresent()) { + resolved.add(materialized.get().toString()); + log.info("event=highlight_asset_materialized project_id={} highlight_id={} type={} target={} source={}", + projectId, highlight.highlightId(), request.type(), request.targetPath(), materialized.get()); + } else { + pending.add(request.requestPath()); + log.info("event=highlight_asset_pending project_id={} highlight_id={} type={} request={}", + projectId, highlight.highlightId(), request.type(), request.requestPath()); + } + } + return new HighlightAssetWorkerResult(projectId, highlight.highlightId(), resolved, pending); + } + + private List readRequests(Path directory) { + try { + try (var files = Files.list(directory)) { + return files.filter(path -> path.getFileName().toString().endsWith(".json")) + .sorted(Comparator.comparing(path -> path.getFileName().toString())) + .map(path -> { + try { + return objectMapper.readValue(path.toFile(), HighlightAssetRequest.class); + } catch (IOException ex) { + throw new IllegalStateException("Unable to read asset request JSON: " + path, ex); + } + }) + .toList(); + } + } catch (IOException ex) { + throw new IllegalStateException("Unable to scan highlight asset requests: " + directory, ex); + } + } + + private Optional materialize(HighlightAssetRequest request, ContentCategory category) { + if ("music".equals(request.type())) { + return resolveOrGenerateMusic(request, category); + } + if ("sfx".equals(request.type())) { + return resolveOrGenerateSfx(request, category); + } + if ("voiceover".equals(request.type())) { + return resolveOrGenerateVoiceover(request); + } + return Optional.empty(); + } + + private Optional resolveOrGenerateMusic(HighlightAssetRequest request, ContentCategory category) { + Optional asset = assetLibrary.select(new EditAssetSelectionRequest(EditAssetType.MUSIC, + category, request.notes(), request.durationSeconds())); + Path target = Path.of(request.targetPath()); + if (asset.isPresent()) { + copy(Path.of(asset.get().path()), target); + return Optional.of(target); + } + return generateToneBed(target, request.durationSeconds()); + } + + private Optional resolveOrGenerateSfx(HighlightAssetRequest request, ContentCategory category) { + Optional exact = assetProvider.resolve(new EditAssetRequest(EditAssetType.SFX, + request.assetKey(), category, request.durationSeconds(), request.notes())); + Path target = Path.of(request.targetPath()); + if (exact.isPresent()) { + copy(Path.of(exact.get().path()), target); + return Optional.of(target); + } + Optional fallback = assetLibrary.select(new EditAssetSelectionRequest(EditAssetType.SFX, + category, request.notes(), request.durationSeconds())); + if (fallback.isPresent()) { + copy(Path.of(fallback.get().path()), target); + return Optional.of(target); + } + return generateImpactTone(target, request.durationSeconds()); + } + + private Optional resolveOrGenerateVoiceover(HighlightAssetRequest request) { + String text = request.notes() == null ? "" : request.notes().trim(); + if (text.isBlank()) { + return Optional.empty(); + } + Path target = Path.of(request.targetPath()); + if (runSay(text, target)) { + return Optional.of(target); + } + if (runEspeak(text, target)) { + return Optional.of(target); + } + return Optional.empty(); + } + + private Optional generateToneBed(Path target, double durationSeconds) { + return generateAudio(target, durationSeconds, "sine=frequency=110:sample_rate=48000", 0.02); + } + + private Optional generateImpactTone(Path target, double durationSeconds) { + return generateAudio(target, Math.max(0.5, Math.min(1.0, durationSeconds)), + "sine=frequency=880:sample_rate=48000", 0.12); + } + + private Optional generateAudio(Path target, double durationSeconds, String source, double volume) { + try { + Files.createDirectories(target.getParent()); + List command = List.of("ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", source, "-t", Double.toString(durationSeconds), + "-af", "volume=" + volume, "-c:a", "aac", target.toString()); + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + if (process.waitFor() == 0) { + return Optional.of(target); + } + log.warn("event=highlight_asset_generation_failed target={} message={}", target, output); + return Optional.empty(); + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + log.warn("event=highlight_asset_generation_failed target={} error_type={} message={}", + target, ex.getClass().getSimpleName(), ex.getMessage()); + return Optional.empty(); + } + } + + private boolean runSay(String text, Path target) { + return runSpeechCommand(List.of("say", "-o", target.toString(), text), target); + } + + private boolean runEspeak(String text, Path target) { + return runSpeechCommand(List.of("espeak", "-w", target.toString(), text), target); + } + + private boolean runSpeechCommand(List command, Path target) { + try { + Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); + String output = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); + if (process.waitFor() == 0 && Files.isRegularFile(target)) { + return true; + } + if (!output.isBlank()) { + log.warn("event=highlight_voiceover_generation_failed target={} output={}", target, output); + } + return false; + } catch (IOException | InterruptedException ex) { + Thread.currentThread().interrupt(); + log.warn("event=highlight_voiceover_generation_failed target={} error_type={} message={}", target, + ex.getClass().getSimpleName(), ex.getMessage()); + return false; + } + } + + private void copy(Path source, Path target) { + try { + Files.createDirectories(target.getParent()); + Files.copy(source, target, StandardCopyOption.REPLACE_EXISTING); + } catch (IOException ex) { + throw new IllegalStateException("Unable to copy generated highlight asset: " + source, ex); + } + } + + public record HighlightAssetWorkerResult( + String projectId, + String highlightId, + List resolvedAssets, + List pendingRequests + ) { + public HighlightAssetWorkerResult { + resolvedAssets = resolvedAssets == null ? List.of() : List.copyOf(resolvedAssets); + pendingRequests = pendingRequests == null ? List.of() : List.copyOf(pendingRequests); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java index 71a1cc7..009da37 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java +++ b/src/main/java/org/example/videoclips/editing/HighlightSourceScheduler.java @@ -39,6 +39,7 @@ public class HighlightSourceScheduler { private final VideoClippingProperties.Editing.HighlightScheduler properties; private final HighlightProjectStore store; private final HighlightSourceAnalyzer analyzer; + private final HighlightDirectorPromptGenerator directorPromptGenerator; private final Clock clock; private final AtomicBoolean scanning = new AtomicBoolean(false); private final AtomicLong scanSequence = new AtomicLong(); @@ -47,20 +48,23 @@ public class HighlightSourceScheduler { public HighlightSourceScheduler( VideoClippingProperties properties, HighlightProjectStore store, - HighlightSourceAnalyzer analyzer + HighlightSourceAnalyzer analyzer, + HighlightDirectorPromptGenerator directorPromptGenerator ) { - this(properties, store, analyzer, Clock.systemUTC()); + this(properties, store, analyzer, directorPromptGenerator, Clock.systemUTC()); } HighlightSourceScheduler( VideoClippingProperties properties, HighlightProjectStore store, HighlightSourceAnalyzer analyzer, + HighlightDirectorPromptGenerator directorPromptGenerator, Clock clock ) { this.properties = properties.getEditing().getHighlightScheduler(); this.store = store; this.analyzer = analyzer; + this.directorPromptGenerator = directorPromptGenerator; this.clock = clock; } @@ -162,6 +166,10 @@ public class HighlightSourceScheduler { log.info("event=highlight_source_moved_to_processed scan_id={} project_id={} processed_file={} " + "processed_directory={}", scanId, projectId, processedFile.getFileName(), processedFile.getParent()); + directorPromptGenerator.generate(projectId); + log.info("event=highlight_director_prompt_generated scan_id={} project_id={} prompt={} readme={}", + scanId, projectId, store.directorDirectory(projectId).resolve("director-prompt.md"), + store.directorDirectory(projectId).resolve("director-brief.md")); log.info("event=highlight_project_created scan_id={} project_id={} source_file={} project_directory={} " + "analysis_file={} elapsed_ms={}", scanId, projectId, processedFile.getFileName(), projectDirectory, "analysis/source-analysis.json", diff --git a/src/main/java/org/example/videoclips/editing/HighlightVisualEffectsStage.java b/src/main/java/org/example/videoclips/editing/HighlightVisualEffectsStage.java new file mode 100644 index 0000000..9b11bef --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HighlightVisualEffectsStage.java @@ -0,0 +1,63 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; +import java.util.Locale; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HighlightVisualEffectsStage { + + private final HighlightProjectStore store; + private final ObjectMapper objectMapper; + + public HighlightVisualEffectsStage(HighlightProjectStore store, ObjectMapper objectMapper) { + this.store = store; + this.objectMapper = objectMapper; + } + + public HighlightVisualEffectsPlan create(String projectId, HighlightDirectorPlan.HighlightItem highlight, + ContentCategory category) { + HighlightVisualEffectsPlan plan = new HighlightVisualEffectsPlan(projectId, highlight.highlightId(), + category.name().toLowerCase(Locale.ROOT), highlight.visualTreatment(), + List.of("crop", "punch-in", "contrast", "vignette", "fade"), + highlight.overlays(), highlight.renderNotes(), Instant.now()); + write(projectId, highlight.highlightId(), plan); + return plan; + } + + private void write(String projectId, String highlightId, HighlightVisualEffectsPlan plan) { + Path directory = store.highlightsDirectory(projectId).resolve(highlightId); + try { + Files.createDirectories(directory); + objectMapper.writerWithDefaultPrettyPrinter() + .writeValue(directory.resolve("visual-effects.json").toFile(), plan); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write highlight visual effects plan", ex); + } + } + + public record HighlightVisualEffectsPlan( + String projectId, + String highlightId, + String category, + String grade, + List effects, + List overlays, + String renderNotes, + Instant createdAt + ) { + public HighlightVisualEffectsPlan { + effects = effects == null ? List.of() : List.copyOf(effects); + overlays = overlays == null ? List.of() : List.copyOf(overlays); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java b/src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java new file mode 100644 index 0000000..04a5945 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/LocalAssetGenerationStage.java @@ -0,0 +1,251 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.time.Instant; +import java.util.ArrayList; +import java.util.HexFormat; +import java.util.List; +import java.util.Optional; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class LocalAssetGenerationStage implements AssetGenerationStage { + + private static final Logger log = LoggerFactory.getLogger(LocalAssetGenerationStage.class); + + private final EditProjectStore store; + private final EditAssetProvider provider; + private final EditAssetLibrary library; + private final Path voiceoverCacheRoot; + private final ObjectMapper objectMapper; + + public LocalAssetGenerationStage(VideoClippingProperties properties, EditProjectStore store, + EditAssetProvider provider, EditAssetLibrary library, + ObjectMapper objectMapper) { + this.store = store; + this.provider = provider; + this.library = library; + this.voiceoverCacheRoot = Path.of(properties.getEditing().getAssets().getVoiceoverFolder()).normalize(); + this.objectMapper = objectMapper; + } + + @Override + public AssetGenerationResult prepare(String projectId, EditPlan plan) { + long startedAt = System.nanoTime(); + Path projectDirectory = store.projectDirectory(projectId); + Path projectAudioDirectory = projectDirectory.resolve("audio"); + Path projectSfxDirectory = projectAudioDirectory.resolve("sfx"); + Path requestsDirectory = projectDirectory.resolve("assets").resolve("requests"); + createDirectory(projectAudioDirectory); + createDirectory(projectSfxDirectory); + createDirectory(requestsDirectory); + + List items = new ArrayList<>(); + boolean readyForRender = true; + + Optional musicCue = plan.audioCues().stream() + .filter(cue -> "music".equals(cue.type())) + .findFirst(); + if (musicCue.isPresent()) { + items.add(materializeMusic(projectId, plan, musicCue.get(), projectAudioDirectory, requestsDirectory)); + } + + if (!plan.voiceover().isEmpty()) { + items.add(materializeVoiceover(projectId, plan, projectAudioDirectory, requestsDirectory)); + } + + for (AudioCue cue : plan.audioCues()) { + if ("sfx".equals(cue.type())) { + AssetGenerationItem item = materializeSfx(projectId, plan, cue, projectSfxDirectory, requestsDirectory); + items.add(item); + if (!item.reused()) { + readyForRender = false; + } + } + } + + AssetGenerationResult result = new AssetGenerationResult(projectId, readyForRender, items, Instant.now()); + writeJson(projectDirectory.resolve("assets").resolve("asset-generation-manifest.json"), result); + writeJson(projectDirectory.resolve("assets").resolve("generated-assets.json"), items); + log.info("event=asset_generation_completed project_id={} item_count={} ready_for_render={} elapsed_ms={}", + projectId, items.size(), readyForRender, (System.nanoTime() - startedAt) / 1_000_000); + return result; + } + + private AssetGenerationItem materializeMusic(String projectId, EditPlan plan, AudioCue cue, + Path projectAudioDirectory, Path requestsDirectory) { + String cacheKey = cacheKey("music", plan.projectId(), plan.style(), cue.assetKey(), cue.notes(), + Double.toString(cue.timelineEndSeconds() - cue.timelineStartSeconds())); + Path cachePath = Path.of(store.projectDirectory(projectId).getParent().toString(), "_asset-cache", + "music", cacheKey + ".wav"); + Path projectTarget = projectAudioDirectory.resolve("music.wav"); + return materialize(projectId, "music", cue.assetKey(), cacheKey, cachePath, projectTarget, + cue.notes(), requestsDirectory, cue.timelineEndSeconds() - cue.timelineStartSeconds(), true); + } + + private AssetGenerationItem materializeVoiceover(String projectId, EditPlan plan, Path projectAudioDirectory, + Path requestsDirectory) { + String voiceoverText = plan.voiceover().stream().map(VoiceoverLine::text).reduce("", (a, b) -> a + "\n" + b); + String cacheKey = cacheKey("voiceover", plan.projectId(), plan.style(), voiceoverText, plan.summary(), + Double.toString(plan.targetDurationSeconds())); + Path cachePath = voiceoverCacheRoot.resolve(cacheKey + ".wav"); + Path projectTarget = projectAudioDirectory.resolve("voiceover.wav"); + return materialize(projectId, "voiceover", cacheKey, cacheKey, cachePath, projectTarget, + "voiceover lines=" + plan.voiceover().size(), requestsDirectory, plan.targetDurationSeconds(), false); + } + + private AssetGenerationItem materializeSfx(String projectId, EditPlan plan, AudioCue cue, Path projectSfxDirectory, + Path requestsDirectory) { + String cacheKey = cacheKey("sfx", plan.projectId(), plan.style(), cue.assetKey(), cue.notes(), + Double.toString(cue.timelineEndSeconds() - cue.timelineStartSeconds()), + Double.toString(cue.gainDb())); + Path cachePath = Path.of(store.projectDirectory(projectId).getParent().toString(), "_asset-cache", + "sfx", cacheKey + ".wav"); + Path projectTarget = projectSfxDirectory.resolve(cue.assetKey() + ".wav"); + return materialize(projectId, "sfx", cue.assetKey(), cacheKey, cachePath, projectTarget, + cue.notes(), requestsDirectory, cue.timelineEndSeconds() - cue.timelineStartSeconds(), true); + } + + private AssetGenerationItem materialize(String projectId, String type, String assetKey, String cacheKey, + Path cachePath, Path projectTarget, String notes, + Path requestsDirectory, double durationSeconds, boolean blocking) { + try { + createDirectory(cachePath.getParent()); + if (Files.isRegularFile(cachePath)) { + copy(cachePath, projectTarget); + log.info("event=asset_materialized project_id={} type={} asset_key={} cache_key={} source={} target={}", + projectId, type, assetKey, cacheKey, cachePath, projectTarget); + return new AssetGenerationItem(type, assetKey, cacheKey, true, cachePath.toString(), + projectTarget.toString(), null, notes); + } + + Optional selected = selectExistingAsset(type, assetKey, notes, durationSeconds); + if (selected.isPresent()) { + copy(Path.of(selected.get().path()), cachePath); + copy(Path.of(selected.get().path()), projectTarget); + copyIfMissing(Path.of(selected.get().path()).resolveSibling(Path.of(selected.get().path()).getFileName() + + ".license.txt"), projectTarget.resolveSibling(projectTarget.getFileName() + ".license.txt")); + log.info("event=asset_materialized project_id={} type={} asset_key={} cache_key={} source={} target={}", + projectId, type, assetKey, cacheKey, selected.get().path(), projectTarget); + return new AssetGenerationItem(type, assetKey, cacheKey, true, selected.get().path(), + projectTarget.toString(), null, notes); + } + + Path requestDirectory = requestsDirectory.resolve(type).resolve(cacheKey); + createDirectory(requestDirectory); + Path requestFile = requestDirectory.resolve("request.md"); + Files.writeString(requestFile, request(projectId, type, assetKey, cacheKey, cachePath, projectTarget, + notes, durationSeconds, blocking), StandardCharsets.UTF_8); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(requestDirectory.resolve("request.json").toFile(), + new AssetGenerationItem(type, assetKey, cacheKey, false, null, projectTarget.toString(), + requestFile.toString(), notes)); + log.info("event=asset_generation_requested project_id={} type={} asset_key={} cache_key={} request={}", + projectId, type, assetKey, cacheKey, requestFile); + return new AssetGenerationItem(type, assetKey, cacheKey, false, null, projectTarget.toString(), + requestFile.toString(), notes); + } catch (IOException ex) { + throw new IllegalStateException("Unable to prepare generated asset: " + type + "/" + assetKey, ex); + } + } + + private Optional selectExistingAsset(String type, String assetKey, String notes, + double durationSeconds) { + if ("music".equals(type)) { + return library.select(new EditAssetSelectionRequest(EditAssetType.MUSIC, null, notes, durationSeconds)); + } + if ("voiceover".equals(type)) { + return provider.list(EditAssetType.VOICEOVER, null).stream().findFirst(); + } + if ("sfx".equals(type)) { + Optional exact = provider.resolve(new EditAssetRequest(EditAssetType.SFX, assetKey, + null, durationSeconds, notes)); + if (exact.isPresent()) { + return exact; + } + return library.select(new EditAssetSelectionRequest(EditAssetType.SFX, null, notes, durationSeconds)); + } + return Optional.empty(); + } + + private String request(String projectId, String type, String assetKey, String cacheKey, Path cachePath, + Path projectTarget, String notes, double durationSeconds, boolean blocking) { + return """ + # Asset Generation Request + + Project: `%s` + Type: `%s` + Asset key: `%s` + Cache key: `%s` + Cache file: `%s` + Project target: `%s` + Blocking for render: `%s` + Duration seconds: %.3f + + Create a reusable asset at the cache file path above. + If an asset already exists at that exact cache path, reuse it. + If you generate a new asset, keep the filename stable so future projects can reuse it. + Notes: + %s + """.formatted(projectId, type, assetKey, cacheKey, cachePath, projectTarget, blocking, durationSeconds, + notes == null ? "" : notes); + } + + private String cacheKey(String type, String... values) { + try { + MessageDigest digest = MessageDigest.getInstance("SHA-256"); + digest.update(type.getBytes(StandardCharsets.UTF_8)); + for (String value : values) { + if (value != null) { + digest.update((byte) 0); + digest.update(value.getBytes(StandardCharsets.UTF_8)); + } + } + return HexFormat.of().formatHex(digest.digest()).substring(0, 16); + } catch (NoSuchAlgorithmException ex) { + throw new IllegalStateException("Unable to compute asset cache key", ex); + } + } + + private void createDirectory(Path directory) { + try { + Files.createDirectories(directory); + } catch (IOException ex) { + throw new IllegalStateException("Unable to create asset generation directory: " + directory, ex); + } + } + + private void copy(Path source, Path target) throws IOException { + if (source.normalize().equals(target.normalize())) { + return; + } + createDirectory(target.getParent()); + Files.copy(source, target, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + } + + private void copyIfMissing(Path source, Path target) throws IOException { + if (Files.isRegularFile(source) && !Files.exists(target)) { + copy(source, target); + } + } + + private void writeJson(Path path, Object value) { + try { + createDirectory(path.getParent()); + objectMapper.writerWithDefaultPrettyPrinter().writeValue(path.toFile(), value); + } catch (IOException ex) { + throw new IllegalStateException("Unable to write asset generation JSON: " + path, ex); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/RenderManifest.java b/src/main/java/org/example/videoclips/editing/RenderManifest.java index 321ba09..2817511 100644 --- a/src/main/java/org/example/videoclips/editing/RenderManifest.java +++ b/src/main/java/org/example/videoclips/editing/RenderManifest.java @@ -6,6 +6,7 @@ import java.util.List; public record RenderManifest( String projectId, List clipIds, + List renderedClipPaths, String outputPath, double durationSeconds, List> commands, @@ -14,6 +15,7 @@ public record RenderManifest( ) { public RenderManifest { clipIds = clipIds == null ? List.of() : List.copyOf(clipIds); + renderedClipPaths = renderedClipPaths == null ? List.of() : List.copyOf(renderedClipPaths); commands = commands == null ? List.of() : List.copyOf(commands); assets = assets == null ? List.of() : List.copyOf(assets); } @@ -26,6 +28,18 @@ public record RenderManifest( List> commands, Instant completedAt ) { - this(projectId, clipIds, outputPath, durationSeconds, commands, List.of(), completedAt); + this(projectId, clipIds, List.of(), outputPath, durationSeconds, commands, List.of(), completedAt); + } + + public RenderManifest( + String projectId, + List clipIds, + String outputPath, + double durationSeconds, + List> commands, + List assets, + Instant completedAt + ) { + this(projectId, clipIds, List.of(), outputPath, durationSeconds, commands, assets, completedAt); } } diff --git a/src/main/java/org/example/videoclips/editing/StoryboardPromptGenerator.java b/src/main/java/org/example/videoclips/editing/StoryboardPromptGenerator.java index cd0446f..e0ba230 100644 --- a/src/main/java/org/example/videoclips/editing/StoryboardPromptGenerator.java +++ b/src/main/java/org/example/videoclips/editing/StoryboardPromptGenerator.java @@ -65,6 +65,17 @@ public class StoryboardPromptGenerator { Create one cinematic edit plan for project `%s`. + The service can only render what the plan expresses, so your plan must use the actual cinematic toolkit + available to the renderer: + - shot selection and timing + - cuts, crossfades, fade-ins, and fade-outs + - dynamic punch-in crop and subtle cinematic grade + - music bed selection and timing + - SFX placement and timing + - voiceover scripting and timing + - short text overlays + - per-segment rendered clips plus the final assembled video + Constraints: - Style: `%s` - Target duration: %d seconds @@ -77,6 +88,7 @@ public class StoryboardPromptGenerator { - Timeline ranges must be chronological and must not overlap. - Prefer an intentional opening hook, escalating middle, and memorable final hero shot. - Use music, SFX, voiceover, text overlays, and visual treatment only when they fit the detected category. + - Make the edit cinematic by directing those elements deliberately, not by trimming footage alone. - Voiceover must be specific to what can be observed in the supplied thumbnails/contact sheets. - Do not write generic praise, generic lifestyle language, or placeholder asset names. - Return strict JSON only. Do not use Markdown fences or add commentary. diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 41c84ad..3c50a4b 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -27,7 +27,7 @@ video-clipping: minimum-scene-duration-seconds: ${VIDEO_EDITING_MINIMUM_SCENE_DURATION_SECONDS:1.0} silence-threshold-db: ${VIDEO_EDITING_SILENCE_THRESHOLD_DB:-35.0} silence-minimum-duration-seconds: ${VIDEO_EDITING_SILENCE_MINIMUM_DURATION_SECONDS:0.5} - target-duration-seconds: ${VIDEO_EDITING_TARGET_DURATION_SECONDS:60} + target-duration-seconds: ${VIDEO_EDITING_TARGET_DURATION_SECONDS:600} output-width: ${VIDEO_EDITING_OUTPUT_WIDTH:1920} output-height: ${VIDEO_EDITING_OUTPUT_HEIGHT:1080} output-frame-rate: ${VIDEO_EDITING_OUTPUT_FRAME_RATE:30} @@ -78,6 +78,12 @@ video-clipping: processed-directory: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_PROCESSED_DIRECTORY:./input/highlights/processed} rejected-directory: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_REJECTED_DIRECTORY:./input/highlights/rejected} poll-interval-ms: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_POLL_INTERVAL_MS:5000} + render-enabled: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_RENDER_ENABLED:true} + require-director-approval: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_REQUIRE_DIRECTOR_APPROVAL:false} + approval-file-name: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_APPROVAL_FILE_NAME:approved.flag} + max-highlights-per-source: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_MAX_HIGHLIGHTS_PER_SOURCE:3} + highlight-min-duration-seconds: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_HIGHLIGHT_MIN_DURATION_SECONDS:8} + highlight-max-duration-seconds: ${VIDEO_EDITING_HIGHLIGHT_SCHEDULER_HIGHLIGHT_MAX_DURATION_SECONDS:35} logging: level: diff --git a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java index fd5313f..3ba886c 100644 --- a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java +++ b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java @@ -31,7 +31,7 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().getMinimumSceneDurationSeconds()).isEqualTo(1.0); assertThat(properties.getEditing().getSilenceThresholdDb()).isEqualTo(-35.0); assertThat(properties.getEditing().getSilenceMinimumDurationSeconds()).isEqualTo(0.5); - assertThat(properties.getEditing().getTargetDurationSeconds()).isEqualTo(60); + assertThat(properties.getEditing().getTargetDurationSeconds()).isEqualTo(600); assertThat(properties.getEditing().getOutputWidth()).isEqualTo(1920); assertThat(properties.getEditing().getOutputHeight()).isEqualTo(1080); assertThat(properties.getEditing().getOutputFrameRate()).isEqualTo(30); diff --git a/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java b/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java index dc30bd6..0dedfc2 100644 --- a/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java +++ b/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java @@ -62,7 +62,14 @@ class CinematicEditingIntegrationTest { Path output = projectDirectory.resolve("final.mp4"); assertThat(output).isRegularFile(); assertThat(probeDuration(output)).isGreaterThan(0); + assertThat(projectDirectory.resolve("rendered-clips/clip_0001.mp4")).isRegularFile(); + assertThat(projectDirectory.resolve("rendered-clips/clip_0002.mp4")).isRegularFile(); + assertThat(projectDirectory.resolve("rendered-clips/clip_0003.mp4")).isRegularFile(); assertThat(projectDirectory.resolve("render-manifest.json")).exists(); + RenderManifest manifest = store.readJson(projectId, "render-manifest.json", RenderManifest.class); + assertThat(manifest.clipIds()).containsExactly("clip_01", "clip_02", "clip_03"); + assertThat(manifest.renderedClipPaths()).hasSize(3) + .allSatisfy(path -> assertThat(Path.of(path)).isRegularFile()); assertThat(projectDirectory.resolve("qa-report.json")).exists(); assertThat(projects.getProject(projectId).status()).isEqualTo(EditProjectStatus.RENDERED); } @@ -128,7 +135,8 @@ class CinematicEditingIntegrationTest { index * 2.0, (index + 1) * 2.0, "cut", "cut", 1, "cinematic grade", "integration sequence")) .toList(); - return new EditPlan(projectId, "category-aware-cinematic-highlights", 60, decisions, List.of(), List.of(), + return new EditPlan(projectId, "category-aware-cinematic-highlights", + decisions.get(decisions.size() - 1).timelineEndSeconds(), decisions, List.of(), List.of(), "mp4-h264-aac-320p", "Generated integration edit"); } diff --git a/src/test/java/org/example/videoclips/editing/EditPlanInboxScannerTest.java b/src/test/java/org/example/videoclips/editing/EditPlanInboxScannerTest.java index 0569ca3..a5237c5 100644 --- a/src/test/java/org/example/videoclips/editing/EditPlanInboxScannerTest.java +++ b/src/test/java/org/example/videoclips/editing/EditPlanInboxScannerTest.java @@ -130,10 +130,11 @@ class EditPlanInboxScannerTest { } private void writeValidPlan() throws Exception { - EditPlan plan = new EditPlan("porsche-edit", "cinematic-porsche-promo", 60, + EditPlan plan = new EditPlan("porsche-edit", "cinematic-porsche-promo", 2.5, List.of(new EditDecision("clip_00001", 0, 2.5, 0, 2.5, "cut", "cut", 1.0, "cinematic", "opening")), - List.of(), List.of(), "mp4-h264-aac-1080p", "Hero edit"); + List.of(new AudioCue("music", "premium-cinematic-drive", 0, 2.5, -14, "opening bed")), + List.of(), "mp4-h264-aac-1080p", "Hero edit"); objectMapper.writeValue(store.inboxDirectory("porsche-edit").resolve("edit-plan.json").toFile(), plan); } diff --git a/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java b/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java index 588fe9a..9fc340a 100644 --- a/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java +++ b/src/test/java/org/example/videoclips/editing/EditPlanValidatorTest.java @@ -37,6 +37,41 @@ class EditPlanValidatorTest { @Test void acceptsValidPlan() { assertThat(validator.validate("project", plan(decision("clip-1", 0, 4, 0, 4, "cut")))).isNotNull(); } + @Test void acceptsContentDrivenPlanShorterThanProjectTarget() { + EditPlan shorter = new EditPlan("project", "cinematic-porsche-promo", 4, + List.of(decision("clip-1", 0, 4, 0, 4, "cut")), + List.of(new AudioCue("music", "premium-cinematic-drive", 0, 4, -14, "bed")), + List.of(), + "mp4-h264-aac-1080p", "summary"); + + assertThat(validator.validate("project", shorter)).isEqualTo(shorter); + } + + @Test void rejectsBareTrimOnlyPlanWithoutCinematicRichness() { + EditPlan bare = new EditPlan("project", "cinematic-porsche-promo", 4, + List.of(new EditDecision("clip-1", 0, 4, 0, 4, "cut", "cut", 1, "none", "reason")), + List.of(), List.of(), List.of(), "mp4-h264-aac-1080p", "summary"); + + rejects(bare); + } + + @Test void rejectsPlanTargetDurationThatDoesNotMatchFinalTimeline() { + EditPlan mismatch = new EditPlan("project", "cinematic-porsche-promo", 8, + List.of(decision("clip-1", 0, 4, 0, 4, "cut")), List.of(), List.of(), + "mp4-h264-aac-1080p", "summary"); + + rejects(mismatch); + } + + @Test void rejectsPlanLongerThanProjectMaximumTarget() { + EditPlan tooLong = new EditPlan("project", "cinematic-porsche-promo", 12, + List.of(decision("clip-1", 0, 8, 0, 8, "cut"), + decision("clip-1", 0, 4, 8, 12, "cut")), + List.of(), List.of(), "mp4-h264-aac-1080p", "summary"); + + rejects(tooLong); + } + @Test void rejectsUnknownClip() { rejects(plan(decision("missing", 0, 4, 0, 4, "cut"))); } @Test void rejectsOutOfRangeSource() { rejects(plan(decision("clip-1", 0, 9, 0, 4, "cut"))); } @@ -141,7 +176,11 @@ class EditPlanValidatorTest { } private EditPlan plan(EditDecision... decisions) { - return new EditPlan("project", "cinematic-porsche-promo", 10, List.of(decisions), List.of(), List.of(), + return new EditPlan("project", "cinematic-porsche-promo", decisions[decisions.length - 1].timelineEndSeconds(), + List.of(decisions), + List.of(new AudioCue("music", "premium-cinematic-drive", 0, + decisions[decisions.length - 1].timelineEndSeconds(), -14, "bed")), + List.of(), "mp4-h264-aac-1080p", "summary"); } diff --git a/src/test/java/org/example/videoclips/editing/HighlightAssetPreparationServiceTest.java b/src/test/java/org/example/videoclips/editing/HighlightAssetPreparationServiceTest.java new file mode 100644 index 0000000..b4f0175 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/HighlightAssetPreparationServiceTest.java @@ -0,0 +1,75 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class HighlightAssetPreparationServiceTest { + + @TempDir + Path tempDir; + + @Test + void writesVoiceoverScriptAndRequestsWhenAssetsAreMissing() throws Exception { + VideoClippingProperties properties = properties(); + ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); + FileSystemHighlightProjectStore store = new FileSystemHighlightProjectStore(properties, mapper); + HighlightProject project = project(); + store.createProject(project, manifest()); + EditAssetProvider assetProvider = mock(EditAssetProvider.class); + EditAssetLibrary assetLibrary = mock(EditAssetLibrary.class); + when(assetLibrary.select(org.mockito.ArgumentMatchers.any())).thenReturn(java.util.Optional.empty()); + when(assetProvider.list(org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.any())) + .thenReturn(List.of()); + when(assetProvider.resolve(org.mockito.ArgumentMatchers.any())).thenReturn(java.util.Optional.empty()); + HighlightAssetPreparationService service = new HighlightAssetPreparationService(properties, store, + assetProvider, assetLibrary, mapper); + HighlightDirectorPlan.HighlightItem highlight = new HighlightDirectorPlan.HighlightItem( + "highlight_001", "candidate_001", "Opening Reveal", 0.0, 12.0, 12.0, + "opening_hook", "premium cinematic grade", "low cinematic pulse", + "whoosh on reveal", List.of("This is a clean first look."), List.of("Pure presence"), + "hold the opening beat"); + + HighlightAssetPreparationService.HighlightAssetPreparationResult result = service.prepare(project.id(), + project, highlight, ContentCategory.GENERIC_VLOG); + + assertThat(result.requestFiles()).isNotEmpty(); + assertThat(Files.exists(store.highlightsDirectory(project.id()).resolve("highlight_001") + .resolve("assets/voiceover/voiceover-script.txt"))).isTrue(); + try (var files = Files.list(store.highlightsDirectory(project.id()).resolve("highlight_001") + .resolve("assets/requests"))) { + assertThat(files.count()).isGreaterThan(0); + } + } + + private VideoClippingProperties properties() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + properties.getEditing().getAssets().setMusicFolder(tempDir.resolve("music").toString()); + properties.getEditing().getAssets().setSfxFolder(tempDir.resolve("sfx").toString()); + properties.getEditing().getAssets().setVoiceoverFolder(tempDir.resolve("voiceover-cache").toString()); + return properties; + } + + private HighlightProject project() { + Instant now = Instant.parse("2026-07-11T08:00:00Z"); + return new HighlightProject("highlight-project-001", "Highlight Project", + HighlightProjectStatus.WAITING_FOR_DIRECTOR, "source.mp4", + tempDir.resolve("highlight-projects/highlight-project-001").toString(), now, now, null); + } + + private HighlightProjectManifest manifest() { + return new HighlightProjectManifest("highlight-project-001", "source.mp4", + HighlightFolderContract.standard(), Instant.parse("2026-07-11T08:00:00Z")); + } +} diff --git a/src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java b/src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java new file mode 100644 index 0000000..fb3c2f7 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/HighlightDirectorFlowServiceTest.java @@ -0,0 +1,109 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class HighlightDirectorFlowServiceTest { + + @TempDir + Path tempDir; + + @Test + void importsDirectorPlanWritesHighlightEditPlanAndPublishesFinalVideo() throws Exception { + VideoClippingProperties properties = properties(); + ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); + FileSystemHighlightProjectStore store = new FileSystemHighlightProjectStore(properties, mapper); + HighlightProject project = project(); + store.createProject(project, manifest()); + store.writeJson(project.id(), "analysis/source-analysis.json", sourceAnalysis()); + store.writeJson(project.id(), "director/edit-plan.json", directorPlan()); + Path source = store.sourceDirectory(project.id()).resolve("source.mp4"); + createDummyFile(source); + HighlightVisualEffectsStage visualEffectsStage = mock(HighlightVisualEffectsStage.class); + HighlightAssetPreparationService assetPreparationService = mock(HighlightAssetPreparationService.class); + HighlightAssetPreparationService.HighlightAssetPreparationResult assetResult = + new HighlightAssetPreparationService.HighlightAssetPreparationResult( + store.highlightsDirectory(project.id()).resolve("highlight_001"), + store.highlightsDirectory(project.id()).resolve("highlight_001").resolve("assets"), + List.of(), List.of()); + when(assetPreparationService.prepare(anyString(), any(), any(), any())).thenReturn(assetResult); + HighlightLocalAssetWorker assetWorker = mock(HighlightLocalAssetWorker.class); + when(assetWorker.process(anyString(), any(), any())).thenReturn( + new HighlightLocalAssetWorker.HighlightAssetWorkerResult(project.id(), "highlight_001", + List.of(), List.of())); + HighlightFfmpegRenderer renderer = mock(HighlightFfmpegRenderer.class); + Path renderedClip = store.highlightsDirectory(project.id()).resolve("highlight_001").resolve("final.mp4"); + createDummyFile(renderedClip); + HighlightFfmpegRenderer.HighlightRenderResult renderResult = new HighlightFfmpegRenderer.HighlightRenderResult( + "highlight_001", renderedClip, renderedClip, List.of(renderedClip), + new RenderManifest(project.id(), List.of("source"), List.of(renderedClip.toString()), + renderedClip.toString(), 12.0, List.of(), List.of(), Instant.now()), + new RenderQaReport(project.id(), true, List.of(), Instant.now())); + when(renderer.render(anyString(), any(), any())).thenReturn(renderResult); + HighlightDirectorFlowService service = new HighlightDirectorFlowService(properties, mapper, store, + visualEffectsStage, assetPreparationService, assetWorker, renderer); + + HighlightDirectorFlowService.HighlightFlowResult result = service.process(project.id(), 1); + + assertThat(result.finalOutputs()).hasSize(1); + assertThat(store.highlightsDirectory(project.id()).resolve("highlight_001/edit-plan.json")).exists(); + assertThat(store.highlightsDirectory(project.id()).resolve("highlight_001/storyboard.md")).exists(); + assertThat(store.projectDirectory(project.id()).resolve("final.mp4")).exists(); + assertThat(store.readJson(project.id(), "project.json", HighlightProject.class).status()) + .isEqualTo(HighlightProjectStatus.RENDERED); + } + + private VideoClippingProperties properties() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + return properties; + } + + private HighlightProject project() { + Instant now = Instant.parse("2026-07-11T08:00:00Z"); + return new HighlightProject("highlight-project-001", "Highlight Project", + HighlightProjectStatus.WAITING_FOR_DIRECTOR, "source.mp4", + tempDir.resolve("highlight-projects/highlight-project-001").toString(), now, now, null); + } + + private HighlightProjectManifest manifest() { + return new HighlightProjectManifest("highlight-project-001", "source.mp4", + HighlightFolderContract.standard(), Instant.parse("2026-07-11T08:00:00Z")); + } + + private HighlightDirectorPlan directorPlan() { + return new HighlightDirectorPlan("highlight-project-001", "source.mp4", "generic_vlog", + List.of(new HighlightDirectorPlan.HighlightItem("highlight_001", "candidate_001", "Opening", + 0.0, 12.0, 12.0, "opening_hook", "premium cinematic grade", + "low pulse", "whoosh", List.of("A clean first look."), List.of("Pure presence"), + "open with presence")), "summary"); + } + + private HighlightSourceAnalysis sourceAnalysis() { + ClipAnalysis source = new ClipAnalysis("source", "source.mp4", 12.0, "hevc", "aac", 1920, 1080, 24.0, + List.of(), null, null, 0.0, 0.0, 0.0); + return new HighlightSourceAnalysis("highlight-project-001", "source.mp4", source, List.of(), null, null, + null, "analysis/scene-segments.json", List.of(), "analysis/audio-analysis.json", + new SourceAudioAnalysis("source", true, -18.0, -3.0, -35.0, 0.5, List.of(), null), + "analysis/visual-analysis.json", new SourceVisualAnalysis("source", 0.5, 0.5, 0.5, 0.5, + "unknown", List.of(), List.of(), "heuristic"), Instant.parse("2026-07-11T08:00:00Z")); + } + + private void createDummyFile(Path file) throws Exception { + Files.createDirectories(file.getParent()); + Files.writeString(file, "dummy"); + } +} diff --git a/src/test/java/org/example/videoclips/editing/HighlightDirectorPromptBackfillTest.java b/src/test/java/org/example/videoclips/editing/HighlightDirectorPromptBackfillTest.java new file mode 100644 index 0000000..e2f68ae --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/HighlightDirectorPromptBackfillTest.java @@ -0,0 +1,83 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class HighlightDirectorPromptBackfillTest { + + @TempDir + Path tempDir; + + @Test + void backfillsMissingHighlightDirectorPromptOnStartup() throws Exception { + FileSystemHighlightProjectStore store = store(); + HighlightProject project = project(); + store.createProject(project, manifest()); + store.writeJson(project.id(), "analysis/source-analysis.json", sourceAnalysis(project.id())); + store.writeJson(project.id(), "analysis/category.json", cinematicAnalysis(project.id())); + store.writeJson(project.id(), "analysis/highlight-candidates.json", new HighlightCandidate[] { + new HighlightCandidate("candidate_001", "clip_00001", 1.0, 9.5, 0.94, + "opening_hook", List.of("hero angle", "strong motion")) + }); + + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + HighlightDirectorPromptBackfill backfill = new HighlightDirectorPromptBackfill(properties, + new HighlightDirectorPromptGenerator(store)); + + backfill.backfill(); + + assertThat(Files.exists(store.directorDirectory(project.id()).resolve("director-prompt.md"))).isTrue(); + assertThat(Files.exists(store.directorDirectory(project.id()).resolve("director-brief.md"))).isTrue(); + } + + private FileSystemHighlightProjectStore store() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + return new FileSystemHighlightProjectStore(properties, new ObjectMapper().findAndRegisterModules()); + } + + private HighlightProject project() { + Instant now = Instant.parse("2026-07-11T08:00:00Z"); + return new HighlightProject("porsche-highlight-001", "Porsche Highlight", + HighlightProjectStatus.CREATED, "porsche.mp4", + tempDir.resolve("highlight-projects/porsche-highlight-001").toString(), now, now, null); + } + + private HighlightProjectManifest manifest() { + return new HighlightProjectManifest("porsche-highlight-001", "porsche.mp4", + HighlightFolderContract.standard(), Instant.parse("2026-07-11T08:00:00Z")); + } + + private HighlightSourceAnalysis sourceAnalysis(String projectId) { + ClipAnalysis source = new ClipAnalysis("clip_00001", "porsche.mp4", 12.0, "h264", "aac", + 1920, 1080, 30.0, List.of("thumb-1.jpg", "thumb-2.jpg"), + "contact.jpg", "proxy.mp4", 0.8, 0.7, 0.9); + return new HighlightSourceAnalysis(projectId, "porsche.mp4", source, List.of("thumb-1.jpg", "thumb-2.jpg"), + "contact.jpg", "proxy.mp4", "waveform.png", "analysis/scene-segments.json", + List.of(new ShotSegment("shot_0001", 0, 12, 12, 0, 6)), + "analysis/audio-analysis.json", new SourceAudioAnalysis(projectId, true, -18.0, -3.0, + -35.0, 0.5, List.of(new AudioSection("audio_section_0001", "unclassified_audio", 0, 12, 12)), + null), + "analysis/visual-analysis.json", new SourceVisualAnalysis(projectId, 0.6, 0.7, 0.8, 0.75, + "unknown_without_face_detector", List.of(new VisualObjectLabel("porsche", 0.82, "metadata_heuristic")), + List.of("thumb-1.jpg"), "metadata_thumbnail_scene_heuristic"), Instant.parse("2026-07-11T08:00:00Z")); + } + + private CinematicHighlightAnalysis cinematicAnalysis(String projectId) { + return new CinematicHighlightAnalysis(projectId, ContentCategory.CAR_VLOG, 0.91, + List.of("car metadata detected", "hero motion present"), + List.of(new HighlightCandidate("candidate_001", "clip_00001", 1.0, 9.5, 0.94, + "opening_hook", List.of("hero angle", "strong motion"))), + Instant.parse("2026-07-11T08:00:00Z")); + } +} diff --git a/src/test/java/org/example/videoclips/editing/HighlightDirectorPromptGeneratorTest.java b/src/test/java/org/example/videoclips/editing/HighlightDirectorPromptGeneratorTest.java new file mode 100644 index 0000000..60059b6 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/HighlightDirectorPromptGeneratorTest.java @@ -0,0 +1,90 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class HighlightDirectorPromptGeneratorTest { + + @TempDir + Path tempDir; + + @Test + void writesHighlightDirectorPromptAndBriefForAnalyzedProject() throws Exception { + FileSystemHighlightProjectStore store = store(); + HighlightProject project = project(); + store.createProject(project, manifest()); + store.writeJson(project.id(), "analysis/source-analysis.json", sourceAnalysis(project.id())); + store.writeJson(project.id(), "analysis/category.json", cinematicAnalysis(project.id())); + store.writeJson(project.id(), "analysis/highlight-candidates.json", new HighlightCandidate[] { + new HighlightCandidate("candidate_001", "clip_00001", 1.0, 9.5, 0.94, + "opening_hook", List.of("hero angle", "strong motion")) + }); + + HighlightDirectorPromptGenerator generator = new HighlightDirectorPromptGenerator(store); + HighlightDirectorPromptGenerator.DirectorPromptFiles files = generator.generate(project.id()); + + assertThat(Files.readString(Path.of(files.promptPath()))) + .contains("analysis/source-analysis.json") + .contains("analysis/highlight-candidates.json") + .contains("director/edit-plan.json") + .contains("car_vlog") + .contains("candidate_001") + .contains("premium, powerful, precise"); + assertThat(Files.readString(Path.of(files.briefPath()))) + .contains("Highlight Director Brief") + .contains("director-prompt.md") + .contains("director/edit-plan.json"); + assertThat(store.readJson(project.id(), "project.json", HighlightProject.class).status()) + .isEqualTo(HighlightProjectStatus.WAITING_FOR_DIRECTOR); + } + + private FileSystemHighlightProjectStore store() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + return new FileSystemHighlightProjectStore(properties, new ObjectMapper().findAndRegisterModules()); + } + + private HighlightProject project() { + Instant now = Instant.parse("2026-07-11T08:00:00Z"); + return new HighlightProject("porsche-highlight-001", "Porsche Highlight", + HighlightProjectStatus.CREATED, "porsche.mp4", + tempDir.resolve("highlight-projects/porsche-highlight-001").toString(), now, now, null); + } + + private HighlightProjectManifest manifest() { + return new HighlightProjectManifest("porsche-highlight-001", "porsche.mp4", + HighlightFolderContract.standard(), Instant.parse("2026-07-11T08:00:00Z")); + } + + private HighlightSourceAnalysis sourceAnalysis(String projectId) { + ClipAnalysis source = new ClipAnalysis("clip_00001", "porsche.mp4", 12.0, "h264", "aac", + 1920, 1080, 30.0, List.of("thumb-1.jpg", "thumb-2.jpg"), + "contact.jpg", "proxy.mp4", 0.8, 0.7, 0.9); + return new HighlightSourceAnalysis(projectId, "porsche.mp4", source, List.of("thumb-1.jpg", "thumb-2.jpg"), + "contact.jpg", "proxy.mp4", "waveform.png", "analysis/scene-segments.json", + List.of(new ShotSegment("shot_0001", 0, 12, 12, 0, 6)), + "analysis/audio-analysis.json", new SourceAudioAnalysis(projectId, true, -18.0, -3.0, + -35.0, 0.5, List.of(new AudioSection("audio_section_0001", "unclassified_audio", 0, 12, 12)), + null), + "analysis/visual-analysis.json", new SourceVisualAnalysis(projectId, 0.6, 0.7, 0.8, 0.75, + "unknown_without_face_detector", List.of(new VisualObjectLabel("porsche", 0.82, "metadata_heuristic")), + List.of("thumb-1.jpg"), "metadata_thumbnail_scene_heuristic"), Instant.parse("2026-07-11T08:00:00Z")); + } + + private CinematicHighlightAnalysis cinematicAnalysis(String projectId) { + return new CinematicHighlightAnalysis(projectId, ContentCategory.CAR_VLOG, 0.91, + List.of("car metadata detected", "hero motion present"), + List.of(new HighlightCandidate("candidate_001", "clip_00001", 1.0, 9.5, 0.94, + "opening_hook", List.of("hero angle", "strong motion"))), + Instant.parse("2026-07-11T08:00:00Z")); + } +} diff --git a/src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java b/src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java new file mode 100644 index 0000000..694d7e0 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/HighlightLocalAssetWorkerTest.java @@ -0,0 +1,64 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class HighlightLocalAssetWorkerTest { + + @TempDir + Path tempDir; + + @Test + void resolvesMusicFromLocalLibraryAndKeepsPendingWhenVoiceoverCannotBeSynthesized() throws Exception { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + properties.getEditing().getAssets().setMusicFolder(tempDir.resolve("music").toString()); + properties.getEditing().getAssets().setSfxFolder(tempDir.resolve("sfx").toString()); + properties.getEditing().getAssets().setVoiceoverFolder(tempDir.resolve("voiceover-cache").toString()); + ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); + FileSystemHighlightProjectStore store = new FileSystemHighlightProjectStore(properties, mapper); + HighlightProject project = new HighlightProject("project-1", "Project", HighlightProjectStatus.CREATED, + "source.mp4", tempDir.resolve("highlight-projects/project-1").toString(), + Instant.parse("2026-07-11T08:00:00Z"), Instant.parse("2026-07-11T08:00:00Z"), null); + store.createProject(project, new HighlightProjectManifest("project-1", "source.mp4", + HighlightFolderContract.standard(), Instant.parse("2026-07-11T08:00:00Z"))); + Path musicSource = tempDir.resolve("music/cinematic-bed.wav"); + Files.createDirectories(musicSource.getParent()); + Files.writeString(musicSource, "music"); + Files.writeString(musicSource.resolveSibling("cinematic-bed.wav.tags.txt"), "cinematic,bed"); + EditAssetProvider assetProvider = mock(EditAssetProvider.class); + EditAssetLibrary assetLibrary = mock(EditAssetLibrary.class); + when(assetLibrary.select(org.mockito.ArgumentMatchers.any())).thenReturn(java.util.Optional.of( + new ResolvedEditAsset(EditAssetType.MUSIC, "cinematic-bed", musicSource.toString(), + "local", "local", ContentCategory.GENERIC_VLOG, List.of("cinematic", "bed")))); + when(assetProvider.resolve(org.mockito.ArgumentMatchers.any())).thenReturn(java.util.Optional.empty()); + when(assetProvider.list(org.mockito.ArgumentMatchers.any(), org.mockito.ArgumentMatchers.any())) + .thenReturn(List.of()); + HighlightAssetPreparationService prep = new HighlightAssetPreparationService(properties, store, + assetProvider, assetLibrary, mapper); + HighlightDirectorPlan.HighlightItem highlight = new HighlightDirectorPlan.HighlightItem( + "highlight_001", "candidate_001", "Opening", 0, 12, 12, "opening_hook", + "premium cinematic grade", "low pulse", "whoosh", + List.of("A clean first look."), List.of("Pure presence"), "open with presence"); + prep.prepare("project-1", project, highlight, ContentCategory.GENERIC_VLOG); + HighlightLocalAssetWorker worker = new HighlightLocalAssetWorker(store, assetProvider, assetLibrary, mapper); + + HighlightLocalAssetWorker.HighlightAssetWorkerResult result = worker.process("project-1", highlight, + ContentCategory.GENERIC_VLOG); + + assertThat(result.resolvedAssets()).isNotEmpty(); + assertThat(store.highlightsDirectory("project-1").resolve("highlight_001/assets/music/music.wav")) + .exists(); + } +} diff --git a/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java b/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java index ae3d8c6..564bb57 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightSourceSchedulerTest.java @@ -27,6 +27,7 @@ class HighlightSourceSchedulerTest { private VideoClippingProperties properties; private FileSystemHighlightProjectStore store; private HighlightSourceAnalyzer analyzer; + private HighlightDirectorPromptGenerator promptGenerator; @BeforeEach void setUp() { @@ -40,6 +41,7 @@ class HighlightSourceSchedulerTest { new HighlightProjectDirectoryInitializer(properties).initialize(); store = new FileSystemHighlightProjectStore(properties, new ObjectMapper().findAndRegisterModules()); analyzer = mock(HighlightSourceAnalyzer.class); + promptGenerator = mock(HighlightDirectorPromptGenerator.class); } @Test @@ -73,6 +75,7 @@ class HighlightSourceSchedulerTest { assertThat(tempDir.resolve("highlight-projects/1/source/1.mp4")).exists(); assertThat(tempDir.resolve("highlight-projects/2")).doesNotExist(); verify(analyzer).analyze("1"); + verify(promptGenerator).generate("1"); } @Test @@ -108,7 +111,7 @@ class HighlightSourceSchedulerTest { when(analyzer.analyze("porsche")).thenReturn(analysis("porsche")); when(analyzer.analyze("porsche-1")).thenReturn(analysis("porsche-1")); when(analyzer.analyze("porsche-drive")).thenReturn(analysis("porsche-drive")); - return new HighlightSourceScheduler(properties, store, analyzer, clock); + return new HighlightSourceScheduler(properties, store, analyzer, promptGenerator, clock); } private HighlightSourceAnalysis analysis(String projectId) { diff --git a/src/test/java/org/example/videoclips/editing/HighlightVisualEffectsStageTest.java b/src/test/java/org/example/videoclips/editing/HighlightVisualEffectsStageTest.java new file mode 100644 index 0000000..221484a --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/HighlightVisualEffectsStageTest.java @@ -0,0 +1,41 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class HighlightVisualEffectsStageTest { + + @TempDir + Path tempDir; + + @Test + void writesExplicitVisualEffectsPlan() throws Exception { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setHighlightProjectDirectory(tempDir.resolve("highlight-projects").toString()); + ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); + FileSystemHighlightProjectStore store = new FileSystemHighlightProjectStore(properties, mapper); + HighlightProject project = new HighlightProject("project-1", "Project", HighlightProjectStatus.CREATED, + "source.mp4", tempDir.resolve("highlight-projects/project-1").toString(), + Instant.parse("2026-07-11T08:00:00Z"), Instant.parse("2026-07-11T08:00:00Z"), null); + store.createProject(project, new HighlightProjectManifest("project-1", "source.mp4", + HighlightFolderContract.standard(), Instant.parse("2026-07-11T08:00:00Z"))); + HighlightVisualEffectsStage stage = new HighlightVisualEffectsStage(store, mapper); + + HighlightVisualEffectsStage.HighlightVisualEffectsPlan plan = stage.create("project-1", + new HighlightDirectorPlan.HighlightItem("highlight_001", "candidate_001", "Opening", 0, 12, 12, + "opening_hook", "premium cinematic grade", "low pulse", "whoosh", + List.of("Narration"), List.of("Title card"), "hold the reveal"), + ContentCategory.GENERIC_VLOG); + + assertThat(plan.effects()).contains("crop", "punch-in", "contrast", "vignette", "fade"); + assertThat(store.highlightsDirectory("project-1").resolve("highlight_001/visual-effects.json")).exists(); + } +} diff --git a/src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java b/src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java new file mode 100644 index 0000000..810212f --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/LocalAssetGenerationStageTest.java @@ -0,0 +1,98 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class LocalAssetGenerationStageTest { + + @TempDir + Path tempDir; + + @Test + void reusesExistingSharedAssetsAndWritesRequestsForMissingVoiceover() throws Exception { + VideoClippingProperties properties = properties(); + Files.createDirectories(tempDir.resolve("shared/music")); + Files.createDirectories(tempDir.resolve("shared/sfx")); + Files.createDirectories(tempDir.resolve("voiceover-cache")); + Files.writeString(tempDir.resolve("shared/music/premium-bed.wav"), "music"); + Files.writeString(tempDir.resolve("shared/sfx/whoosh.wav"), "sfx"); + + properties.getEditing().getAssets().setMusicFolder(tempDir.resolve("shared/music").toString()); + properties.getEditing().getAssets().setSfxFolder(tempDir.resolve("shared/sfx").toString()); + properties.getEditing().getAssets().setVoiceoverFolder(tempDir.resolve("voiceover-cache").toString()); + + FileSystemEditProjectStore store = store(properties); + store.createProject("project"); + + LocalAssetGenerationStage stage = new LocalAssetGenerationStage(properties, store, + new LocalEditAssetProvider(properties), + new LocalEditAssetLibrary(new LocalEditAssetProvider(properties)), + new ObjectMapper().findAndRegisterModules()); + + EditPlan plan = new EditPlan("project", "cinematic-porsche-promo", 4, + List.of( + new EditDecision("clip-1", 0, 4, 0, 4, "cut", "cut", 1.0, "cinematic grade", "opening") + ), + List.of( + new AudioCue("music", "premium-bed", 0, 4, -14, "drive bed"), + new AudioCue("sfx", "whoosh", 1, 2, -6, "impact") + ), + List.of(new VoiceoverLine("Precision in motion.", 0, 2, "confident")), + "mp4-h264-aac-1080p", + "summary"); + + AssetGenerationResult result = stage.prepare("project", plan); + + assertThat(result.readyForRender()).isTrue(); + assertThat(tempDir.resolve("projects/project/audio/music.wav")).exists(); + assertThat(tempDir.resolve("projects/project/audio/sfx/whoosh.wav")).exists(); + assertThat(tempDir.resolve("projects/project/assets/asset-generation-manifest.json")).exists(); + assertThat(tempDir.resolve("projects/project/assets/generated-assets.json")).exists(); + assertThat(tempDir.resolve("projects/project/assets/requests/voiceover")).exists(); + } + + @Test + void writesBlockingRequestsWhenSfxIsMissing() throws Exception { + VideoClippingProperties properties = properties(); + properties.getEditing().getAssets().setMusicFolder(tempDir.resolve("shared/music").toString()); + properties.getEditing().getAssets().setSfxFolder(tempDir.resolve("shared/sfx").toString()); + properties.getEditing().getAssets().setVoiceoverFolder(tempDir.resolve("voiceover-cache").toString()); + + FileSystemEditProjectStore store = store(properties); + store.createProject("project"); + + LocalAssetGenerationStage stage = new LocalAssetGenerationStage(properties, store, + new LocalEditAssetProvider(properties), + new LocalEditAssetLibrary(new LocalEditAssetProvider(properties)), + new ObjectMapper().findAndRegisterModules()); + + EditPlan plan = new EditPlan("project", "cinematic-porsche-promo", 4, + List.of(new EditDecision("clip-1", 0, 4, 0, 4, "cut", "cut", 1.0, "cinematic grade", "opening")), + List.of(new AudioCue("sfx", "engine-hit", 1, 2, -6, "impact")), + List.of(), "mp4-h264-aac-1080p", "summary"); + + AssetGenerationResult result = stage.prepare("project", plan); + + assertThat(result.readyForRender()).isFalse(); + assertThat(tempDir.resolve("projects/project/assets/requests/sfx")).exists(); + } + + private VideoClippingProperties properties() { + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().setProjectDirectory(tempDir.resolve("projects").toString()); + return properties; + } + + private FileSystemEditProjectStore store(VideoClippingProperties properties) { + return new FileSystemEditProjectStore(properties, new ObjectMapper().findAndRegisterModules()); + } +} diff --git a/src/test/java/org/example/videoclips/editing/StoryboardPromptGeneratorTest.java b/src/test/java/org/example/videoclips/editing/StoryboardPromptGeneratorTest.java index 87831d7..41f866e 100644 --- a/src/test/java/org/example/videoclips/editing/StoryboardPromptGeneratorTest.java +++ b/src/test/java/org/example/videoclips/editing/StoryboardPromptGeneratorTest.java @@ -30,6 +30,7 @@ class StoryboardPromptGeneratorTest { assertThat(prompt) .contains("clipId: clip_00001") .contains("durationSeconds: 8.0") + .contains("The service can only render what the plan expresses") .contains("Target duration: 60 seconds") .contains("Style: `cinematic-porsche-promo`") .contains("\"decisions\"")