From 2d0c3317105e6fe1328985c6643a54a08fc5677c Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Sat, 11 Jul 2026 01:20:39 +0200 Subject: [PATCH] Add cinematic dynamic crop rendering --- ...uction-cinematic-highlight-editing-plan.md | 4 +-- .../editing/FfmpegEditRenderer.java | 13 +++++++- .../editing/FfmpegEditRendererTest.java | 32 +++++++++++++++---- 3 files changed, 40 insertions(+), 9 deletions(-) diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index 5ac7341..db5b235 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -307,8 +307,8 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 10: Implement edit plan validation so impossible timestamps, missing assets, invalid overlays, and unsupported effects fail before render. - [x] Milestone 11: Implement asset provider interfaces for voiceover, music, SFX, fonts, and LUTs. - [x] Milestone 12: Add a local licensed asset library with category tags and deterministic asset selection. -- [ ] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests. -- [ ] Milestone 13 progress: Renderer now applies timed text overlays, records resolved/planned edit assets in the render manifest, ducks music under voiceover, and normalizes final mix loudness with configurable mastering values. Dynamic crop keyframes remain. +- [x] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests. +- [x] Milestone 13 progress: Renderer now applies timed text overlays, records resolved/planned edit assets in the render manifest, adds dynamic punch-in crop motion, ducks music under voiceover, and normalizes final mix loudness with configurable mastering values. - [ ] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters. - [ ] Milestone 15: Add an operator runbook for placing a video, running the service locally, choosing a model, reviewing the director plan, and finding final clips. - [ ] Milestone 16: Add integration tests with small fixture videos for family, food, car, and generic content. diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index a399904..21744b2 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -164,7 +164,9 @@ public class FfmpegEditRenderer implements EditRenderer { List segmentCommand(String source, EditDecision decision, Path output) { double outputDuration = (decision.sourceEndSeconds() - decision.sourceStartSeconds()) / decision.playbackSpeed(); - StringBuilder filter = new StringBuilder("scale=%d:%d:force_original_aspect_ratio=decrease,".formatted( + StringBuilder filter = new StringBuilder(); + filter.append(dynamicCropFilter(decision.visualTreatment())); + filter.append("scale=%d:%d:force_original_aspect_ratio=decrease,".formatted( properties.getOutputWidth(), properties.getOutputHeight()) + "pad=%d:%d:(ow-iw)/2:(oh-ih)/2,format=yuv420p".formatted( properties.getOutputWidth(), properties.getOutputHeight())); @@ -201,6 +203,15 @@ public class FfmpegEditRenderer implements EditRenderer { + ",vignette=PI/7"; } + String dynamicCropFilter(String visualTreatment) { + if (visualTreatment == null || visualTreatment.isBlank() || "none".equalsIgnoreCase(visualTreatment)) { + return ""; + } + return "crop=w='iw*0.96':h='ih*0.96':" + + "x='(iw-out_w)/2+(iw-out_w)*0.25*sin(t*0.6)':" + + "y='(ih-out_h)/2+(ih-out_h)*0.18*cos(t*0.5)',"; + } + String audioTempoFilter(double speed) { if (speed < 0.5) { return "atempo=0.5,atempo=" + (speed / 0.5); diff --git a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java index c8c3e6b..303e04b 100644 --- a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java +++ b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java @@ -22,7 +22,7 @@ class FfmpegEditRendererTest { var command = renderer.segmentCommand("source.mp4", decision, Path.of("segment.mp4")); assertThat(command).containsSubsequence("-ss", "1.25", "-to", "5.5", "-i", "source.mp4"); - assertThat(command).anyMatch(argument -> argument.startsWith( + assertThat(command).anyMatch(argument -> argument.contains( "scale=1920:1080:force_original_aspect_ratio=decrease," + "pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p")); assertThat(command).containsSubsequence("-c:v", "libx264", "-c:a", "aac"); @@ -105,14 +105,33 @@ class FfmpegEditRendererTest { var command = renderer.segmentCommand("source.mp4", decision, Path.of("segment.mp4")); - assertThat(command).contains("scale=1920:1080:force_original_aspect_ratio=decrease," - + "pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/2.0," - + "eq=contrast=1.12:saturation=1.18:brightness=-0.015," - + "unsharp=5:5:0.55:3:3:0.25,vignette=PI/7," - + "fade=t=in:st=0:d=0.5,fade=t=out:st=1.5:d=0.5"); + assertThat(command).anySatisfy(argument -> assertThat(argument).contains( + "scale=1920:1080:force_original_aspect_ratio=decrease," + + "pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/2.0," + + "eq=contrast=1.12:saturation=1.18:brightness=-0.015," + + "unsharp=5:5:0.55:3:3:0.25,vignette=PI/7," + + "fade=t=in:st=0:d=0.5,fade=t=out:st=1.5:d=0.5")); assertThat(command).containsSubsequence("-af", "atempo=2.0"); } + @Test + void appliesDynamicCropBeforeScalingForCinematicTreatments() { + VideoClippingProperties properties = new VideoClippingProperties(); + FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class), + mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null); + EditDecision decision = new EditDecision("clip-1", 0, 4, 0, 4, + "cut", "cut", 1, "grade", "reason"); + + var command = renderer.segmentCommand("source.mp4", decision, Path.of("segment.mp4")); + + assertThat(command).anySatisfy(argument -> assertThat(argument).contains( + "crop=w='iw*0.96':h='ih*0.96':" + + "x='(iw-out_w)/2+(iw-out_w)*0.25*sin(t*0.6)':" + + "y='(ih-out_h)/2+(ih-out_h)*0.18*cos(t*0.5)'," + + "scale=1920:1080:force_original_aspect_ratio=decrease," + + "pad=1920:1080:(ow-iw)/2:(oh-ih)/2,format=yuv420p,setpts=PTS/1.0")); + } + @Test void skipsCinematicGradeWhenVisualTreatmentIsNone() { VideoClippingProperties properties = new VideoClippingProperties(); @@ -124,6 +143,7 @@ class FfmpegEditRendererTest { var command = renderer.segmentCommand("source.mp4", decision, Path.of("segment.mp4")); assertThat(command).noneMatch(argument -> argument.contains("eq=contrast=1.12")); + assertThat(command).noneMatch(argument -> argument.contains("crop=w='iw*0.96'")); } @Test