From 9b56e89b4c608ca4668b3e4c4a4db2f5f309f825 Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Sat, 11 Jul 2026 01:25:38 +0200 Subject: [PATCH] Add cinematic render QA report --- ...uction-cinematic-highlight-editing-plan.md | 3 +- .../editing/FfmpegEditRenderer.java | 133 ++++++++++++++++++ .../videoclips/editing/RenderQaCheck.java | 9 ++ .../videoclips/editing/RenderQaReport.java | 15 ++ .../CinematicEditingIntegrationTest.java | 1 + .../editing/FfmpegEditRendererTest.java | 27 ++++ .../editing/RenderQaReportTest.java | 25 ++++ 7 files changed, 212 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/example/videoclips/editing/RenderQaCheck.java create mode 100644 src/main/java/org/example/videoclips/editing/RenderQaReport.java create mode 100644 src/test/java/org/example/videoclips/editing/RenderQaReportTest.java diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index db5b235..1c3c1e9 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -309,7 +309,8 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 12: Add a local licensed asset library with category tags and deterministic asset selection. - [x] Milestone 13: Implement renderer v2 with transitions, speed ramps, dynamic crops, overlays, visual effects, audio ducking, loudness normalization, and render manifests. - [x] Milestone 13 progress: Renderer now applies timed text overlays, records resolved/planned edit assets in the render manifest, adds dynamic punch-in crop motion, ducks music under voiceover, and normalizes final mix loudness with configurable mastering values. -- [ ] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters. +- [x] Milestone 14: Implement QA checks for black frames, silence, clipping, missing assets, duration mismatch, unsafe text placement, and failed FFmpeg filters. +- [x] Milestone 14 progress: Renderer now writes `qa-report.json` with structural checks, asset resolution checks, overlay safety checks, audio mastering checks, FFmpeg command completion checks, and FFmpeg probe checks for black frames, long silence, and audio clipping. - [ ] Milestone 15: Add an operator runbook for placing a video, running the service locally, choosing a model, reviewing the director plan, and finding final clips. - [ ] Milestone 16: Add integration tests with small fixture videos for family, food, car, and generic content. - [ ] Milestone 17: Add benchmark metrics for analysis time, render time, token usage, asset generation cost, and final output size. diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index 21744b2..abdba6e 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -152,9 +152,11 @@ public class FfmpegEditRenderer implements EditRenderer { copy(videoTimeline, output); } double duration = plan.decisions().get(plan.decisions().size() - 1).timelineEndSeconds(); + RenderQaReport qaReport = buildQaReport(projectId, plan, output, duration, commands, assets); RenderManifest manifest = new RenderManifest(projectId, plan.decisions().stream().map(EditDecision::clipId).toList(), output.toString(), duration, List.copyOf(commands), assets, Instant.now()); + store.writeJson(projectId, "qa-report.json", qaReport); store.writeJson(projectId, "render-manifest.json", manifest); projectService.updateProject(projectId, EditProjectStatus.RENDERED, Path.of(project.inputDirectory()), null); if (observability != null) { @@ -357,6 +359,137 @@ public class FfmpegEditRenderer implements EditRenderer { return List.copyOf(assets); } + RenderQaReport buildQaReport(String projectId, EditPlan plan, Path output, double duration, + List> commands, List assets) { + List checks = new ArrayList<>(); + checks.add(outputExistsCheck(output)); + checks.add(durationMatchesTimelineCheck(plan, duration)); + checks.add(requiredAssetsResolvedCheck(plan, assets)); + checks.add(overlaysSafeCheck(plan, duration)); + checks.add(audioMasteringCheck(commands)); + checks.add(ffmpegCommandsCompletedCheck(commands)); + checks.add(runQaProbe("black_frames", blackDetectCommand(output), "ERROR", + result -> result.exitCode() == 0 && !result.output().contains("black_start"), + "blackdetect found one or more black frame ranges")); + checks.add(runQaProbe("long_silence", silenceDetectCommand(output), "WARNING", + result -> result.exitCode() == 0 && !result.output().contains("silence_start"), + "silencedetect found one or more long silent ranges")); + checks.add(runQaProbe("audio_clipping", clippingDetectCommand(output), "WARNING", + this::audioPeakIsSafe, + "audio peak is at or above -0.1 dBFS")); + boolean passed = checks.stream().noneMatch(check -> !check.passed() && "ERROR".equals(check.severity())); + return new RenderQaReport(projectId, passed, checks, Instant.now()); + } + + private RenderQaCheck outputExistsCheck(Path output) { + boolean exists = Files.isRegularFile(output); + return new RenderQaCheck("output_exists", exists, "ERROR", + exists ? "final output exists" : "final output is missing: " + output); + } + + private RenderQaCheck durationMatchesTimelineCheck(EditPlan plan, double duration) { + double expected = plan.decisions().isEmpty() ? 0 : plan.decisions().get(plan.decisions().size() - 1) + .timelineEndSeconds(); + boolean matches = Math.abs(expected - duration) <= 0.05; + return new RenderQaCheck("duration_matches_timeline", matches, "ERROR", + "expected=%s actual=%s tolerance=0.05".formatted(expected, duration)); + } + + private RenderQaCheck requiredAssetsResolvedCheck(EditPlan plan, List assets) { + List missing = new ArrayList<>(); + for (AudioCue cue : plan.audioCues()) { + if ("sfx".equals(cue.type()) && assets.stream() + .noneMatch(asset -> asset.type() == EditAssetType.SFX && cue.assetKey().equals(asset.assetKey()))) { + missing.add("sfx:" + cue.assetKey()); + } + } + if (!plan.voiceover().isEmpty() && assets.stream().noneMatch(asset -> asset.type() == EditAssetType.VOICEOVER)) { + missing.add("voiceover"); + } + return new RenderQaCheck("required_assets_resolved", missing.isEmpty(), "ERROR", + missing.isEmpty() ? "all required render assets were resolved" : "missing assets: " + missing); + } + + private RenderQaCheck overlaysSafeCheck(EditPlan plan, double duration) { + List unsafe = plan.overlays().stream() + .filter(overlay -> overlay.timelineStartSeconds() < 0 + || overlay.timelineEndSeconds() > duration + || overlay.timelineStartSeconds() >= overlay.timelineEndSeconds() + || !safeOverlayPlacements().contains(overlay.placement())) + .map(TextOverlay::text) + .toList(); + return new RenderQaCheck("text_overlays_safe", unsafe.isEmpty(), "ERROR", + unsafe.isEmpty() ? "all overlays are inside the timeline and safe placements" + : "unsafe overlays: " + unsafe); + } + + private List safeOverlayPlacements() { + return List.of("lower_left_safe", "upper_left_safe", "upper_right_safe", "center_safe", "lower_center_safe"); + } + + private RenderQaCheck audioMasteringCheck(List> commands) { + boolean audioMixPresent = commands.stream().flatMap(List::stream).anyMatch(argument -> argument.contains("amix")); + boolean mastered = commands.stream().flatMap(List::stream).anyMatch(argument -> argument.contains("loudnorm")); + boolean passed = !audioMixPresent || mastered; + return new RenderQaCheck("audio_mastering_applied", passed, "WARNING", + passed ? "audio mix is normalized when present" : "audio mix is missing loudnorm normalization"); + } + + private RenderQaCheck ffmpegCommandsCompletedCheck(List> commands) { + return new RenderQaCheck("ffmpeg_commands_completed", !commands.isEmpty(), "ERROR", + "recorded_command_count=" + commands.size()); + } + + private RenderQaCheck runQaProbe(String name, List command, String severity, + java.util.function.Predicate passed, + String failureDetails) { + try { + ProcessResult result = executor.execute(command); + if (passed.test(result)) { + return new RenderQaCheck(name, true, severity, "probe passed"); + } + return new RenderQaCheck(name, false, severity, result.exitCode() == 0 + ? failureDetails : "probe exited with code " + result.exitCode()); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + return new RenderQaCheck(name, false, severity, "probe interrupted"); + } catch (IOException ex) { + return new RenderQaCheck(name, false, severity, "probe failed: " + ex.getMessage()); + } + } + + List blackDetectCommand(Path output) { + return List.of(properties.getFfmpegBinary(), "-hide_banner", "-v", "info", "-i", output.toString(), + "-vf", "blackdetect=d=0.5:pic_th=0.98", "-an", "-f", "null", "-"); + } + + List silenceDetectCommand(Path output) { + return List.of(properties.getFfmpegBinary(), "-hide_banner", "-v", "info", "-i", output.toString(), + "-af", "silencedetect=noise=-45dB:d=2", "-vn", "-f", "null", "-"); + } + + List clippingDetectCommand(Path output) { + return List.of(properties.getFfmpegBinary(), "-hide_banner", "-v", "info", "-i", output.toString(), + "-af", "astats=metadata=1:reset=1", "-vn", "-f", "null", "-"); + } + + boolean audioPeakIsSafe(ProcessResult result) { + if (result.exitCode() != 0) { + return false; + } + return result.output().lines() + .filter(line -> line.contains("Peak level dB")) + .map(line -> line.substring(line.lastIndexOf(':') + 1).trim()) + .mapToDouble(value -> { + try { + return Double.parseDouble(value); + } catch (NumberFormatException ex) { + return -99; + } + }) + .noneMatch(peak -> peak >= -0.1); + } + private void writeConcatFile(Path file, List segments) { String content = segments.stream() .map(path -> "file '" + path.toAbsolutePath().toString().replace("'", "'\\''") + "'") diff --git a/src/main/java/org/example/videoclips/editing/RenderQaCheck.java b/src/main/java/org/example/videoclips/editing/RenderQaCheck.java new file mode 100644 index 0000000..698dace --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/RenderQaCheck.java @@ -0,0 +1,9 @@ +package org.example.videoclips.editing; + +public record RenderQaCheck( + String name, + boolean passed, + String severity, + String details +) { +} diff --git a/src/main/java/org/example/videoclips/editing/RenderQaReport.java b/src/main/java/org/example/videoclips/editing/RenderQaReport.java new file mode 100644 index 0000000..049a4a4 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/RenderQaReport.java @@ -0,0 +1,15 @@ +package org.example.videoclips.editing; + +import java.time.Instant; +import java.util.List; + +public record RenderQaReport( + String projectId, + boolean passed, + List checks, + Instant completedAt +) { + public RenderQaReport { + checks = checks == null ? List.of() : List.copyOf(checks); + } +} diff --git a/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java b/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java index 4b49f34..dffbb1f 100644 --- a/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java +++ b/src/test/java/org/example/videoclips/editing/CinematicEditingIntegrationTest.java @@ -61,6 +61,7 @@ class CinematicEditingIntegrationTest { assertThat(output).isRegularFile(); assertThat(probeDuration(output)).isGreaterThan(0); assertThat(projectDirectory.resolve("render-manifest.json")).exists(); + assertThat(projectDirectory.resolve("qa-report.json")).exists(); assertThat(projects.getProject(projectId).status()).isEqualTo(EditProjectStatus.RENDERED); } diff --git a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java index 303e04b..a3f601d 100644 --- a/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java +++ b/src/test/java/org/example/videoclips/editing/FfmpegEditRendererTest.java @@ -184,6 +184,33 @@ class FfmpegEditRendererTest { assertThat(command).containsSubsequence("-c:v", "libx264", "-c:a", "copy"); } + @Test + void buildsQaProbeCommands() { + VideoClippingProperties properties = new VideoClippingProperties(); + FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class), + mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null); + + assertThat(renderer.blackDetectCommand(Path.of("final.mp4"))) + .containsSubsequence("-vf", "blackdetect=d=0.5:pic_th=0.98"); + assertThat(renderer.silenceDetectCommand(Path.of("final.mp4"))) + .containsSubsequence("-af", "silencedetect=noise=-45dB:d=2"); + assertThat(renderer.clippingDetectCommand(Path.of("final.mp4"))) + .containsSubsequence("-af", "astats=metadata=1:reset=1"); + } + + @Test + void detectsUnsafeAudioPeaksFromAstatsOutput() { + VideoClippingProperties properties = new VideoClippingProperties(); + FfmpegEditRenderer renderer = new FfmpegEditRenderer(properties, mock(EditProjectStore.class), + mock(EditProjectService.class), mock(EditPlanValidator.class), command -> null); + + assertThat(renderer.audioPeakIsSafe(new FfmpegEditRenderer.ProcessResult(0, + "[Parsed_astats] Peak level dB: -1.2"))).isTrue(); + assertThat(renderer.audioPeakIsSafe(new FfmpegEditRenderer.ProcessResult(0, + "[Parsed_astats] Peak level dB: 0.0"))).isFalse(); + assertThat(renderer.audioPeakIsSafe(new FfmpegEditRenderer.ProcessResult(1, "failed"))).isFalse(); + } + @Test void chainsAudioTempoAtQuarterSpeed() { VideoClippingProperties properties = new VideoClippingProperties(); diff --git a/src/test/java/org/example/videoclips/editing/RenderQaReportTest.java b/src/test/java/org/example/videoclips/editing/RenderQaReportTest.java new file mode 100644 index 0000000..e7416c9 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/RenderQaReportTest.java @@ -0,0 +1,25 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.junit.jupiter.api.Test; + +import java.time.Instant; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +class RenderQaReportTest { + + @Test + void roundTripsQaReportAsJson() throws Exception { + ObjectMapper mapper = new ObjectMapper().findAndRegisterModules(); + RenderQaReport report = new RenderQaReport("project", true, + List.of(new RenderQaCheck("output_exists", true, "ERROR", "final output exists")), + Instant.parse("2026-07-10T10:00:00Z")); + + String json = mapper.writeValueAsString(report); + + assertThat(json).contains("\"checks\"").contains("output_exists").contains("\"passed\":true"); + assertThat(mapper.readValue(json, RenderQaReport.class)).isEqualTo(report); + } +}