From 2dc3ec97f43c899653bc400e9f5f04f6e199399b Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Sat, 11 Jul 2026 09:28:13 +0200 Subject: [PATCH] Add cinematic benchmark metrics --- ...uction-cinematic-highlight-editing-plan.md | 3 ++- .../editing/AiDirectorPromptGenerator.java | 26 ++++++++++++++++++- .../videoclips/editing/EditObservability.java | 20 +++++++++++--- .../editing/FfmpegEditRenderer.java | 11 +++++++- .../AiDirectorPromptGeneratorTest.java | 10 +++++++ .../editing/EditObservabilityTest.java | 9 ++++++- 6 files changed, 72 insertions(+), 7 deletions(-) diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index 9ea281a..10d1b2b 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -315,7 +315,8 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 15 progress: Added `docs/cinematic-editing-runbook.md` with local setup, source placement, AI director model guidance, edit-plan review, rendering, final output lookup, QA report review, and troubleshooting. - [x] Milestone 16: Add integration tests with small fixture videos for family, food, car, and generic content. - [x] Milestone 16 progress: Added FFmpeg-backed local director integration coverage for car, food, family, and generic fixture projects, verifying category artifacts, highlight candidates, and director prompts. -- [ ] Milestone 17: Add benchmark metrics for analysis time, render time, token usage, asset generation cost, and final output size. +- [x] Milestone 17: Add benchmark metrics for analysis time, render time, token usage, asset generation cost, and final output size. +- [x] Milestone 17 progress: Added Micrometer benchmark summaries for analysis clip/error counts, render output duration, render output size, AI director prompt token estimate, and asset generation cost baseline. - [ ] Milestone 18: Add a human review mode where the user can approve or edit the AI director plan before rendering. ## Testing Strategy diff --git a/src/main/java/org/example/videoclips/editing/AiDirectorPromptGenerator.java b/src/main/java/org/example/videoclips/editing/AiDirectorPromptGenerator.java index 3237e8b..cb39ca8 100644 --- a/src/main/java/org/example/videoclips/editing/AiDirectorPromptGenerator.java +++ b/src/main/java/org/example/videoclips/editing/AiDirectorPromptGenerator.java @@ -1,6 +1,7 @@ package org.example.videoclips.editing; import org.example.videoclips.config.VideoClippingProperties; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; import org.springframework.stereotype.Component; import org.slf4j.Logger; @@ -22,16 +23,28 @@ public class AiDirectorPromptGenerator { private final StoryboardPromptGenerator storyboardPromptGenerator; private final String promptFileName; private final String expectedPlanFileName; + private final EditObservability observability; public AiDirectorPromptGenerator( EditProjectStore store, StoryboardPromptGenerator storyboardPromptGenerator, VideoClippingProperties properties + ) { + this(store, storyboardPromptGenerator, properties, null); + } + + @Autowired + public AiDirectorPromptGenerator( + EditProjectStore store, + StoryboardPromptGenerator storyboardPromptGenerator, + VideoClippingProperties properties, + EditObservability observability ) { this.store = store; this.storyboardPromptGenerator = storyboardPromptGenerator; this.promptFileName = safeFileName(properties.getEditing().getLocalDirector().getDirectorPromptFileName()); this.expectedPlanFileName = safeFileName(properties.getEditing().getLocalDirector().getExpectedPlanFileName()); + this.observability = observability; } public DirectorPromptFiles generate(String projectId) { @@ -40,16 +53,27 @@ public class AiDirectorPromptGenerator { Path projectDirectory = store.projectDirectory(projectId); Path promptPath = projectDirectory.resolve(promptFileName); Path readmePath = projectDirectory.resolve(README_FILE_NAME); + String prompt = directorPrompt(project, analysis); try { - Files.writeString(promptPath, directorPrompt(project, analysis)); + Files.writeString(promptPath, prompt); Files.writeString(readmePath, readme()); } catch (IOException ex) { throw new IllegalStateException("Unable to write AI director files for project: " + projectId, ex); } log.info("event=ai_director_prompt_generated project_id={} path={}", projectId, promptPath); + if (observability != null) { + observability.directorPromptGenerated(projectId, promptPath.toString(), estimateTokens(prompt), 0.0); + } return new DirectorPromptFiles(promptPath.toString(), readmePath.toString()); } + int estimateTokens(String prompt) { + if (prompt == null || prompt.isBlank()) { + return 0; + } + return (int) Math.ceil(prompt.length() / 4.0); + } + private String directorPrompt(EditProject project, EditProjectAnalysis analysis) { return """ # Local AI Director Assignment diff --git a/src/main/java/org/example/videoclips/editing/EditObservability.java b/src/main/java/org/example/videoclips/editing/EditObservability.java index 52de9fa..5ddf483 100644 --- a/src/main/java/org/example/videoclips/editing/EditObservability.java +++ b/src/main/java/org/example/videoclips/editing/EditObservability.java @@ -31,6 +31,8 @@ public class EditObservability { public void analysisCompleted(String projectId, long elapsedNanos, int clipCount, int errorCount) { registry.timer("video.clipping.edit.analysis.duration").record(Duration.ofNanos(elapsedNanos)); + registry.summary("video.clipping.edit.analysis.clip.count").record(clipCount); + registry.summary("video.clipping.edit.analysis.error.count").record(errorCount); log.info("event=edit_analysis_completed project_id={} elapsed_ms={} clip_count={} error_count={}", projectId, elapsedNanos / 1_000_000, clipCount, errorCount); } @@ -48,10 +50,13 @@ public class EditObservability { log.info("event=edit_render_started project_id={} decision_count={}", projectId, decisionCount); } - public void renderCompleted(String projectId, long elapsedNanos, String outputPath, double durationSeconds) { + public void renderCompleted(String projectId, long elapsedNanos, String outputPath, double durationSeconds, + long outputSizeBytes) { registry.timer("video.clipping.edit.render.duration").record(Duration.ofNanos(elapsedNanos)); - log.info("event=edit_render_completed project_id={} elapsed_ms={} output={} duration_seconds={}", - projectId, elapsedNanos / 1_000_000, outputPath, durationSeconds); + registry.summary("video.clipping.edit.render.output.duration.seconds").record(durationSeconds); + registry.summary("video.clipping.edit.render.output.size.bytes").record(outputSizeBytes); + log.info("event=edit_render_completed project_id={} elapsed_ms={} output={} duration_seconds={} output_size_bytes={}", + projectId, elapsedNanos / 1_000_000, outputPath, durationSeconds, outputSizeBytes); } public void renderFailed(String projectId, long elapsedNanos, RuntimeException failure) { @@ -60,4 +65,13 @@ public class EditObservability { log.error("event=edit_render_failed project_id={} elapsed_ms={} error_type={} message={}", projectId, elapsedNanos / 1_000_000, failure.getClass().getSimpleName(), failure.getMessage()); } + + public void directorPromptGenerated(String projectId, String path, int estimatedPromptTokens, + double estimatedAssetGenerationCostUsd) { + registry.summary("video.clipping.edit.director.prompt.tokens").record(estimatedPromptTokens); + registry.summary("video.clipping.edit.asset.generation.cost.usd").record(estimatedAssetGenerationCostUsd); + log.info("event=edit_director_prompt_benchmark project_id={} path={} estimated_prompt_tokens={} " + + "estimated_asset_generation_cost_usd={}", + projectId, path, estimatedPromptTokens, estimatedAssetGenerationCostUsd); + } } diff --git a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java index abdba6e..e08c1b9 100644 --- a/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java +++ b/src/main/java/org/example/videoclips/editing/FfmpegEditRenderer.java @@ -160,7 +160,8 @@ public class FfmpegEditRenderer implements EditRenderer { store.writeJson(projectId, "render-manifest.json", manifest); projectService.updateProject(projectId, EditProjectStatus.RENDERED, Path.of(project.inputDirectory()), null); if (observability != null) { - observability.renderCompleted(projectId, System.nanoTime() - startedAt, output.toString(), duration); + observability.renderCompleted(projectId, System.nanoTime() - startedAt, output.toString(), duration, + fileSize(output)); } } @@ -517,6 +518,14 @@ public class FfmpegEditRenderer implements EditRenderer { } } + private long fileSize(Path file) { + try { + return Files.size(file); + } catch (IOException ex) { + return 0; + } + } + private void run(List command) { try { ProcessResult result = executor.execute(command); diff --git a/src/test/java/org/example/videoclips/editing/AiDirectorPromptGeneratorTest.java b/src/test/java/org/example/videoclips/editing/AiDirectorPromptGeneratorTest.java index 9747971..44af202 100644 --- a/src/test/java/org/example/videoclips/editing/AiDirectorPromptGeneratorTest.java +++ b/src/test/java/org/example/videoclips/editing/AiDirectorPromptGeneratorTest.java @@ -70,6 +70,16 @@ class AiDirectorPromptGeneratorTest { new AiDirectorPromptGenerator(store, new StoryboardPromptGenerator(store), properties)); } + @Test + void estimatesPromptTokensFromPromptCharacters() { + FileSystemEditProjectStore store = store(properties()); + AiDirectorPromptGenerator generator = new AiDirectorPromptGenerator( + store, new StoryboardPromptGenerator(store), properties()); + + assertThat(generator.estimateTokens("123456789")).isEqualTo(3); + assertThat(generator.estimateTokens("")).isZero(); + } + private void writeProjectFiles(FileSystemEditProjectStore store) { store.createProject("porsche-edit"); Instant now = Instant.parse("2026-07-10T10:00:00Z"); diff --git a/src/test/java/org/example/videoclips/editing/EditObservabilityTest.java b/src/test/java/org/example/videoclips/editing/EditObservabilityTest.java index 95968cf..904e22b 100644 --- a/src/test/java/org/example/videoclips/editing/EditObservabilityTest.java +++ b/src/test/java/org/example/videoclips/editing/EditObservabilityTest.java @@ -16,11 +16,18 @@ class EditObservabilityTest { observability.analysisStarted("project", 35); observability.analysisCompleted("project", 2_000_000, 35, 0); observability.renderStarted("project", 12); - observability.renderCompleted("project", 5_000_000, "final.mp4", 60); + observability.renderCompleted("project", 5_000_000, "final.mp4", 60, 1_024); + observability.directorPromptGenerated("project", "ai-director-prompt.md", 2_500, 0.0); assertThat(registry.counter("video.clipping.edit.projects.created").count()).isEqualTo(1); assertThat(registry.timer("video.clipping.edit.analysis.duration").count()).isEqualTo(1); + assertThat(registry.summary("video.clipping.edit.analysis.clip.count").totalAmount()).isEqualTo(35); + assertThat(registry.summary("video.clipping.edit.analysis.error.count").totalAmount()).isZero(); assertThat(registry.timer("video.clipping.edit.render.duration").count()).isEqualTo(1); + assertThat(registry.summary("video.clipping.edit.render.output.duration.seconds").totalAmount()).isEqualTo(60); + assertThat(registry.summary("video.clipping.edit.render.output.size.bytes").totalAmount()).isEqualTo(1_024); + assertThat(registry.summary("video.clipping.edit.director.prompt.tokens").totalAmount()).isEqualTo(2_500); + assertThat(registry.summary("video.clipping.edit.asset.generation.cost.usd").count()).isEqualTo(1); assertThat(registry.counter("video.clipping.edit.render.failures").count()).isZero(); }