From 82e557905e27bf1686e37afb98479eb77497fddf Mon Sep 17 00:00:00 2001 From: JSLMPR Date: Sat, 11 Jul 2026 10:49:11 +0200 Subject: [PATCH] Add local CV visual analysis provider --- docs/local-cv-visual-analysis-guide.md | 130 ++++++++++++++ ...uction-cinematic-highlight-editing-plan.md | 1 + .../config/VideoClippingProperties.java | 49 ++++++ .../HeuristicVisualAnalysisProvider.java | 102 +++++++++++ .../LocalCvVisualAnalysisProvider.java | 117 +++++++++++++ .../editing/SourceVisualAnalyzer.java | 144 +++++++--------- .../editing/VisualAnalysisProvider.java | 8 + src/main/resources/application.yml | 5 + .../config/VideoClippingPropertiesTest.java | 14 ++ .../LocalCvVisualAnalysisProviderTest.java | 94 ++++++++++ .../editing/SourceVisualAnalyzerTest.java | 70 ++++++++ tools/local_cv_worker.py | 163 ++++++++++++++++++ 12 files changed, 818 insertions(+), 79 deletions(-) create mode 100644 docs/local-cv-visual-analysis-guide.md create mode 100644 src/main/java/org/example/videoclips/editing/HeuristicVisualAnalysisProvider.java create mode 100644 src/main/java/org/example/videoclips/editing/LocalCvVisualAnalysisProvider.java create mode 100644 src/main/java/org/example/videoclips/editing/VisualAnalysisProvider.java create mode 100644 src/test/java/org/example/videoclips/editing/LocalCvVisualAnalysisProviderTest.java create mode 100644 tools/local_cv_worker.py diff --git a/docs/local-cv-visual-analysis-guide.md b/docs/local-cv-visual-analysis-guide.md new file mode 100644 index 0000000..a7c3dcc --- /dev/null +++ b/docs/local-cv-visual-analysis-guide.md @@ -0,0 +1,130 @@ +# Local CV Visual Analysis Guide + +This service can use a local computer-vision worker for highlight visual analysis. + +## Default Behavior + +By default the service uses the built-in heuristic provider: + +```yaml +video-clipping: + editing: + visual-analysis: + provider: heuristic +``` + +This keeps the app runnable without Python, GPU drivers, model downloads, or a separate service. + +## Enable A Local CV Worker + +Run a local worker that accepts JSON over HTTP, then start this Spring service with: + +```yaml +video-clipping: + editing: + visual-analysis: + provider: local-cv + endpoint: http://127.0.0.1:8091/v1/analyze-visuals + timeout-ms: 30000 + fallback-to-heuristic: true +``` + +Equivalent environment variables: + +```bash +VIDEO_EDITING_VISUAL_ANALYSIS_PROVIDER=local-cv +VIDEO_EDITING_VISUAL_ANALYSIS_ENDPOINT=http://127.0.0.1:8091/v1/analyze-visuals +VIDEO_EDITING_VISUAL_ANALYSIS_TIMEOUT_MS=30000 +VIDEO_EDITING_VISUAL_ANALYSIS_FALLBACK_TO_HEURISTIC=true +``` + +This repository includes an optional starter worker: + +```bash +python3 -m venv .venv-local-cv +. .venv-local-cv/bin/activate +pip install fastapi uvicorn opencv-python + +# Optional YOLO object detection: +pip install ultralytics +export LOCAL_CV_YOLO_MODEL=yolov8n.pt + +uvicorn tools.local_cv_worker:app --host 127.0.0.1 --port 8091 +``` + +## Worker Request Contract + +The Spring service sends: + +```json +{ + "source": { + "clipId": "porsche-drive", + "sourcePath": "output/highlight-projects/porsche-drive/source/porsche.mp4", + "durationSeconds": 42.0, + "videoCodec": "h264", + "audioCodec": "aac", + "width": 1920, + "height": 1080, + "frameRate": 30.0 + }, + "thumbnails": [ + "output/highlight-projects/porsche-drive/analysis/frames/porsche_0001.jpg" + ], + "shotSegments": [ + { + "shotId": "shot_0001", + "startSeconds": 0.0, + "endSeconds": 8.0, + "durationSeconds": 8.0, + "representativeTimestampSeconds": 4.0 + } + ] +} +``` + +## Worker Response Contract + +The worker must return: + +```json +{ + "clipId": "porsche-drive", + "blurScore": 0.81, + "exposureScore": 0.67, + "motionScore": 0.73, + "compositionScore": 0.79, + "facePresence": "faces_detected", + "objectLabels": [ + { + "label": "car", + "confidence": 0.91, + "source": "yolo" + }, + { + "label": "luxury sports car", + "confidence": 0.84, + "source": "clip" + } + ], + "representativeThumbnails": [ + "output/highlight-projects/porsche-drive/analysis/frames/porsche_0001.jpg" + ], + "analysisMethod": "local_cv_yolo_clip_mediapipe" +} +``` + +Scores must be normalized from `0.0` to `1.0`. + +## Recommended Local Model Split + +- Use `YOLO` for object detection such as `car`, `person`, `food`, `dog`, `bottle`, and scene objects. +- Use `CLIP` or `SigLIP` for semantic labels such as `luxury sports car`, `family birthday`, or `restaurant dish`. +- Use `MediaPipe` or OpenCV Haar cascades for face presence when lightweight local face detection is enough. +- Use OpenCV/Laplacian variance and brightness histograms for blur and exposure scores. + +## Runtime Behavior + +When `fallback-to-heuristic=true`, the service logs `event=local_cv_visual_analysis_fallback` and writes heuristic visual analysis if the CV worker is down or returns an error. + +When `fallback-to-heuristic=false`, local CV failures fail the source analysis and the source video is rejected by the scheduler. diff --git a/docs/production-cinematic-highlight-editing-plan.md b/docs/production-cinematic-highlight-editing-plan.md index 3cf185c..8c69544 100644 --- a/docs/production-cinematic-highlight-editing-plan.md +++ b/docs/production-cinematic-highlight-editing-plan.md @@ -307,6 +307,7 @@ The plan should be strict JSON so a cheaper model or deterministic renderer can - [x] Milestone 5 progress: Added deterministic FFmpeg audio analysis for mean/max volume, silence detection, missing-audio handling, persisted `analysis/audio-analysis.json`, and source-analysis embedding. Non-silent sections are labeled `unclassified_audio` until a speech/music/noise classifier or ASR provider is added. - [x] Milestone 6: Implement visual analysis for blur, exposure, motion, faces, object labels, composition quality, and representative thumbnails. - [x] Milestone 6 progress: Added persisted `analysis/visual-analysis.json` and source-analysis embedding with blur, exposure, motion, composition, representative thumbnail, face-presence, and object-label fields. Current implementation uses deterministic metadata, thumbnail, and scene-density heuristics; face/object recognition is explicitly marked heuristic until a CV model is connected. +- [x] Milestone 6 local CV extension: Added a configurable `local-cv` visual-analysis provider that posts source metadata, thumbnails, and shot segments to a local HTTP CV worker, persists the returned normalized visual analysis, and falls back to heuristic analysis when configured. See `docs/local-cv-visual-analysis-guide.md`. - [x] Milestone 7: Implement content category classification for family vlog, food vlog, car vlog, and generic fallback. - [x] Milestone 8: Implement category-specific highlight candidate scoring. - [x] Milestone 9: Implement strict AI director prompts and JSON schemas for category-aware highlight edit plans. diff --git a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java index 1f7f433..cbc472a 100644 --- a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java +++ b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java @@ -550,6 +550,8 @@ public class VideoClippingProperties { private final Assets assets = new Assets(); + private final VisualAnalysis visualAnalysis = new VisualAnalysis(); + private final LocalDirector localDirector = new LocalDirector(); private final HighlightScheduler highlightScheduler = new HighlightScheduler(); @@ -782,6 +784,10 @@ public class VideoClippingProperties { return assets; } + public VisualAnalysis getVisualAnalysis() { + return visualAnalysis; + } + public LocalDirector getLocalDirector() { return localDirector; } @@ -842,6 +848,49 @@ public class VideoClippingProperties { } } + public static class VisualAnalysis { + private String provider = "heuristic"; + + private String endpoint = "http://127.0.0.1:8091/v1/analyze-visuals"; + + @Min(1) + private long timeoutMs = 30000; + + private boolean fallbackToHeuristic = true; + + public String getProvider() { + return provider; + } + + public void setProvider(String provider) { + this.provider = provider; + } + + public String getEndpoint() { + return endpoint; + } + + public void setEndpoint(String endpoint) { + this.endpoint = endpoint; + } + + public long getTimeoutMs() { + return timeoutMs; + } + + public void setTimeoutMs(long timeoutMs) { + this.timeoutMs = timeoutMs; + } + + public boolean isFallbackToHeuristic() { + return fallbackToHeuristic; + } + + public void setFallbackToHeuristic(boolean fallbackToHeuristic) { + this.fallbackToHeuristic = fallbackToHeuristic; + } + } + public static class LocalDirector { private boolean enabled = true; diff --git a/src/main/java/org/example/videoclips/editing/HeuristicVisualAnalysisProvider.java b/src/main/java/org/example/videoclips/editing/HeuristicVisualAnalysisProvider.java new file mode 100644 index 0000000..f411833 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/HeuristicVisualAnalysisProvider.java @@ -0,0 +1,102 @@ +package org.example.videoclips.editing; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class HeuristicVisualAnalysisProvider implements VisualAnalysisProvider { + + @Override + public SourceVisualAnalysis analyze( + ClipAnalysis source, + List thumbnails, + List shotSegments + ) { + double blurScore = scoreOrDefault(source.sharpnessScore(), 0.5); + double exposureScore = scoreOrDefault(source.brightnessScore(), 0.5); + double motionScore = scoreOrDefault(source.motionScore(), motionFromShots(source.durationSeconds(), shotSegments)); + return new SourceVisualAnalysis( + source.clipId(), + blurScore, + exposureScore, + motionScore, + compositionScore(source, thumbnails), + "unknown_without_face_detector", + labels(source), + representativeThumbnails(thumbnails), + "metadata_thumbnail_scene_heuristic" + ); + } + + private double scoreOrDefault(double value, double fallback) { + if (value > 0) { + return round(clamp(value)); + } + return round(clamp(fallback)); + } + + private double motionFromShots(double durationSeconds, List shotSegments) { + if (durationSeconds <= 0 || shotSegments == null || shotSegments.isEmpty()) { + return 0.3; + } + double cutsPerMinute = Math.max(0, shotSegments.size() - 1) / durationSeconds * 60.0; + return clamp(0.25 + cutsPerMinute / 20.0); + } + + private double compositionScore(ClipAnalysis source, List thumbnails) { + double aspectRatio = source.height() == 0 ? 0 : source.width() / (double) source.height(); + double aspectScore = aspectRatio >= 1.70 && aspectRatio <= 1.90 ? 0.8 : 0.55; + double thumbnailScore = thumbnails == null || thumbnails.isEmpty() ? 0.45 : 0.7; + return round((aspectScore + thumbnailScore) / 2.0); + } + + private List labels(ClipAnalysis source) { + String haystack = (source.clipId() + " " + source.sourcePath()).toLowerCase(Locale.ROOT); + List labels = new ArrayList<>(); + addIfContains(labels, haystack, "porsche", "porsche", 0.82); + addIfContains(labels, haystack, "car", "car", 0.74); + addIfContains(labels, haystack, "drive", "car", 0.62); + addIfContains(labels, haystack, "food", "food", 0.72); + addIfContains(labels, haystack, "recipe", "food", 0.66); + addIfContains(labels, haystack, "family", "person", 0.62); + addIfContains(labels, haystack, "birthday", "person", 0.58); + if (labels.isEmpty()) { + labels.add(new VisualObjectLabel("unknown", 0.2, "metadata_heuristic")); + } + return labels.stream().distinct().toList(); + } + + private void addIfContains(List labels, String haystack, String needle, String label, + double confidence) { + if (haystack.contains(needle)) { + labels.add(new VisualObjectLabel(label, confidence, "metadata_heuristic")); + } + } + + private List representativeThumbnails(List thumbnails) { + if (thumbnails == null || thumbnails.isEmpty()) { + return List.of(); + } + if (thumbnails.size() <= 3) { + return List.copyOf(thumbnails); + } + return List.of( + thumbnails.get(0), + thumbnails.get(thumbnails.size() / 2), + thumbnails.get(thumbnails.size() - 1) + ); + } + + private double clamp(double value) { + return Math.max(0, Math.min(1, value)); + } + + private double round(double value) { + return Math.round(value * 1000.0) / 1000.0; + } +} diff --git a/src/main/java/org/example/videoclips/editing/LocalCvVisualAnalysisProvider.java b/src/main/java/org/example/videoclips/editing/LocalCvVisualAnalysisProvider.java new file mode 100644 index 0000000..5c21d43 --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/LocalCvVisualAnalysisProvider.java @@ -0,0 +1,117 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.List; + +@Component +@ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) +public class LocalCvVisualAnalysisProvider implements VisualAnalysisProvider { + + private final VideoClippingProperties.Editing.VisualAnalysis properties; + private final ObjectMapper objectMapper; + private final HttpExecutor httpExecutor; + + @Autowired + public LocalCvVisualAnalysisProvider(VideoClippingProperties properties, ObjectMapper objectMapper) { + this(properties, objectMapper, new JdkHttpExecutor()); + } + + LocalCvVisualAnalysisProvider( + VideoClippingProperties properties, + ObjectMapper objectMapper, + HttpExecutor httpExecutor + ) { + this.properties = properties.getEditing().getVisualAnalysis(); + this.objectMapper = objectMapper; + this.httpExecutor = httpExecutor; + } + + @Override + public SourceVisualAnalysis analyze(ClipAnalysis source, List thumbnails, List shotSegments) { + LocalCvRequest request = new LocalCvRequest(source, List.copyOf(thumbnails), List.copyOf(shotSegments)); + try { + String body = objectMapper.writeValueAsString(request); + HttpCall call = new HttpCall(properties.getEndpoint(), properties.getTimeoutMs(), body); + HttpResult result = httpExecutor.execute(call); + if (result.statusCode() < 200 || result.statusCode() >= 300) { + throw new IllegalStateException("Local CV service returned HTTP " + result.statusCode()); + } + SourceVisualAnalysis analysis = objectMapper.readValue(result.body(), SourceVisualAnalysis.class); + return normalized(source, thumbnails, analysis); + } catch (InterruptedException ex) { + Thread.currentThread().interrupt(); + throw new IllegalStateException("Interrupted while calling local CV visual analysis service", ex); + } catch (IOException ex) { + throw new IllegalStateException("Unable to call local CV visual analysis service", ex); + } + } + + private SourceVisualAnalysis normalized( + ClipAnalysis source, + List thumbnails, + SourceVisualAnalysis analysis + ) { + return new SourceVisualAnalysis( + valueOrDefault(analysis.clipId(), source.clipId()), + clamp(analysis.blurScore()), + clamp(analysis.exposureScore()), + clamp(analysis.motionScore()), + clamp(analysis.compositionScore()), + valueOrDefault(analysis.facePresence(), "unknown"), + analysis.objectLabels() == null ? List.of() : List.copyOf(analysis.objectLabels()), + analysis.representativeThumbnails() == null || analysis.representativeThumbnails().isEmpty() + ? List.copyOf(thumbnails) + : List.copyOf(analysis.representativeThumbnails()), + valueOrDefault(analysis.analysisMethod(), "local_cv") + ); + } + + private String valueOrDefault(String value, String fallback) { + return value == null || value.isBlank() ? fallback : value; + } + + private double clamp(double value) { + return Math.max(0, Math.min(1, Math.round(value * 1000.0) / 1000.0)); + } + + record LocalCvRequest(ClipAnalysis source, List thumbnails, List shotSegments) { + } + + record HttpCall(String endpoint, long timeoutMs, String body) { + } + + record HttpResult(int statusCode, String body) { + } + + @FunctionalInterface + interface HttpExecutor { + HttpResult execute(HttpCall call) throws IOException, InterruptedException; + } + + private static class JdkHttpExecutor implements HttpExecutor { + private final HttpClient httpClient = HttpClient.newHttpClient(); + + @Override + public HttpResult execute(HttpCall call) throws IOException, InterruptedException { + HttpRequest request = HttpRequest.newBuilder() + .uri(URI.create(call.endpoint())) + .timeout(Duration.ofMillis(call.timeoutMs())) + .header("Content-Type", "application/json") + .POST(HttpRequest.BodyPublishers.ofString(call.body())) + .build(); + HttpResponse response = httpClient.send(request, HttpResponse.BodyHandlers.ofString()); + return new HttpResult(response.statusCode(), response.body()); + } + } +} diff --git a/src/main/java/org/example/videoclips/editing/SourceVisualAnalyzer.java b/src/main/java/org/example/videoclips/editing/SourceVisualAnalyzer.java index f6fd55e..60e8dc8 100644 --- a/src/main/java/org/example/videoclips/editing/SourceVisualAnalyzer.java +++ b/src/main/java/org/example/videoclips/editing/SourceVisualAnalyzer.java @@ -1,101 +1,87 @@ package org.example.videoclips.editing; +import org.example.videoclips.config.VideoClippingProperties; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.stereotype.Component; -import java.util.ArrayList; import java.util.List; -import java.util.Locale; @Component @ConditionalOnProperty(name = "video-clipping.editing.enabled", havingValue = "true", matchIfMissing = true) public class SourceVisualAnalyzer { + private static final Logger log = LoggerFactory.getLogger(SourceVisualAnalyzer.class); + + private final VideoClippingProperties.Editing.VisualAnalysis properties; + private final VisualAnalysisProvider heuristicProvider; + private final VisualAnalysisProvider localCvProvider; + + public SourceVisualAnalyzer() { + this(new VideoClippingProperties(), new HeuristicVisualAnalysisProvider(), null); + } + + @Autowired + public SourceVisualAnalyzer( + VideoClippingProperties properties, + HeuristicVisualAnalysisProvider heuristicProvider, + LocalCvVisualAnalysisProvider localCvProvider + ) { + this(properties, heuristicProvider, (VisualAnalysisProvider) localCvProvider); + } + + SourceVisualAnalyzer( + VideoClippingProperties properties, + VisualAnalysisProvider heuristicProvider, + VisualAnalysisProvider localCvProvider + ) { + this.properties = properties.getEditing().getVisualAnalysis(); + this.heuristicProvider = heuristicProvider; + this.localCvProvider = localCvProvider; + } + public SourceVisualAnalysis analyze( ClipAnalysis source, List thumbnails, List shotSegments ) { - double blurScore = scoreOrDefault(source.sharpnessScore(), 0.5); - double exposureScore = scoreOrDefault(source.brightnessScore(), 0.5); - double motionScore = scoreOrDefault(source.motionScore(), motionFromShots(source.durationSeconds(), shotSegments)); - return new SourceVisualAnalysis( - source.clipId(), - blurScore, - exposureScore, - motionScore, - compositionScore(source, thumbnails), - "unknown_without_face_detector", - labels(source), - representativeThumbnails(thumbnails), - "metadata_thumbnail_scene_heuristic" - ); - } - - private double scoreOrDefault(double value, double fallback) { - if (value > 0) { - return round(clamp(value)); + if ("local-cv".equalsIgnoreCase(properties.getProvider())) { + return localCvAnalysis(source, thumbnails, shotSegments); } - return round(clamp(fallback)); + return heuristicProvider.analyze(source, thumbnails, shotSegments); } - private double motionFromShots(double durationSeconds, List shotSegments) { - if (durationSeconds <= 0 || shotSegments == null || shotSegments.isEmpty()) { - return 0.3; + private SourceVisualAnalysis localCvAnalysis( + ClipAnalysis source, + List thumbnails, + List shotSegments + ) { + try { + SourceVisualAnalysis analysis = localCvProvider.analyze(source, thumbnails, shotSegments); + log.info("event=local_cv_visual_analysis_completed clip_id={} endpoint={} object_labels={} method={}", + source.clipId(), properties.getEndpoint(), analysis.objectLabels().size(), + analysis.analysisMethod()); + return analysis; + } catch (RuntimeException ex) { + if (!properties.isFallbackToHeuristic()) { + throw ex; + } + log.warn("event=local_cv_visual_analysis_fallback clip_id={} endpoint={} error_type={}", + source.clipId(), properties.getEndpoint(), ex.getClass().getSimpleName()); + SourceVisualAnalysis fallback = heuristicProvider.analyze(source, thumbnails, shotSegments); + return new SourceVisualAnalysis( + fallback.clipId(), + fallback.blurScore(), + fallback.exposureScore(), + fallback.motionScore(), + fallback.compositionScore(), + fallback.facePresence(), + fallback.objectLabels(), + fallback.representativeThumbnails(), + "local_cv_failed_fallback_" + fallback.analysisMethod() + ); } - double cutsPerMinute = Math.max(0, shotSegments.size() - 1) / durationSeconds * 60.0; - return clamp(0.25 + cutsPerMinute / 20.0); - } - - private double compositionScore(ClipAnalysis source, List thumbnails) { - double aspectRatio = source.height() == 0 ? 0 : source.width() / (double) source.height(); - double aspectScore = aspectRatio >= 1.70 && aspectRatio <= 1.90 ? 0.8 : 0.55; - double thumbnailScore = thumbnails == null || thumbnails.isEmpty() ? 0.45 : 0.7; - return round((aspectScore + thumbnailScore) / 2.0); - } - - private List labels(ClipAnalysis source) { - String haystack = (source.clipId() + " " + source.sourcePath()).toLowerCase(Locale.ROOT); - List labels = new ArrayList<>(); - addIfContains(labels, haystack, "porsche", "porsche", 0.82); - addIfContains(labels, haystack, "car", "car", 0.74); - addIfContains(labels, haystack, "drive", "car", 0.62); - addIfContains(labels, haystack, "food", "food", 0.72); - addIfContains(labels, haystack, "recipe", "food", 0.66); - addIfContains(labels, haystack, "family", "person", 0.62); - addIfContains(labels, haystack, "birthday", "person", 0.58); - if (labels.isEmpty()) { - labels.add(new VisualObjectLabel("unknown", 0.2, "metadata_heuristic")); - } - return labels.stream().distinct().toList(); - } - - private void addIfContains(List labels, String haystack, String needle, String label, - double confidence) { - if (haystack.contains(needle)) { - labels.add(new VisualObjectLabel(label, confidence, "metadata_heuristic")); - } - } - - private List representativeThumbnails(List thumbnails) { - if (thumbnails == null || thumbnails.isEmpty()) { - return List.of(); - } - if (thumbnails.size() <= 3) { - return List.copyOf(thumbnails); - } - return List.of( - thumbnails.get(0), - thumbnails.get(thumbnails.size() / 2), - thumbnails.get(thumbnails.size() - 1) - ); - } - - private double clamp(double value) { - return Math.max(0, Math.min(1, value)); - } - - private double round(double value) { - return Math.round(value * 1000.0) / 1000.0; } } diff --git a/src/main/java/org/example/videoclips/editing/VisualAnalysisProvider.java b/src/main/java/org/example/videoclips/editing/VisualAnalysisProvider.java new file mode 100644 index 0000000..6e3700a --- /dev/null +++ b/src/main/java/org/example/videoclips/editing/VisualAnalysisProvider.java @@ -0,0 +1,8 @@ +package org.example.videoclips.editing; + +import java.util.List; + +public interface VisualAnalysisProvider { + + SourceVisualAnalysis analyze(ClipAnalysis source, List thumbnails, List shotSegments); +} diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml index 3401147..ac5b0a4 100644 --- a/src/main/resources/application.yml +++ b/src/main/resources/application.yml @@ -48,6 +48,11 @@ video-clipping: fonts-folder: ${VIDEO_EDITING_ASSETS_FONTS_FOLDER:./input/highlights/assets/fonts} luts-folder: ${VIDEO_EDITING_ASSETS_LUTS_FOLDER:./input/highlights/assets/luts} voiceover-folder: ${VIDEO_EDITING_ASSETS_VOICEOVER_FOLDER:./output/highlight-projects/_voiceover-cache} + visual-analysis: + provider: ${VIDEO_EDITING_VISUAL_ANALYSIS_PROVIDER:heuristic} + endpoint: ${VIDEO_EDITING_VISUAL_ANALYSIS_ENDPOINT:http://127.0.0.1:8091/v1/analyze-visuals} + timeout-ms: ${VIDEO_EDITING_VISUAL_ANALYSIS_TIMEOUT_MS:30000} + fallback-to-heuristic: ${VIDEO_EDITING_VISUAL_ANALYSIS_FALLBACK_TO_HEURISTIC:true} local-director: enabled: ${VIDEO_EDITING_LOCAL_DIRECTOR_ENABLED:true} source-directory: ${VIDEO_EDITING_LOCAL_DIRECTOR_SOURCE_DIRECTORY:./input/editing/source} diff --git a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java index 842de19..815bc1f 100644 --- a/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java +++ b/src/test/java/org/example/videoclips/config/VideoClippingPropertiesTest.java @@ -54,6 +54,11 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().getAssets().getLutsFolder()).isEqualTo("./input/highlights/assets/luts"); assertThat(properties.getEditing().getAssets().getVoiceoverFolder()) .isEqualTo("./output/highlight-projects/_voiceover-cache"); + assertThat(properties.getEditing().getVisualAnalysis().getProvider()).isEqualTo("heuristic"); + assertThat(properties.getEditing().getVisualAnalysis().getEndpoint()) + .isEqualTo("http://127.0.0.1:8091/v1/analyze-visuals"); + assertThat(properties.getEditing().getVisualAnalysis().getTimeoutMs()).isEqualTo(30000); + assertThat(properties.getEditing().getVisualAnalysis().isFallbackToHeuristic()).isTrue(); assertThat(properties.getEditing().getLocalDirector().isEnabled()).isTrue(); assertThat(properties.getEditing().getLocalDirector().getSourceDirectory()) .isEqualTo("./input/editing/source"); @@ -92,6 +97,10 @@ class VideoClippingPropertiesTest { "video-clipping.editing.assets.fonts-folder=/tmp/fonts", "video-clipping.editing.assets.luts-folder=/tmp/luts", "video-clipping.editing.assets.voiceover-folder=/tmp/voiceover", + "video-clipping.editing.visual-analysis.provider=local-cv", + "video-clipping.editing.visual-analysis.endpoint=http://localhost:9000/analyze", + "video-clipping.editing.visual-analysis.timeout-ms=12000", + "video-clipping.editing.visual-analysis.fallback-to-heuristic=false", "video-clipping.editing.local-director.enabled=false", "video-clipping.editing.local-director.source-directory=/tmp/source", "video-clipping.editing.local-director.poll-interval-ms=9000", @@ -126,6 +135,11 @@ class VideoClippingPropertiesTest { assertThat(properties.getEditing().getAssets().getFontsFolder()).isEqualTo("/tmp/fonts"); assertThat(properties.getEditing().getAssets().getLutsFolder()).isEqualTo("/tmp/luts"); assertThat(properties.getEditing().getAssets().getVoiceoverFolder()).isEqualTo("/tmp/voiceover"); + assertThat(properties.getEditing().getVisualAnalysis().getProvider()).isEqualTo("local-cv"); + assertThat(properties.getEditing().getVisualAnalysis().getEndpoint()) + .isEqualTo("http://localhost:9000/analyze"); + assertThat(properties.getEditing().getVisualAnalysis().getTimeoutMs()).isEqualTo(12000); + assertThat(properties.getEditing().getVisualAnalysis().isFallbackToHeuristic()).isFalse(); assertThat(properties.getEditing().getLocalDirector().isEnabled()).isFalse(); assertThat(properties.getEditing().getLocalDirector().getSourceDirectory()).isEqualTo("/tmp/source"); assertThat(properties.getEditing().getLocalDirector().getPollIntervalMs()).isEqualTo(9000); diff --git a/src/test/java/org/example/videoclips/editing/LocalCvVisualAnalysisProviderTest.java b/src/test/java/org/example/videoclips/editing/LocalCvVisualAnalysisProviderTest.java new file mode 100644 index 0000000..8bfecc2 --- /dev/null +++ b/src/test/java/org/example/videoclips/editing/LocalCvVisualAnalysisProviderTest.java @@ -0,0 +1,94 @@ +package org.example.videoclips.editing; + +import com.fasterxml.jackson.databind.ObjectMapper; +import org.example.videoclips.config.VideoClippingProperties; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.atomic.AtomicReference; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class LocalCvVisualAnalysisProviderTest { + + @Test + void postsVisualAnalysisRequestToConfiguredLocalCvEndpoint() { + AtomicReference call = new AtomicReference<>(); + VideoClippingProperties properties = new VideoClippingProperties(); + properties.getEditing().getVisualAnalysis().setEndpoint("http://localhost:9000/analyze"); + properties.getEditing().getVisualAnalysis().setTimeoutMs(12000); + LocalCvVisualAnalysisProvider provider = new LocalCvVisualAnalysisProvider( + properties, + new ObjectMapper().findAndRegisterModules(), + request -> { + call.set(request); + return new LocalCvVisualAnalysisProvider.HttpResult(200, """ + { + "clipId": "source", + "blurScore": 0.81, + "exposureScore": 0.67, + "motionScore": 0.73, + "compositionScore": 0.79, + "facePresence": "faces_detected", + "objectLabels": [ + {"label": "car", "confidence": 0.91, "source": "yolo"} + ], + "representativeThumbnails": ["thumb-2.jpg"], + "analysisMethod": "local_cv_yolo_clip" + } + """); + } + ); + + SourceVisualAnalysis analysis = provider.analyze(clip(), List.of("thumb-1.jpg", "thumb-2.jpg"), + List.of(new ShotSegment("shot_0001", 0, 8, 8, 0, 4))); + + assertThat(call.get().endpoint()).isEqualTo("http://localhost:9000/analyze"); + assertThat(call.get().timeoutMs()).isEqualTo(12000); + assertThat(call.get().body()).contains("\"clipId\":\"source\"", "\"thumbnails\":[\"thumb-1.jpg\""); + assertThat(analysis.objectLabels()).containsExactly(new VisualObjectLabel("car", 0.91, "yolo")); + assertThat(analysis.facePresence()).isEqualTo("faces_detected"); + assertThat(analysis.analysisMethod()).isEqualTo("local_cv_yolo_clip"); + } + + @Test + void rejectsNonSuccessfulLocalCvResponses() { + LocalCvVisualAnalysisProvider provider = new LocalCvVisualAnalysisProvider( + new VideoClippingProperties(), + new ObjectMapper().findAndRegisterModules(), + request -> new LocalCvVisualAnalysisProvider.HttpResult(500, "{}") + ); + + assertThrows(IllegalStateException.class, () -> provider.analyze(clip(), List.of(), List.of())); + } + + @Test + void reportsLocalCvIoAndInterruptionFailures() { + assertThrows(IllegalStateException.class, () -> new LocalCvVisualAnalysisProvider( + new VideoClippingProperties(), + new ObjectMapper().findAndRegisterModules(), + request -> { + throw new IOException("connection refused"); + }).analyze(clip(), List.of(), List.of())); + + try { + assertThrows(IllegalStateException.class, () -> new LocalCvVisualAnalysisProvider( + new VideoClippingProperties(), + new ObjectMapper().findAndRegisterModules(), + request -> { + throw new InterruptedException("stopped"); + }).analyze(clip(), List.of(), List.of())); + assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + } + + private ClipAnalysis clip() { + return new ClipAnalysis("source", "source.mp4", 8.0, "h264", "aac", + 1920, 1080, 30.0, List.of(), null, null, 0, 0, 0); + } +} diff --git a/src/test/java/org/example/videoclips/editing/SourceVisualAnalyzerTest.java b/src/test/java/org/example/videoclips/editing/SourceVisualAnalyzerTest.java index 44db00e..12be4fd 100644 --- a/src/test/java/org/example/videoclips/editing/SourceVisualAnalyzerTest.java +++ b/src/test/java/org/example/videoclips/editing/SourceVisualAnalyzerTest.java @@ -53,6 +53,76 @@ class SourceVisualAnalyzerTest { assertThat(analysis.representativeThumbnails()).isEmpty(); } + @Test + void usesLocalCvProviderWhenConfigured() { + org.example.videoclips.config.VideoClippingProperties properties = + new org.example.videoclips.config.VideoClippingProperties(); + properties.getEditing().getVisualAnalysis().setProvider("local-cv"); + SourceVisualAnalyzer localAnalyzer = new SourceVisualAnalyzer( + properties, + (source, thumbnails, shotSegments) -> { + throw new AssertionError("heuristic fallback should not run"); + }, + (source, thumbnails, shotSegments) -> new SourceVisualAnalysis(source.clipId(), 0.9, 0.8, + 0.7, 0.6, "faces_detected", + List.of(new VisualObjectLabel("person", 0.91, "mediapipe")), thumbnails, "local_cv") + ); + + SourceVisualAnalysis analysis = localAnalyzer.analyze( + clip("family", "family.mp4", 0, 0, 0), + List.of("thumb.jpg"), + List.of() + ); + + assertThat(analysis.analysisMethod()).isEqualTo("local_cv"); + assertThat(analysis.objectLabels()).containsExactly(new VisualObjectLabel("person", 0.91, "mediapipe")); + } + + @Test + void fallsBackToHeuristicWhenLocalCvFailsAndFallbackIsEnabled() { + org.example.videoclips.config.VideoClippingProperties properties = + new org.example.videoclips.config.VideoClippingProperties(); + properties.getEditing().getVisualAnalysis().setProvider("local-cv"); + SourceVisualAnalyzer localAnalyzer = new SourceVisualAnalyzer( + properties, + new HeuristicVisualAnalysisProvider(), + (source, thumbnails, shotSegments) -> { + throw new IllegalStateException("local cv unavailable"); + } + ); + + SourceVisualAnalysis analysis = localAnalyzer.analyze( + clip("porsche", "porsche.mp4", 0, 0, 0), + List.of("thumb.jpg"), + List.of() + ); + + assertThat(analysis.analysisMethod()).isEqualTo("local_cv_failed_fallback_metadata_thumbnail_scene_heuristic"); + assertThat(analysis.objectLabels()).containsExactly(new VisualObjectLabel("porsche", 0.82, + "metadata_heuristic")); + } + + @Test + void failsWhenLocalCvFailsAndFallbackIsDisabled() { + org.example.videoclips.config.VideoClippingProperties properties = + new org.example.videoclips.config.VideoClippingProperties(); + properties.getEditing().getVisualAnalysis().setProvider("local-cv"); + properties.getEditing().getVisualAnalysis().setFallbackToHeuristic(false); + SourceVisualAnalyzer localAnalyzer = new SourceVisualAnalyzer( + properties, + new HeuristicVisualAnalysisProvider(), + (source, thumbnails, shotSegments) -> { + throw new IllegalStateException("local cv unavailable"); + } + ); + + org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, () -> localAnalyzer.analyze( + clip("porsche", "porsche.mp4", 0, 0, 0), + List.of("thumb.jpg"), + List.of() + )); + } + private ClipAnalysis clip(String clipId, String sourcePath, double sharpness, double brightness, double motion) { return new ClipAnalysis(clipId, sourcePath, 24.0, "h264", "aac", 1920, 1080, 30.0, List.of(), null, null, motion, brightness, sharpness); diff --git a/tools/local_cv_worker.py b/tools/local_cv_worker.py new file mode 100644 index 0000000..18fa39b --- /dev/null +++ b/tools/local_cv_worker.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Optional local CV worker for the Spring visual-analysis provider. + +Run: + pip install fastapi uvicorn opencv-python + # Optional object detection: + pip install ultralytics + LOCAL_CV_YOLO_MODEL=yolov8n.pt uvicorn tools.local_cv_worker:app --host 127.0.0.1 --port 8091 +""" + +from __future__ import annotations + +import os +from pathlib import Path +from typing import Any + +try: + import cv2 # type: ignore +except ImportError: # pragma: no cover - optional runtime dependency + cv2 = None + +try: + from fastapi import FastAPI +except ImportError as exc: # pragma: no cover - fail clearly at worker startup + raise SystemExit("Install FastAPI first: pip install fastapi uvicorn") from exc + +try: + from ultralytics import YOLO # type: ignore +except ImportError: # pragma: no cover - optional runtime dependency + YOLO = None + + +app = FastAPI(title="Local CV Visual Analysis Worker") +_yolo_model: Any | None = None + + +@app.post("/v1/analyze-visuals") +def analyze_visuals(payload: dict[str, Any]) -> dict[str, Any]: + source = payload.get("source", {}) + thumbnails = [str(item) for item in payload.get("thumbnails", [])] + readable_thumbnails = [path for path in thumbnails if Path(path).is_file()] + quality = quality_scores(readable_thumbnails) + labels = object_labels(readable_thumbnails) + + return { + "clipId": source.get("clipId", "source"), + "blurScore": quality["blurScore"], + "exposureScore": quality["exposureScore"], + "motionScore": motion_score(payload.get("shotSegments", []), source.get("durationSeconds", 0)), + "compositionScore": composition_score(source, readable_thumbnails), + "facePresence": face_presence(readable_thumbnails), + "objectLabels": labels or [{"label": "unknown", "confidence": 0.2, "source": "local_cv_worker"}], + "representativeThumbnails": representative_thumbnails(readable_thumbnails or thumbnails), + "analysisMethod": "local_cv_worker_opencv_yolo" if labels else "local_cv_worker_opencv", + } + + +def quality_scores(thumbnails: list[str]) -> dict[str, float]: + if cv2 is None or not thumbnails: + return {"blurScore": 0.5, "exposureScore": 0.5} + + blur_scores: list[float] = [] + exposure_scores: list[float] = [] + for thumbnail in thumbnails: + image = cv2.imread(thumbnail) + if image is None: + continue + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + laplacian_variance = float(cv2.Laplacian(gray, cv2.CV_64F).var()) + blur_scores.append(clamp(laplacian_variance / 500.0)) + mean_brightness = float(gray.mean()) / 255.0 + exposure_scores.append(clamp(1.0 - abs(mean_brightness - 0.5) * 2.0)) + + return { + "blurScore": average_or_default(blur_scores, 0.5), + "exposureScore": average_or_default(exposure_scores, 0.5), + } + + +def object_labels(thumbnails: list[str]) -> list[dict[str, Any]]: + model = yolo_model() + if model is None or not thumbnails: + return [] + + labels: dict[str, float] = {} + for result in model(thumbnails, verbose=False): + names = getattr(result, "names", {}) + boxes = getattr(result, "boxes", None) + if boxes is None: + continue + for box in boxes: + label = str(names.get(int(box.cls[0]), "object")) + confidence = float(box.conf[0]) + labels[label] = max(labels.get(label, 0.0), confidence) + + return [ + {"label": label, "confidence": round(clamp(confidence), 3), "source": "yolo"} + for label, confidence in sorted(labels.items(), key=lambda item: item[1], reverse=True)[:10] + ] + + +def yolo_model() -> Any | None: + global _yolo_model + model_path = os.getenv("LOCAL_CV_YOLO_MODEL") + if YOLO is None or not model_path: + return None + if _yolo_model is None: + _yolo_model = YOLO(model_path) + return _yolo_model + + +def face_presence(thumbnails: list[str]) -> str: + if cv2 is None or not thumbnails: + return "unknown_without_face_detector" + cascade_path = getattr(cv2.data, "haarcascades", "") + "haarcascade_frontalface_default.xml" + if not Path(cascade_path).is_file(): + return "unknown_without_face_detector" + cascade = cv2.CascadeClassifier(cascade_path) + for thumbnail in thumbnails: + image = cv2.imread(thumbnail) + if image is None: + continue + gray = cv2.cvtColor(image, cv2.COLOR_BGR2GRAY) + faces = cascade.detectMultiScale(gray, scaleFactor=1.1, minNeighbors=4) + if len(faces) > 0: + return "faces_detected" + return "no_faces_detected" + + +def motion_score(shot_segments: list[dict[str, Any]], duration_seconds: float) -> float: + try: + duration = float(duration_seconds) + except (TypeError, ValueError): + duration = 0.0 + if duration <= 0: + return 0.3 + cuts_per_minute = max(0, len(shot_segments) - 1) / duration * 60.0 + return round(clamp(0.25 + cuts_per_minute / 20.0), 3) + + +def composition_score(source: dict[str, Any], thumbnails: list[str]) -> float: + width = float(source.get("width", 0) or 0) + height = float(source.get("height", 0) or 0) + aspect_ratio = width / height if height else 0 + aspect_score = 0.8 if 1.70 <= aspect_ratio <= 1.90 else 0.55 + thumbnail_score = 0.7 if thumbnails else 0.45 + return round((aspect_score + thumbnail_score) / 2.0, 3) + + +def representative_thumbnails(thumbnails: list[str]) -> list[str]: + if len(thumbnails) <= 3: + return thumbnails + return [thumbnails[0], thumbnails[len(thumbnails) // 2], thumbnails[-1]] + + +def average_or_default(values: list[float], default: float) -> float: + if not values: + return default + return round(sum(values) / len(values), 3) + + +def clamp(value: float) -> float: + return max(0.0, min(1.0, value))