feat(vision): default localpoc to Qwen2.5-VL with loud moondream fallback

Make the stronger Tier-2 judge the localpoc default (vision-caption-script ->
vision_caption_llamacpp.py). Guard it so it never silently degrades: resolveCaptionScript
checks the llama.cpp backend is provisioned (binary env + weights) and, if not, falls
back to moondream with a WARN (event=vision_backend_not_ready). Worker defaults the
model/mmproj to the repo's ./models/qwen2.5-vl-3b paths, so only the machine-specific
binary env is mandatory. mvn verify: 294 tests green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

docs/LOCAL-MODELS.md

@@ -69,19 +69,25 @@ cmake --build build --config Release -j --target llama-mtmd-cli   # -> build/bin
 #    Record license/SHA-256 next to each file, per the asset-provenance policy.
 ```
 
-Enable it (no code change — config + env only):
+Enable it (no code change — config + env). **localpoc already defaults `vision-caption-script` to the
+llama.cpp worker**; if the binary env or weights are missing it **falls back to moondream with a loud WARN**
+(`event=vision_backend_not_ready`), so the default never silently degrades. The model/mmproj default to the
+repo paths below, so only the (machine-specific) **binary env** is mandatory:
 
-```yaml
-# application-localpoc.yml  (video-clipping.editing)
-vision-caption-script: ./tools/vision_caption_llamacpp.py
-```
 ```bash
-export LLAMACPP_MTMD_BIN=/abs/llama.cpp/build/bin/llama-mtmd-cli
+# FAST (recommended): resident server — model loads ONCE, ~3x faster per clip.
-export LLAMACPP_VLM_MODEL=/abs/models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf
+export LLAMACPP_SERVER_BIN=/abs/llama.cpp/build/bin/llama-server
-export LLAMACPP_VLM_MMPROJ=/abs/models/qwen2.5-vl-3b/mmproj-F16.gguf
+# or SIMPLE: per-frame CLI (reloads the model each frame)
-# LLAMACPP_VLM_NTOKENS=64 (optional)
+# export LLAMACPP_MTMD_BIN=/abs/llama.cpp/build/bin/llama-mtmd-cli
+
+# Optional — default to ./models/qwen2.5-vl-3b/ if unset:
+# export LLAMACPP_VLM_MODEL=/abs/.../Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf
+# export LLAMACPP_VLM_MMPROJ=/abs/.../mmproj-F16.gguf
+# export LLAMACPP_VLM_NTOKENS=64   LLAMACPP_SERVER_PORT=8123
 ```
 
+To force moondream instead: set `vision-caption-script: ./tools/vision_caption.py`.
+
 **Runtime GOTCHA — force CPU on an Intel Mac.** The build enables Metal by default, but this machine's
 integrated GPU times out on the vision encoder (`ggml_metal_synchronize: command buffer failed … GPU Timeout`).
 `tools/vision_caption_llamacpp.py` therefore always passes `-ngl 0 --no-mmproj-offload` (pure CPU/AVX). Expect

src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java

@@ -42,13 +42,50 @@ public class HighlightVisionDirector {
     private final String captionScript;
     private final ObjectMapper objectMapper;
 
+    private static final String MOONDREAM_SCRIPT = "./tools/vision_caption.py";
+    private static final String LLAMACPP_DEFAULT_MODEL = "./models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf";
+    private static final String LLAMACPP_DEFAULT_MMPROJ = "./models/qwen2.5-vl-3b/mmproj-F16.gguf";
+
     public HighlightVisionDirector(VideoClippingProperties properties, ObjectMapper objectMapper) {
         this.worker = properties.getEditing().getLocalAssetWorker();
         this.ffmpegBinary = properties.getEditing().getFfmpegBinary();
-        this.captionScript = properties.getEditing().getVisionCaptionScript();
+        this.captionScript = resolveCaptionScript(properties.getEditing().getVisionCaptionScript());
         this.objectMapper = objectMapper;
     }
 
+    /**
+     * Resolve the effective captioner. When the llama.cpp backend is configured but not fully provisioned (no
+     * binary env var, or missing weights), fall back to moondream and WARN loudly — so a default of the stronger
+     * model never silently degrades into a wrong pick on an unprovisioned machine.
+     */
+    static String resolveCaptionScript(String configured) {
+        if (configured == null || !configured.contains("vision_caption_llamacpp")) {
+            return configured;   // moondream or a custom backend — use as-is
+        }
+        boolean scriptOk = isFile(configured);
+        boolean binOk = isFile(System.getenv("LLAMACPP_SERVER_BIN")) || isFile(System.getenv("LLAMACPP_MTMD_BIN"));
+        boolean modelOk = isFile(envOr("LLAMACPP_VLM_MODEL", LLAMACPP_DEFAULT_MODEL));
+        boolean mmprojOk = isFile(envOr("LLAMACPP_VLM_MMPROJ", LLAMACPP_DEFAULT_MMPROJ));
+        if (scriptOk && binOk && modelOk && mmprojOk) {
+            log.info("event=vision_backend backend=llamacpp script={}", configured);
+            return configured;
+        }
+        log.warn("event=vision_backend_not_ready backend=llamacpp fallback=moondream "
+                        + "script_ok={} binary_env_ok={} model_ok={} mmproj_ok={} "
+                        + "(set LLAMACPP_SERVER_BIN or LLAMACPP_MTMD_BIN and provision the weights to use Qwen)",
+                scriptOk, binOk, modelOk, mmprojOk);
+        return MOONDREAM_SCRIPT;
+    }
+
+    private static boolean isFile(String path) {
+        return path != null && !path.isBlank() && Files.isRegularFile(Path.of(path.trim()));
+    }
+
+    private static String envOr(String name, String fallback) {
+        String v = System.getenv(name);
+        return v == null || v.isBlank() ? fallback : v;
+    }
+
     /**
      * Captions {@code samples} frames evenly across the source with the local VLM (one worker call, model
      * loaded once). Returns timed captions describing the key action/achievement at each moment, or an empty

src/main/resources/application-localpoc.yml

@@ -28,6 +28,11 @@ video-clipping:
     # R14 subject-tracking reframe: follow the detected subject (YOLO/CV venv) instead of a static centre crop.
     subject-reframe-enabled: true
 
+    # Tier-2 vision JUDGE: prefer the stronger Qwen2.5-VL (llama.cpp/GGUF) backend. It falls back to moondream
+    # with a loud WARN if the binary env (LLAMACPP_SERVER_BIN / LLAMACPP_MTMD_BIN) or weights aren't provisioned
+    # — so this default never silently degrades. See docs/LOCAL-MODELS.md.
+    vision-caption-script: ./tools/vision_caption_llamacpp.py
+
     assets:
       # Empty/absent asset folders -> the pipeline generates assets with local models instead of copying.
       music-folder: ./input/localpoc/assets/music

src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java

@@ -77,6 +77,18 @@ class HighlightVisionDirectorTest {
         assertThat(curve[5]).isEqualTo(1.0);   // window ~5.5s -> nearest 5.0s "arms raised celebration"
     }
 
+    @Test
+    void llamacppBackendFallsBackToMoondreamWhenNotProvisioned() {
+        // A non-llama.cpp script is used as-is.
+        assertThat(HighlightVisionDirector.resolveCaptionScript("./tools/vision_caption.py"))
+                .isEqualTo("./tools/vision_caption.py");
+        assertThat(HighlightVisionDirector.resolveCaptionScript(null)).isNull();
+        // The llama.cpp backend requires a binary env (LLAMACPP_SERVER_BIN / LLAMACPP_MTMD_BIN), which is not
+        // set in the test JVM -> it must fall back to moondream rather than silently use a broken backend.
+        assertThat(HighlightVisionDirector.resolveCaptionScript("./tools/vision_caption_llamacpp.py"))
+                .isEqualTo("./tools/vision_caption.py");
+    }
+
     @Test
     void detectsAnticipatoryDescriptions() {
         assertThat(HighlightVisionDirector.isAnticipatory("The person is about to kick the ball")).isTrue();

tools/vision_caption_llamacpp.py

@@ -120,8 +120,9 @@ def main() -> int:
     parser.add_argument("--output", required=True)
     args = parser.parse_args()
 
-    model = os.environ.get("LLAMACPP_VLM_MODEL", "")
+    # Default to the repo's provisioned paths so only the (machine-specific) binary needs an env var.
-    mmproj = os.environ.get("LLAMACPP_VLM_MMPROJ", "")
+    model = os.environ.get("LLAMACPP_VLM_MODEL") or "./models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf"
+    mmproj = os.environ.get("LLAMACPP_VLM_MMPROJ") or "./models/qwen2.5-vl-3b/mmproj-F16.gguf"
     ntokens = int(os.environ.get("LLAMACPP_VLM_NTOKENS", "64"))
     server_bin = os.environ.get("LLAMACPP_SERVER_BIN", "")
     cli_bin = os.environ.get("LLAMACPP_MTMD_BIN", "")