diff --git a/docs/LOCAL-MODELS.md b/docs/LOCAL-MODELS.md index e5d632f..42f9f13 100644 --- a/docs/LOCAL-MODELS.md +++ b/docs/LOCAL-MODELS.md @@ -69,19 +69,25 @@ cmake --build build --config Release -j --target llama-mtmd-cli # -> build/bin # Record license/SHA-256 next to each file, per the asset-provenance policy. ``` -Enable it (no code change — config + env only): +Enable it (no code change — config + env). **localpoc already defaults `vision-caption-script` to the +llama.cpp worker**; if the binary env or weights are missing it **falls back to moondream with a loud WARN** +(`event=vision_backend_not_ready`), so the default never silently degrades. The model/mmproj default to the +repo paths below, so only the (machine-specific) **binary env** is mandatory: -```yaml -# application-localpoc.yml (video-clipping.editing) -vision-caption-script: ./tools/vision_caption_llamacpp.py -``` ```bash -export LLAMACPP_MTMD_BIN=/abs/llama.cpp/build/bin/llama-mtmd-cli -export LLAMACPP_VLM_MODEL=/abs/models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf -export LLAMACPP_VLM_MMPROJ=/abs/models/qwen2.5-vl-3b/mmproj-F16.gguf -# LLAMACPP_VLM_NTOKENS=64 (optional) +# FAST (recommended): resident server — model loads ONCE, ~3x faster per clip. +export LLAMACPP_SERVER_BIN=/abs/llama.cpp/build/bin/llama-server +# or SIMPLE: per-frame CLI (reloads the model each frame) +# export LLAMACPP_MTMD_BIN=/abs/llama.cpp/build/bin/llama-mtmd-cli + +# Optional — default to ./models/qwen2.5-vl-3b/ if unset: +# export LLAMACPP_VLM_MODEL=/abs/.../Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf +# export LLAMACPP_VLM_MMPROJ=/abs/.../mmproj-F16.gguf +# export LLAMACPP_VLM_NTOKENS=64 LLAMACPP_SERVER_PORT=8123 ``` +To force moondream instead: set `vision-caption-script: ./tools/vision_caption.py`. + **Runtime GOTCHA — force CPU on an Intel Mac.** The build enables Metal by default, but this machine's integrated GPU times out on the vision encoder (`ggml_metal_synchronize: command buffer failed … GPU Timeout`). `tools/vision_caption_llamacpp.py` therefore always passes `-ngl 0 --no-mmproj-offload` (pure CPU/AVX). Expect diff --git a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java index 5419e80..d03428c 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java +++ b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java @@ -42,13 +42,50 @@ public class HighlightVisionDirector { private final String captionScript; private final ObjectMapper objectMapper; + private static final String MOONDREAM_SCRIPT = "./tools/vision_caption.py"; + private static final String LLAMACPP_DEFAULT_MODEL = "./models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf"; + private static final String LLAMACPP_DEFAULT_MMPROJ = "./models/qwen2.5-vl-3b/mmproj-F16.gguf"; + public HighlightVisionDirector(VideoClippingProperties properties, ObjectMapper objectMapper) { this.worker = properties.getEditing().getLocalAssetWorker(); this.ffmpegBinary = properties.getEditing().getFfmpegBinary(); - this.captionScript = properties.getEditing().getVisionCaptionScript(); + this.captionScript = resolveCaptionScript(properties.getEditing().getVisionCaptionScript()); this.objectMapper = objectMapper; } + /** + * Resolve the effective captioner. When the llama.cpp backend is configured but not fully provisioned (no + * binary env var, or missing weights), fall back to moondream and WARN loudly — so a default of the stronger + * model never silently degrades into a wrong pick on an unprovisioned machine. + */ + static String resolveCaptionScript(String configured) { + if (configured == null || !configured.contains("vision_caption_llamacpp")) { + return configured; // moondream or a custom backend — use as-is + } + boolean scriptOk = isFile(configured); + boolean binOk = isFile(System.getenv("LLAMACPP_SERVER_BIN")) || isFile(System.getenv("LLAMACPP_MTMD_BIN")); + boolean modelOk = isFile(envOr("LLAMACPP_VLM_MODEL", LLAMACPP_DEFAULT_MODEL)); + boolean mmprojOk = isFile(envOr("LLAMACPP_VLM_MMPROJ", LLAMACPP_DEFAULT_MMPROJ)); + if (scriptOk && binOk && modelOk && mmprojOk) { + log.info("event=vision_backend backend=llamacpp script={}", configured); + return configured; + } + log.warn("event=vision_backend_not_ready backend=llamacpp fallback=moondream " + + "script_ok={} binary_env_ok={} model_ok={} mmproj_ok={} " + + "(set LLAMACPP_SERVER_BIN or LLAMACPP_MTMD_BIN and provision the weights to use Qwen)", + scriptOk, binOk, modelOk, mmprojOk); + return MOONDREAM_SCRIPT; + } + + private static boolean isFile(String path) { + return path != null && !path.isBlank() && Files.isRegularFile(Path.of(path.trim())); + } + + private static String envOr(String name, String fallback) { + String v = System.getenv(name); + return v == null || v.isBlank() ? fallback : v; + } + /** * Captions {@code samples} frames evenly across the source with the local VLM (one worker call, model * loaded once). Returns timed captions describing the key action/achievement at each moment, or an empty diff --git a/src/main/resources/application-localpoc.yml b/src/main/resources/application-localpoc.yml index 79f64ed..458bc2e 100644 --- a/src/main/resources/application-localpoc.yml +++ b/src/main/resources/application-localpoc.yml @@ -28,6 +28,11 @@ video-clipping: # R14 subject-tracking reframe: follow the detected subject (YOLO/CV venv) instead of a static centre crop. subject-reframe-enabled: true + # Tier-2 vision JUDGE: prefer the stronger Qwen2.5-VL (llama.cpp/GGUF) backend. It falls back to moondream + # with a loud WARN if the binary env (LLAMACPP_SERVER_BIN / LLAMACPP_MTMD_BIN) or weights aren't provisioned + # — so this default never silently degrades. See docs/LOCAL-MODELS.md. + vision-caption-script: ./tools/vision_caption_llamacpp.py + assets: # Empty/absent asset folders -> the pipeline generates assets with local models instead of copying. music-folder: ./input/localpoc/assets/music diff --git a/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java b/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java index 64e1b59..0dabdf3 100644 --- a/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java +++ b/src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java @@ -77,6 +77,18 @@ class HighlightVisionDirectorTest { assertThat(curve[5]).isEqualTo(1.0); // window ~5.5s -> nearest 5.0s "arms raised celebration" } + @Test + void llamacppBackendFallsBackToMoondreamWhenNotProvisioned() { + // A non-llama.cpp script is used as-is. + assertThat(HighlightVisionDirector.resolveCaptionScript("./tools/vision_caption.py")) + .isEqualTo("./tools/vision_caption.py"); + assertThat(HighlightVisionDirector.resolveCaptionScript(null)).isNull(); + // The llama.cpp backend requires a binary env (LLAMACPP_SERVER_BIN / LLAMACPP_MTMD_BIN), which is not + // set in the test JVM -> it must fall back to moondream rather than silently use a broken backend. + assertThat(HighlightVisionDirector.resolveCaptionScript("./tools/vision_caption_llamacpp.py")) + .isEqualTo("./tools/vision_caption.py"); + } + @Test void detectsAnticipatoryDescriptions() { assertThat(HighlightVisionDirector.isAnticipatory("The person is about to kick the ball")).isTrue(); diff --git a/tools/vision_caption_llamacpp.py b/tools/vision_caption_llamacpp.py index 7fb7c83..a57834d 100644 --- a/tools/vision_caption_llamacpp.py +++ b/tools/vision_caption_llamacpp.py @@ -120,8 +120,9 @@ def main() -> int: parser.add_argument("--output", required=True) args = parser.parse_args() - model = os.environ.get("LLAMACPP_VLM_MODEL", "") - mmproj = os.environ.get("LLAMACPP_VLM_MMPROJ", "") + # Default to the repo's provisioned paths so only the (machine-specific) binary needs an env var. + model = os.environ.get("LLAMACPP_VLM_MODEL") or "./models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf" + mmproj = os.environ.get("LLAMACPP_VLM_MMPROJ") or "./models/qwen2.5-vl-3b/mmproj-F16.gguf" ntokens = int(os.environ.get("LLAMACPP_VLM_NTOKENS", "64")) server_bin = os.environ.get("LLAMACPP_SERVER_BIN", "") cli_bin = os.environ.get("LLAMACPP_MTMD_BIN", "")