diff --git a/docs/LOCAL-MODELS.md b/docs/LOCAL-MODELS.md index fe71965..a28adc8 100644 --- a/docs/LOCAL-MODELS.md +++ b/docs/LOCAL-MODELS.md @@ -37,3 +37,46 @@ can be reproduced (reference environment: **x86_64 macOS, no GPU**; a Linux/GPU - **Licensing blocker for commercial use:** MusicGen (CC-BY-NC), AudioLDM2 (CC-BY-NC-SA) and YOLOv8 (AGPL) are non-commercial/copyleft. Swap in commercially-licensed models/assets before any commercial release. moondream2 (Apache-2.0) and Piper are fine. + +## Optional stronger Tier-2 VLM: Qwen2.5-VL via llama.cpp (recommended upgrade) + +moondream2 is small and, on hard footage (distant, portrait), cannot reliably tell a highlight from an +aftermath — which caps the director's vision **judge** (see `docs/cinematic-quality-rules.md` R15). A stronger +local VLM is a **drop-in**: the director calls a captioner script with a fixed manifest→JSON contract, so only +the configured script path changes. **`tools/vision_caption_llamacpp.py`** implements that contract against +**llama.cpp** (GGUF) — which runs on this **x86 CPU via AVX SIMD** and **bypasses the torch==2.2.2 / +transformers 4.x / no-xformers trap entirely** (no PyTorch involved). + +**Recommended model:** `Qwen2.5-VL-3B-Instruct` (Apache-2.0 — commercial-friendly; verify the model card), +`Q4_K_M` GGUF + its `mmproj` vision projector. `Qwen3-VL-2B/4B` (official GGUF) or `Gemma 3 4B` are alternatives. + +Provision **offline** (on a networked machine, then copy the files over — nothing downloads at service runtime): + +```bash +# 1) Build llama.cpp with the multimodal CLI (one-time, needs a compiler; NOT in a certified/offline env) +git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp && cmake -B build && cmake --build build -j +# -> produces build/bin/llama-mtmd-cli + +# 2) Fetch the GGUF weights + mmproj (e.g. from a bartowski/Mungert/official Qwen GGUF repo) +# Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf and mmproj-Qwen2.5-VL-3B-Instruct-f16.gguf +# Record license/SHA-256 next to each file, per the asset-provenance policy. +``` + +Enable it (no code change — config + env only): + +```yaml +# application-localpoc.yml (video-clipping.editing) +vision-caption-script: ./tools/vision_caption_llamacpp.py +``` +```bash +export LLAMACPP_MTMD_BIN=/abs/llama.cpp/build/bin/llama-mtmd-cli +export LLAMACPP_VLM_MODEL=/abs/models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf +export LLAMACPP_VLM_MMPROJ=/abs/models/qwen2.5-vl-3b/mmproj-Qwen2.5-VL-3B-Instruct-f16.gguf +# LLAMACPP_VLM_NTOKENS=64 (optional) +``` + +Notes: the worker invokes `llama-mtmd-cli` per frame (serverless, fully offline; the mmap'd model stays warm in +the OS cache across frames). A persistent `llama-server` backend would be faster for large batches — a future +optimisation, not required. It is **unverified whether Qwen2.5-VL breaks the specific bowling case** — it is +substantially more capable than moondream, so it likely improves discrimination, but that is a hypothesis to +test, not a guarantee. diff --git a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java index a66f8b5..c8f8d9e 100644 --- a/src/main/java/org/example/videoclips/config/VideoClippingProperties.java +++ b/src/main/java/org/example/videoclips/config/VideoClippingProperties.java @@ -563,6 +563,13 @@ public class VideoClippingProperties { /** Frames sampled per shot for subject tracking (more = smoother path, slower). */ private int subjectTrackSamples = 5; + /** + * Tier-2 vision captioner script. Default is the moondream2 (transformers) worker. Set to + * {@code ./tools/vision_caption_llamacpp.py} to use a stronger GGUF VLM via llama.cpp (e.g. + * Qwen2.5-VL-3B) — same manifest/output contract, so nothing else changes. See docs/LOCAL-MODELS.md. + */ + private String visionCaptionScript = "./tools/vision_caption.py"; + /** * Max length (seconds) of the montage's single pre-climax "action/tension" build shot. Caps the case * where a distant action spike on a long source produces one ultra-long continuous shot (dead air + @@ -821,6 +828,14 @@ public class VideoClippingProperties { this.subjectTrackSamples = subjectTrackSamples; } + public String getVisionCaptionScript() { + return visionCaptionScript; + } + + public void setVisionCaptionScript(String visionCaptionScript) { + this.visionCaptionScript = visionCaptionScript; + } + public double getMontageMaxBuildSeconds() { return montageMaxBuildSeconds; } diff --git a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java index 1edc8ce..7433333 100644 --- a/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java +++ b/src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java @@ -27,7 +27,6 @@ import java.util.Locale; public class HighlightVisionDirector { private static final Logger log = LoggerFactory.getLogger(HighlightVisionDirector.class); - private static final String SCRIPT = "tools/vision_caption.py"; private static final long TIMEOUT_SECONDS = 600; /** * A descriptive, open-ended question. Small VLMs give far more discriminative answers to this than to a @@ -40,11 +39,13 @@ public class HighlightVisionDirector { private final VideoClippingProperties.Editing.LocalAssetWorker worker; private final String ffmpegBinary; + private final String captionScript; private final ObjectMapper objectMapper; public HighlightVisionDirector(VideoClippingProperties properties, ObjectMapper objectMapper) { this.worker = properties.getEditing().getLocalAssetWorker(); this.ffmpegBinary = properties.getEditing().getFfmpegBinary(); + this.captionScript = properties.getEditing().getVisionCaptionScript(); this.objectMapper = objectMapper; } @@ -379,7 +380,7 @@ public class HighlightVisionDirector { Path outputFile = workDir.resolve("vision-captions.json"); objectMapper.writeValue(manifestFile.toFile(), manifest); - List command = List.of(worker.getPythonBinary(), SCRIPT, + List command = List.of(worker.getPythonBinary(), captionScript, "--manifest", manifestFile.toString(), "--output", outputFile.toString()); Process process = new ProcessBuilder(command).redirectErrorStream(true).start(); String out = new String(process.getInputStream().readAllBytes(), StandardCharsets.UTF_8); diff --git a/tools/vision_caption_llamacpp.py b/tools/vision_caption_llamacpp.py new file mode 100644 index 0000000..8bdcab1 --- /dev/null +++ b/tools/vision_caption_llamacpp.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Tier-2 vision captioner backed by llama.cpp (GGUF) instead of PyTorch/transformers. + +Drop-in replacement for tools/vision_caption.py: SAME manifest/output contract, so the Java +HighlightVisionDirector is unchanged — only the configured script path differs. Uses a stronger local VLM +(e.g. Qwen2.5-VL-3B-Instruct) through llama.cpp's multimodal CLI (`llama-mtmd-cli --mmproj`), which runs on +this Intel x86 CPU via AVX SIMD and sidesteps the torch==2.2.2 / transformers 4.x version trap entirely. + +Manifest JSON: [{"id": "...", "image": "/abs/path.jpg", "question": "..."}] +Output JSON: [{"id": "...", "answer": "..."}] + +Fully offline. Paths come from env vars (so the Java side needs no backend-specific args): + LLAMACPP_MTMD_BIN path to the llama.cpp `llama-mtmd-cli` binary (required) + LLAMACPP_VLM_MODEL path to the VLM GGUF weights, e.g. Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf (required) + LLAMACPP_VLM_MMPROJ path to the matching multimodal projector GGUF (mmproj-*.gguf) (required) + LLAMACPP_VLM_NTOKENS max tokens to generate per answer (optional, default 64) + +Per-frame invocation keeps the worker simple and serverless (no loopback). The model is memory-mapped, so +after the first call the OS page cache keeps repeated loads cheap. For heavy batches a persistent +`llama-server` backend would be faster — a documented future optimisation, not required here. +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys + + +def _clean(text: str) -> str: + # Strip llama.cpp control/log noise and whitespace; keep the model's answer text. + text = text.replace("\x1b[0m", "").strip() + # Drop obvious log lines (timings, "llama_", "main:", "mtmd_", "encoding image") if any leak to stdout. + lines = [ln for ln in text.splitlines() + if ln.strip() and not re.match(r"^(llama_|main:|mtmd_|clip_|ggml_|build:|load|encoding image|\s*[\d.]+\s*(ms|tokens))", + ln.strip(), re.IGNORECASE)] + return " ".join(lines).strip() + + +def _caption(binary: str, model: str, mmproj: str, image: str, question: str, ntokens: int) -> str: + command = [ + binary, "-m", model, "--mmproj", mmproj, "--image", image, + "-p", question, "--temp", "0", "-n", str(ntokens), + ] + proc = subprocess.run(command, capture_output=True, text=True, timeout=600) + if proc.returncode != 0: + sys.stderr.write("vision_caption_llamacpp: cli failed (%s): %s\n" % (proc.returncode, proc.stderr[-500:])) + return "" + return _clean(proc.stdout) + + +def main() -> int: + parser = argparse.ArgumentParser(description="Local VLM frame captioner (llama.cpp / GGUF)") + parser.add_argument("--manifest", required=True) + parser.add_argument("--output", required=True) + args = parser.parse_args() + + binary = os.environ.get("LLAMACPP_MTMD_BIN", "") + model = os.environ.get("LLAMACPP_VLM_MODEL", "") + mmproj = os.environ.get("LLAMACPP_VLM_MMPROJ", "") + ntokens = int(os.environ.get("LLAMACPP_VLM_NTOKENS", "64")) + missing = [name for name, val in + (("LLAMACPP_MTMD_BIN", binary), ("LLAMACPP_VLM_MODEL", model), ("LLAMACPP_VLM_MMPROJ", mmproj)) + if not val or not os.path.exists(val)] + if missing: + sys.stderr.write("vision_caption_llamacpp: missing/invalid env paths: %s\n" % ", ".join(missing)) + return 2 + + with open(args.manifest, "r", encoding="utf-8") as handle: + items = json.load(handle) + + results = [] + for item in items: + answer = "" + try: + answer = _caption(binary, model, mmproj, item["image"], item["question"], ntokens) + except Exception as exc: # a single bad frame must never fail the batch + sys.stderr.write("vision_caption_llamacpp: frame %s failed: %s\n" % (item.get("id"), exc)) + results.append({"id": item.get("id"), "answer": answer}) + + os.makedirs(os.path.dirname(os.path.abspath(args.output)), exist_ok=True) + with open(args.output, "w", encoding="utf-8") as handle: + json.dump(results, handle, ensure_ascii=False, indent=2) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())