jsl
/
video_editing_poc
1
0
Fork 1
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(vision): default localpoc to Qwen2.5-VL with loud moondream fallback 

Make the stronger Tier-2 judge the localpoc default (vision-caption-script ->
vision_caption_llamacpp.py). Guard it so it never silently degrades: resolveCaptionScript
checks the llama.cpp backend is provisioned (binary env + weights) and, if not, falls
back to moondream with a WARN (event=vision_backend_not_ready). Worker defaults the
model/mmproj to the repo's ./models/qwen2.5-vl-3b paths, so only the machine-specific
binary env is mandatory. mvn verify: 294 tests green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
JSLMPR 2026-07-26 17:55:46 +02:00
parent 8c0ddc1796
commit 70dffaea6c
5 changed files with 73 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
docs/LOCAL-MODELS.md
View File

@ -69,19 +69,25 @@ cmake --build build --config Release -j --target llama-mtmd-cli # -> build/bin
# Record license/SHA-256 next to each file, per the asset-provenance policy.
```
Enable it (no code change — config + env only):
Enable it (no code change — config + env). **localpoc already defaults `vision-caption-script` to the
llama.cpp worker**; if the binary env or weights are missing it **falls back to moondream with a loud WARN**
(`event=vision_backend_not_ready`), so the default never silently degrades. The model/mmproj default to the
repo paths below, so only the (machine-specific) **binary env** is mandatory:
```yaml
# application-localpoc.yml (video-clipping.editing)
vision-caption-script: ./tools/vision_caption_llamacpp.py
```
```bash
export LLAMACPP_MTMD_BIN=/abs/llama.cpp/build/bin/llama-mtmd-cli
export LLAMACPP_VLM_MODEL=/abs/models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf
export LLAMACPP_VLM_MMPROJ=/abs/models/qwen2.5-vl-3b/mmproj-F16.gguf
# LLAMACPP_VLM_NTOKENS=64 (optional)
# FAST (recommended): resident server — model loads ONCE, ~3x faster per clip.
export LLAMACPP_SERVER_BIN=/abs/llama.cpp/build/bin/llama-server
# or SIMPLE: per-frame CLI (reloads the model each frame)
# export LLAMACPP_MTMD_BIN=/abs/llama.cpp/build/bin/llama-mtmd-cli
# Optional — default to ./models/qwen2.5-vl-3b/ if unset:
# export LLAMACPP_VLM_MODEL=/abs/.../Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf
# export LLAMACPP_VLM_MMPROJ=/abs/.../mmproj-F16.gguf
# export LLAMACPP_VLM_NTOKENS=64 LLAMACPP_SERVER_PORT=8123
```
To force moondream instead: set `vision-caption-script: ./tools/vision_caption.py`.
**Runtime GOTCHA — force CPU on an Intel Mac.** The build enables Metal by default, but this machine's
integrated GPU times out on the vision encoder (`ggml_metal_synchronize: command buffer failed … GPU Timeout`).
`tools/vision_caption_llamacpp.py` therefore always passes `-ngl 0 --no-mmproj-offload` (pure CPU/AVX). Expect

39
src/main/java/org/example/videoclips/editing/HighlightVisionDirector.java
View File

@ -42,13 +42,50 @@ public class HighlightVisionDirector {
private final String captionScript;
private final ObjectMapper objectMapper;
private static final String MOONDREAM_SCRIPT = "./tools/vision_caption.py";
private static final String LLAMACPP_DEFAULT_MODEL = "./models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf";
private static final String LLAMACPP_DEFAULT_MMPROJ = "./models/qwen2.5-vl-3b/mmproj-F16.gguf";
public HighlightVisionDirector(VideoClippingProperties properties, ObjectMapper objectMapper) {
this.worker = properties.getEditing().getLocalAssetWorker();
this.ffmpegBinary = properties.getEditing().getFfmpegBinary();
this.captionScript = properties.getEditing().getVisionCaptionScript();
this.captionScript = resolveCaptionScript(properties.getEditing().getVisionCaptionScript());
this.objectMapper = objectMapper;
}
/**
* Resolve the effective captioner. When the llama.cpp backend is configured but not fully provisioned (no
* binary env var, or missing weights), fall back to moondream and WARN loudly — so a default of the stronger
* model never silently degrades into a wrong pick on an unprovisioned machine.
*/
static String resolveCaptionScript(String configured) {
if (configured == null || !configured.contains("vision_caption_llamacpp")) {
return configured; // moondream or a custom backend — use as-is
}
boolean scriptOk = isFile(configured);
boolean binOk = isFile(System.getenv("LLAMACPP_SERVER_BIN")) || isFile(System.getenv("LLAMACPP_MTMD_BIN"));
boolean modelOk = isFile(envOr("LLAMACPP_VLM_MODEL", LLAMACPP_DEFAULT_MODEL));
boolean mmprojOk = isFile(envOr("LLAMACPP_VLM_MMPROJ", LLAMACPP_DEFAULT_MMPROJ));
if (scriptOk && binOk && modelOk && mmprojOk) {
log.info("event=vision_backend backend=llamacpp script={}", configured);
return configured;
}
log.warn("event=vision_backend_not_ready backend=llamacpp fallback=moondream "
+ "script_ok={} binary_env_ok={} model_ok={} mmproj_ok={} "
+ "(set LLAMACPP_SERVER_BIN or LLAMACPP_MTMD_BIN and provision the weights to use Qwen)",
scriptOk, binOk, modelOk, mmprojOk);
return MOONDREAM_SCRIPT;
}
private static boolean isFile(String path) {
return path != null && !path.isBlank() && Files.isRegularFile(Path.of(path.trim()));
}
private static String envOr(String name, String fallback) {
String v = System.getenv(name);
return v == null || v.isBlank() ? fallback : v;
}
/**
* Captions {@code samples} frames evenly across the source with the local VLM (one worker call, model
* loaded once). Returns timed captions describing the key action/achievement at each moment, or an empty

5
src/main/resources/application-localpoc.yml
View File

@ -28,6 +28,11 @@ video-clipping:
# R14 subject-tracking reframe: follow the detected subject (YOLO/CV venv) instead of a static centre crop.
subject-reframe-enabled: true
# Tier-2 vision JUDGE: prefer the stronger Qwen2.5-VL (llama.cpp/GGUF) backend. It falls back to moondream
# with a loud WARN if the binary env (LLAMACPP_SERVER_BIN / LLAMACPP_MTMD_BIN) or weights aren't provisioned
# — so this default never silently degrades. See docs/LOCAL-MODELS.md.
vision-caption-script: ./tools/vision_caption_llamacpp.py
assets:
# Empty/absent asset folders -> the pipeline generates assets with local models instead of copying.
music-folder: ./input/localpoc/assets/music

12
src/test/java/org/example/videoclips/editing/HighlightVisionDirectorTest.java
View File

@ -77,6 +77,18 @@ class HighlightVisionDirectorTest {
assertThat(curve[5]).isEqualTo(1.0); // window ~5.5s -> nearest 5.0s "arms raised celebration"
}
@Test
void llamacppBackendFallsBackToMoondreamWhenNotProvisioned() {
// A non-llama.cpp script is used as-is.
assertThat(HighlightVisionDirector.resolveCaptionScript("./tools/vision_caption.py"))
.isEqualTo("./tools/vision_caption.py");
assertThat(HighlightVisionDirector.resolveCaptionScript(null)).isNull();
// The llama.cpp backend requires a binary env (LLAMACPP_SERVER_BIN / LLAMACPP_MTMD_BIN), which is not
// set in the test JVM -> it must fall back to moondream rather than silently use a broken backend.
assertThat(HighlightVisionDirector.resolveCaptionScript("./tools/vision_caption_llamacpp.py"))
.isEqualTo("./tools/vision_caption.py");
}
@Test
void detectsAnticipatoryDescriptions() {
assertThat(HighlightVisionDirector.isAnticipatory("The person is about to kick the ball")).isTrue();

5
tools/vision_caption_llamacpp.py
View File

@ -120,8 +120,9 @@ def main() -> int:
parser.add_argument("--output", required=True)
args = parser.parse_args()
model = os.environ.get("LLAMACPP_VLM_MODEL", "")
mmproj = os.environ.get("LLAMACPP_VLM_MMPROJ", "")
# Default to the repo's provisioned paths so only the (machine-specific) binary needs an env var.
model = os.environ.get("LLAMACPP_VLM_MODEL") or "./models/qwen2.5-vl-3b/Qwen2.5-VL-3B-Instruct-Q4_K_M.gguf"
mmproj = os.environ.get("LLAMACPP_VLM_MMPROJ") or "./models/qwen2.5-vl-3b/mmproj-F16.gguf"
ntokens = int(os.environ.get("LLAMACPP_VLM_NTOKENS", "64"))
server_bin = os.environ.get("LLAMACPP_SERVER_BIN", "")
cli_bin = os.environ.get("LLAMACPP_MTMD_BIN", "")