From a3c92f70bf632576bd02abf1192ce97b5074902d Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 31 Aug 2026 21:28:48 -0500 Subject: [PATCH] feat(deploy-vllm): swap DeepSeek-R1-Distill-Qwen-32B for Gemma 4 26B A4B AWQ (t_gemma4_swap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Retired DeepSeek-R1-Distill-Qwen-32B after confirming its auto tool-choice reliability is a known, documented DeepSeek-R1-distillation limitation (trained on pure reasoning traces, no function-calling data — upstream GitHub-confirmed, not a config gap). Model choice moved to Gemma 4 26B A4B (Google, Apache 2.0, US-origin, matches Ryan's model-origin preference): - cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit — MoE (25.2B total / 3.8B active), chosen over the dense 31B variant for smaller on-disk footprint (~17.2GB vs ~20.9GB), buying more KV-cache headroom on this 24GB card - max_model_len=65536 (comfortably over Hermes's 64K floor; native context is 256K, no extension trick needed) - Native gemma4 tool-call-parser + gemma4 reasoning-parser (both registered in this host's vLLM 0.28.0) — purpose-built for this model's actual output format, not a same-family approximation - kv_cache_dtype: int4_per_token_head from the outset (learned from the DeepSeek swap's fp16->fp8->int4 trial-and-error escalation) Bug found and fixed during deployment: the repo's config.json declares quant_method 'compressed-tensors' (llm-compressor output) despite the repo name saying 'AWQ-4bit'. Passing --quantization awq explicitly caused a hard pydantic ValidationError on every startup attempt. Fix: omit the quantization field entirely and let vLLM auto-detect from the model's own config.json — confirmed clean single-attempt start, NRestarts=0, once removed. Verified live: - /health 200, /v1/models confirms max_model_len=65536 - Live completion: correct answer, no unwanted reasoning trace by default - tool_choice=auto with a clear trigger prompt: correct tool_calls response with valid JSON args — the exact test DeepSeek-R1-Distill failed (it either answered in plain text or burned tokens reasoning about how to call the tool instead of calling it) - tool_choice=auto with an irrelevant tool present: correctly answered in plain text, did not over-trigger the tool - Ansible idempotent re-run confirmed: changed=0, NRestarts=0, clean journalctl (zero error/traceback lines) after a fresh restart Known follow-up (not done here): Hindsight's HINDSIGHT_API_LLM_MODEL cluster config still references the retired DeepSeek-R1-Distill-Qwen-32B (itself a follow-up from the prior Qwen2.5-32B swap) — needs another GitOps update to point at Gemma-4-26B-A4B-it-AWQ. --- ansible/host_vars/astro-orbiter/vars.yml | 88 +++++++++++++++--------- 1 file changed, 54 insertions(+), 34 deletions(-) diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index 330a0b6..f90700c 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -148,45 +148,65 @@ llm_staged_models: # default), left to be set client-side per the model card's guidance — # flagging for whoever wires this into Hermes profile configs next. vllm_models: - - id: "DeepSeek-R1-Distill-Qwen-32B-AWQ" - hf_repo: "casperhansen/deepseek-r1-distill-qwen-32b-awq" + - id: "Gemma-4-26B-A4B-it-AWQ" + hf_repo: "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit" role: primary - quantization: awq + # NO quantization field set (unlike the AutoAWQ-quantized DeepSeek/ + # Qwen2.5 models above) — live test (2026-09-01) found this repo's + # config.json declares quant_method: "compressed-tensors" (llm-compressor + # tool output, not classic AutoAWQ), even though the repo name says + # "AWQ-4bit". Passing --quantization awq explicitly caused a hard + # pydantic ValidationError at every single startup attempt: "Quantization + # method specified in the model config (compressed-tensors) does not + # match the quantization method specified in the `quantization` argument + # (awq)." vLLM auto-detects the quant method correctly from the model's + # own config.json when --quantization is omitted — confirmed fix, clean + # start. Lesson: don't trust a HF repo's naming convention ("...-AWQ...") + # for the `quantization:` field here — check config.json's quant_method. port: 8000 - max_model_len: 32768 - # VRAM correction (2026-09-01, live test): weights alone load at 18.17 - # GiB (confirmed via journalctl "Model loading took 18.17 GiB memory"). - # fp16 KV cache at 32768 ctx needs 8.0 GiB per vLLM's own error message - # ("To serve at least one request with the model's max seq len (32768), - # 8.0 GiB KV cache is needed") — 18.17 + 8.0 = 26.17GB, DOES NOT FIT a - # 24GB card even at gpu_memory_utilization=1.0. Confirmed via 3 live - # crash-loop attempts at 0.95 (usable ~23.35GiB budget): vLLM's own - # ValueError reported "estimated maximum model length is 14528" at - # whatever KV budget was actually available — nowhere close to 32768. - # FIX: --kv-cache-dtype fp8 halves KV cache memory (~4.0GiB instead of - # 8.0GiB) — 18.17 + 4.0 = ~22.2GB, fits with ~1.4GB headroom at 0.95. - # fp8 KV cache is a standard vLLM feature (not experimental for this - # vLLM version), minor precision loss in attention scores, no known - # material quality impact for a distilled reasoning model at this scale. - # VRAM correction round 3 (2026-09-01, live test): manual - # kv_cache_memory_bytes did NOT fix it either — same OOM pattern - # (small ~150MB alloc failing with only ~52MB actually free), meaning - # real GPU usage at warmup exceeds what profiling/reservation account - # for by roughly ~1GB (unaccounted FlashInfer warmup workspace buffers, - # not sized during profiling or reservation). Rather than keep guessing - # at a KV byte budget with ~0 margin, switched KV dtype from fp8 (8-bit) - # to int4_per_token_head (4-bit) — roughly HALVES KV cache footprint - # for the same 32768 ctx (~2.0GiB instead of ~4.0-4.3GiB), buying back - # ~2GiB of real headroom to absorb whatever the unaccounted warmup - # overhead actually is. Reverted to standard percentage-based - # gpu_memory_utilization (no kv_cache_memory_bytes override) so vLLM's - # own profiling determines the exact KV budget again, now with much - # more slack in play. Some precision loss vs fp8 is expected for KV - # cache in int4 — acceptable tradeoff for a homelab box; revisit if - # DeepSeek's output quality visibly degrades once running. + # Ryan direction (2026-09-01, t_gemma4_swap): DeepSeek-R1-Distill-Qwen-32B + # retired after confirming its `auto` tool-choice reliability is a known, + # documented DeepSeek-R1-distillation limitation (trained on pure + # reasoning traces, no function-calling data — GitHub-confirmed upstream, + # not a vLLM config gap). Replaced with Gemma 4 26B A4B (Google, + # Apache 2.0, US-origin — matches Ryan's standing model-origin + # preference, unlike Qwen/DeepSeek). Chose MoE (26B A4B, 3.8B active) + # over the dense 31B variant: ~3.7GB smaller on-disk AWQ footprint + # (17.2GB vs 20.9GB) buys more KV-cache headroom on this tight 24GB + # card, and decode should be faster (memory-bandwidth-bound on active + # params, not total params). Tradeoff accepted: MoE scores lower than + # dense on the Tau2 tool-use benchmark (68.2% vs 76.9%) but still beats + # every other size in the family except the 31B on most reasoning + # benchmarks. Model choice: cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit — + # AutoAWQ 4-bit group_size=32, MoE expert layers (gate/up/down/router) + # explicitly excluded from quantization ("ignore" list in config.json) + # per standard llm-compressor MoE quant practice — only the dense + # attention/projection layers are 4-bit, experts stay higher precision. + # Native architecture: Gemma4ForConditionalGeneration (registered + # natively in this host's installed vLLM 0.28.0 — vllm/model_executor/ + # models/registry.py line 415 — no plugin/trust-remote-code needed). + # Native max_position_embeddings: 262144 (256K) — Hermes's 64K floor is + # comfortably covered without any context-extension trick. + max_model_len: 65536 + # VRAM math (not yet live-validated — see swap validation log below + # once run): AWQ weights ~17.2GB on disk (dense attn 4-bit + MoE + # experts higher-precision, per config.json's compressed-tensors + # ignore list). Starting the KV cache dtype at int4_per_token_head + # from the outset (rather than fp16 -> fp8 -> int4 trial-and-error like + # the DeepSeek swap) since that same escalation pattern is expected to + # repeat on this VRAM-constrained card for any 20+ GB model at >32K ctx. kv_cache_dtype: int4_per_token_head gpu_memory_utilization: 0.95 enforce_eager: true + # Native tool-calling + reasoning support (no `hermes` workaround + # needed, unlike DeepSeek-R1-Distill): Gemma4EngineToolParser and + # Gemma4ParserReasoningAdapter are both registered natively in this + # host's vLLM 0.28.0 (vllm/tool_parsers/__init__.py, + # vllm/reasoning/__init__.py) — purpose-built for this model's actual + # output format, not a same-family approximation. + enable_auto_tool_choice: true + tool_call_parser: gemma4 + reasoning_parser: gemma4 enabled: true - id: "Qwen3-8B-AWQ" hf_repo: "Qwen/Qwen3-8B-AWQ"