llm-inference-multimodel: repoint qwen shadow slot to Qwen2.5-14B-Instruct-1M (base Qwen disqualified, n_ctx_train=32768)

ansible/roles/llm-inference-multimodel/defaults/main.yml

@@ -62,19 +62,29 @@ llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-t
 llm_toolcall_service_name: llama-server-toolcall
 llm_toolcall_model_id: mistral-small-24b-instruct-2501
 
-# --- Shadow instance (port 8002, Qwen2.5-14B-Instruct) -----------------------
+# --- Shadow instance (port 8002) ---------------------------------------------
-# Shadow-deployment candidate to eventually replace the toolcall (8001) slot
+# History (2026-08-06): Qwen2.5-14B-Instruct (base) was deployed to this slot
-# per /home/hermes/reports/local-llm-64k-context-recommendation.md. Runs
+# and DISQUALIFIED — live /v1/models meta reported n_ctx_train=32768, well
-# alongside 8000/8001 during validation — does NOT replace either.
+# under the 64K Hermes floor (the model card's "128K" figure conflated
-# GATE: VRAM co-residency NOT confirmed as of 2026-08-06 — see role README /
+# YaRN-extended inference-time scaling with actual trained context; disabled
-# deployment report. Do not enable/start this service until that is resolved
+# by default, not baked in). Llama-3.1-8B-Instruct was tried next — cleared
-# (either free VRAM by stopping llama-server-toolcall for the shadow window,
+# the context gate (verified live n_ctx_train=131072) but failed the
-# or reduce ctx-size). llm_qwen_service_enabled defaults to false so a normal
+# tool-calling validation harness badly (8/10 hallucination-stress prompts
-# playbook run will template the unit but NOT start it.
+# triggered spurious tool_calls even at temp=0.1 with the correct official
+# chat template) — purged from disk and Ansible entirely, see git log.
+# Current candidate: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct
+# checkpoint with genuine additional long-context pretraining, NOT the same
+# weights as the disqualified base model above. MUST re-verify n_ctx_train
+# live from /v1/models before trusting any claim — the base-Qwen and
+# Llama-3.1 lessons both apply here.
+# GATE: VRAM co-residency with Phi-4(8000)+Mistral(8001) not guaranteed —
+# stop llama-server-toolcall for the shadow-test window if needed (proven
+# pattern from prior attempts). llm_qwen_service_enabled defaults to false so
+# a normal playbook run will template the unit but NOT start it.
 llm_qwen_service_enabled: false
 llm_qwen_port: 8002
-llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-Q5_K_M.gguf"
+llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf"
-llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q5_K_M.gguf"
+llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-1M-GGUF/resolve/main/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf"
 llm_qwen_model_min_bytes: 9500000000   # guard threshold; complete file ~10.5GB
 llm_qwen_ctx_size: 65536
 llm_qwen_parallel: 1
@@ -82,7 +92,7 @@ llm_qwen_gpu_layers: 99
 llm_qwen_batch_size: 2048
 llm_qwen_ubatch_size: 512
 llm_qwen_service_name: llama-server-qwen
-llm_qwen_model_id: qwen2.5-14b-instruct
+llm_qwen_model_id: qwen2.5-14b-instruct-1m
 llm_qwen_expected_vram_gb: 16.5
 
 # --- Existing Gemma baseline (rollback target — never modified by this role) -

ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2

@@ -1,5 +1,5 @@
 [Unit]
-Description=llama-server (shadow) — Qwen2.5-14B-Instruct Q5_K_M (OpenAI-compatible inference, 64K ctx)
+Description=llama-server (shadow) — Qwen2.5-14B-Instruct-1M Q5_K_M (OpenAI-compatible inference, 64K ctx)
 After=network.target nvidia-persistenced.service
 Wants=nvidia-persistenced.service