From b741f9b20be76908aa6c9ac954243391025123b8 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Thu, 6 Aug 2026 10:39:45 -0500 Subject: [PATCH] llm-inference-multimodel: repoint qwen shadow slot to Qwen2.5-14B-Instruct-1M (base Qwen disqualified, n_ctx_train=32768) --- .../defaults/main.yml | 34 ++++++++++++------- .../templates/llama-server-qwen.service.j2 | 2 +- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 377659c..0d1908a 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -62,19 +62,29 @@ llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-t llm_toolcall_service_name: llama-server-toolcall llm_toolcall_model_id: mistral-small-24b-instruct-2501 -# --- Shadow instance (port 8002, Qwen2.5-14B-Instruct) ----------------------- -# Shadow-deployment candidate to eventually replace the toolcall (8001) slot -# per /home/hermes/reports/local-llm-64k-context-recommendation.md. Runs -# alongside 8000/8001 during validation — does NOT replace either. -# GATE: VRAM co-residency NOT confirmed as of 2026-08-06 — see role README / -# deployment report. Do not enable/start this service until that is resolved -# (either free VRAM by stopping llama-server-toolcall for the shadow window, -# or reduce ctx-size). llm_qwen_service_enabled defaults to false so a normal -# playbook run will template the unit but NOT start it. +# --- Shadow instance (port 8002) --------------------------------------------- +# History (2026-08-06): Qwen2.5-14B-Instruct (base) was deployed to this slot +# and DISQUALIFIED — live /v1/models meta reported n_ctx_train=32768, well +# under the 64K Hermes floor (the model card's "128K" figure conflated +# YaRN-extended inference-time scaling with actual trained context; disabled +# by default, not baked in). Llama-3.1-8B-Instruct was tried next — cleared +# the context gate (verified live n_ctx_train=131072) but failed the +# tool-calling validation harness badly (8/10 hallucination-stress prompts +# triggered spurious tool_calls even at temp=0.1 with the correct official +# chat template) — purged from disk and Ansible entirely, see git log. +# Current candidate: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct +# checkpoint with genuine additional long-context pretraining, NOT the same +# weights as the disqualified base model above. MUST re-verify n_ctx_train +# live from /v1/models before trusting any claim — the base-Qwen and +# Llama-3.1 lessons both apply here. +# GATE: VRAM co-residency with Phi-4(8000)+Mistral(8001) not guaranteed — +# stop llama-server-toolcall for the shadow-test window if needed (proven +# pattern from prior attempts). llm_qwen_service_enabled defaults to false so +# a normal playbook run will template the unit but NOT start it. llm_qwen_service_enabled: false llm_qwen_port: 8002 -llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-Q5_K_M.gguf" -llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q5_K_M.gguf" +llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf" +llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-1M-GGUF/resolve/main/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf" llm_qwen_model_min_bytes: 9500000000 # guard threshold; complete file ~10.5GB llm_qwen_ctx_size: 65536 llm_qwen_parallel: 1 @@ -82,7 +92,7 @@ llm_qwen_gpu_layers: 99 llm_qwen_batch_size: 2048 llm_qwen_ubatch_size: 512 llm_qwen_service_name: llama-server-qwen -llm_qwen_model_id: qwen2.5-14b-instruct +llm_qwen_model_id: qwen2.5-14b-instruct-1m llm_qwen_expected_vram_gb: 16.5 # --- Existing Gemma baseline (rollback target — never modified by this role) - diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2 index 76a11ae..119b0f8 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2 @@ -1,5 +1,5 @@ [Unit] -Description=llama-server (shadow) — Qwen2.5-14B-Instruct Q5_K_M (OpenAI-compatible inference, 64K ctx) +Description=llama-server (shadow) — Qwen2.5-14B-Instruct-1M Q5_K_M (OpenAI-compatible inference, 64K ctx) After=network.target nvidia-persistenced.service Wants=nvidia-persistenced.service