rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

llm-inference-multimodel: repoint qwen shadow slot to Qwen2.5-14B-Instruct-1M (base Qwen disqualified, n_ctx_train=32768)

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-06 10:39:45 -05:00
parent a3c1342837
commit b741f9b20b
2 changed files with 23 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
ansible/roles/llm-inference-multimodel/defaults/main.yml
View File

@@ -62,19 +62,29 @@ llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-t
llm_toolcall_service_name: llama-server-toolcall
llm_toolcall_model_id: mistral-small-24b-instruct-2501
# --- Shadow instance (port 8002, Qwen2.5-14B-Instruct) -----------------------
# Shadow-deployment candidate to eventually replace the toolcall (8001) slot
# per /home/hermes/reports/local-llm-64k-context-recommendation.md. Runs
# alongside 8000/8001 during validation — does NOT replace either.
# GATE: VRAM co-residency NOT confirmed as of 2026-08-06 — see role README /
# deployment report. Do not enable/start this service until that is resolved
# (either free VRAM by stopping llama-server-toolcall for the shadow window,
# or reduce ctx-size). llm_qwen_service_enabled defaults to false so a normal
# playbook run will template the unit but NOT start it.
# --- Shadow instance (port 8002) ---------------------------------------------
# History (2026-08-06): Qwen2.5-14B-Instruct (base) was deployed to this slot
# and DISQUALIFIED — live /v1/models meta reported n_ctx_train=32768, well
# under the 64K Hermes floor (the model card's "128K" figure conflated
# YaRN-extended inference-time scaling with actual trained context; disabled
# by default, not baked in). Llama-3.1-8B-Instruct was tried next — cleared
# the context gate (verified live n_ctx_train=131072) but failed the
# tool-calling validation harness badly (8/10 hallucination-stress prompts
# triggered spurious tool_calls even at temp=0.1 with the correct official
# chat template) — purged from disk and Ansible entirely, see git log.
# Current candidate: Qwen2.5-14B-Instruct-1M (bartowski GGUF) — distinct
# checkpoint with genuine additional long-context pretraining, NOT the same
# weights as the disqualified base model above. MUST re-verify n_ctx_train
# live from /v1/models before trusting any claim — the base-Qwen and
# Llama-3.1 lessons both apply here.
# GATE: VRAM co-residency with Phi-4(8000)+Mistral(8001) not guaranteed —
# stop llama-server-toolcall for the shadow-test window if needed (proven
# pattern from prior attempts). llm_qwen_service_enabled defaults to false so
# a normal playbook run will template the unit but NOT start it.
llm_qwen_service_enabled: false
llm_qwen_port: 8002
llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-Q5_K_M.gguf"
llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q5_K_M.gguf"
llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf"
llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-1M-GGUF/resolve/main/Qwen2.5-14B-Instruct-1M-Q5_K_M.gguf"
llm_qwen_model_min_bytes: 9500000000 # guard threshold; complete file ~10.5GB
llm_qwen_ctx_size: 65536
llm_qwen_parallel: 1
@@ -82,7 +92,7 @@ llm_qwen_gpu_layers: 99
llm_qwen_batch_size: 2048
llm_qwen_ubatch_size: 512
llm_qwen_service_name: llama-server-qwen
llm_qwen_model_id: qwen2.5-14b-instruct
llm_qwen_model_id: qwen2.5-14b-instruct-1m
llm_qwen_expected_vram_gb: 16.5
# --- Existing Gemma baseline (rollback target — never modified by this role) -

2
ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2
View File

@@ -1,5 +1,5 @@
[Unit]
Description=llama-server (shadow) — Qwen2.5-14B-Instruct Q5_K_M (OpenAI-compatible inference, 64K ctx)
Description=llama-server (shadow) — Qwen2.5-14B-Instruct-1M Q5_K_M (OpenAI-compatible inference, 64K ctx)
After=network.target nvidia-persistenced.service
Wants=nvidia-persistenced.service