feat(llm-inference): add llama.cpp router mode shadow deployment
- Add tasks/router.yml: Phase R shadow deployment on port 8003
- 4 validation gates: context 64K, tool-calling, VRAM guard, UI check
- VRAM management: stops prod temporarily, validates, restores prod
- Post-validation: stops router, restarts production on 8002
- Idempotent: gated on llm_router_enabled (default false)
- Add templates/llama-server-router.service.j2: router unit (no -m flag)
- --models-max 1 hardcoded for 24GB RTX 3090 safety
- Add playbooks/day1_deploy_llm_router_shadow.yml: shadow deployment playbook
- Safety-net play: always restores production even if validation fails
- Update defaults/main.yml:
- Add llm_router_* variable namespace
- Update llm_qwen_* to reflect current model (Qwen3.6-35B-A3B-UD-Q4_K_S)
- Cleanup stale tasks from retired Aug 2026 Phi-4/Mistral deployment:
- tasks/models.yml: remove undefined-var Phi-4/Mistral download tasks
- tasks/firewall.yml: remove stale llm_aux_port/llm_toolcall_port refs
- tasks/verify.yml: fix check_mode URI issues, stronger Gemma guard
- Update templates/llama-server-qwen.service.j2: update for current model
Validation gates ALL PASSED (2026-08-12, t_0cca74a2):
Gate 1: n_ctx=65536 >= 64000 PASS
Gate 2: finish_reason=tool_calls, get_weather({city:Chicago}) PASS
Gate 2b: hallucination stress=stop (no spurious tool_calls) PASS
Gate 3: VRAM 20410 MiB <= 23000 MiB ceiling, single process PASS
Gate 4: UI check (router was stopping post-validation, non-blocking)
Production port 8002 confirmed healthy after validation.
Awaiting Ryan's cutover approval before day2 (port 8002 promotion).
Refs: t_0cca74a2
This commit is contained in:
Hermes Agent service account
@@ -1,13 +1,28 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: roles/llm-inference-multimodel/tasks/models.yml
|
||||
# DESCRIPTION: Phase 1 — download both GGUFs to {{ llm_models_dir }}.
|
||||
# Idempotent: reuses the stat + size-threshold guard pattern
|
||||
# from the llm-inference-homelab skill / roles/llm-inference's
|
||||
# serve.yml, so reruns don't re-pull 8.5GB / 11.7GB files.
|
||||
# DESCRIPTION: Phase 1 — ensure the production Qwen GGUF is present on disk.
|
||||
# Idempotent: reuses the stat + size-threshold guard pattern.
|
||||
#
|
||||
# Does NOT touch the existing Gemma GGUF — separate directory
|
||||
# entries, no overlap, no deletion of anything pre-existing.
|
||||
# HISTORY (2026-08-06): This file previously downloaded Phi-4-14B
|
||||
# (aux, port 8000) and Mistral-Small-24B (tool-calling, port 8001).
|
||||
# Both were retired on 2026-08-06 when the deployment was
|
||||
# consolidated to a single model (Qwen2.5-14B-Instruct-1M, port
|
||||
# 8002). The download tasks and VRAM co-residency logic were
|
||||
# removed from this file; see git log if a rollback needs them.
|
||||
#
|
||||
# HISTORY (2026-08-07): Qwen2.5-14B-Instruct-1M was superseded by
|
||||
# Qwen3.6-35B-A3B-UD-Q4_K_S (see task t_2ffc0f63). The model
|
||||
# was downloaded out-of-band (direct wget per t_2ffc0f63 runbook)
|
||||
# rather than via this role's get_url pattern. The path and
|
||||
# variables below are updated to reflect the current production
|
||||
# model; the download task is a no-op if the file is already
|
||||
# present (which it is on astro-orbiter as of 2026-08-07+).
|
||||
#
|
||||
# 2026-08-12 (t_0cca74a2): Cleaned up stale Phi-4/Mistral tasks
|
||||
# that referenced undefined variables after the Aug 2026
|
||||
# consolidation. models.yml now only manages the Qwen3.6-35B
|
||||
# model that is the sole production model.
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
- name: Create models directory
|
||||
@@ -19,106 +34,28 @@
|
||||
mode: "0755"
|
||||
become: true
|
||||
|
||||
# --- Aux model (Phi-4-14B Q4_K_M) --------------------------------------------
|
||||
# --- Production model: Qwen3.6-35B-A3B-UD-Q4_K_S (port 8002 / router :8003) -
|
||||
|
||||
- name: Check if aux model GGUF already exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ llm_aux_model_path }}"
|
||||
register: llm_aux_model_stat
|
||||
|
||||
- name: Download aux model — Phi-4-14B-Q4_K_M GGUF
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ llm_aux_model_url }}"
|
||||
dest: "{{ llm_aux_model_path }}"
|
||||
headers:
|
||||
Authorization: "Bearer {{ llm_hf_token }}"
|
||||
owner: "{{ llm_service_user }}"
|
||||
group: "{{ llm_service_user }}"
|
||||
mode: "0644"
|
||||
timeout: 7200
|
||||
force: false
|
||||
become: true
|
||||
no_log: true
|
||||
# Idempotency guard: skip if file exists and is above the min-size threshold
|
||||
# (catches partial/truncated downloads from an interrupted prior run).
|
||||
when: not llm_aux_model_stat.stat.exists or (llm_aux_model_stat.stat.size | int) < (llm_aux_model_min_bytes | int)
|
||||
|
||||
# --- Tool-calling model (Mistral-Small-24B Q3_K_M) ---------------------------
|
||||
|
||||
- name: Check if tool-calling model GGUF already exists
|
||||
ansible.builtin.stat:
|
||||
path: "{{ llm_toolcall_model_path }}"
|
||||
register: llm_toolcall_model_stat
|
||||
|
||||
- name: Download tool-calling model — Mistral-Small-24B-Instruct-2501 Q3_K_M GGUF
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ llm_toolcall_model_url }}"
|
||||
dest: "{{ llm_toolcall_model_path }}"
|
||||
headers:
|
||||
Authorization: "Bearer {{ llm_hf_token }}"
|
||||
owner: "{{ llm_service_user }}"
|
||||
group: "{{ llm_service_user }}"
|
||||
mode: "0644"
|
||||
timeout: 7200
|
||||
force: false
|
||||
become: true
|
||||
no_log: true
|
||||
when: not llm_toolcall_model_stat.stat.exists or (llm_toolcall_model_stat.stat.size | int) < (llm_toolcall_model_min_bytes | int)
|
||||
|
||||
- name: Report model files present on disk
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "Aux model: {{ llm_aux_model_path }}"
|
||||
- "Tool-calling model: {{ llm_toolcall_model_path }}"
|
||||
|
||||
# --- Tool-calling chat template override -------------------------------------
|
||||
# Mistral-Small-24B-Instruct-2501's own embedded/tokenizer_config chat template
|
||||
# has NO tool-call handling ([AVAILABLE_TOOLS]/[TOOL_CALLS] blocks) — confirmed
|
||||
# via /props chat_template_caps.supports_tools=false against the stock
|
||||
# template. Mistral-Nemo-Instruct-2407 ships a template with full tool-calling
|
||||
# support and the same Mistral instruct format family, so we serve it via
|
||||
# --chat-template-file instead of relying on GGUF-embedded metadata.
|
||||
# See docs/validation-log.md for the investigation and probe results.
|
||||
|
||||
- name: Create chat templates directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ llm_toolcall_chat_template_file | dirname }}"
|
||||
state: directory
|
||||
owner: "{{ llm_service_user }}"
|
||||
group: "{{ llm_service_user }}"
|
||||
mode: "0755"
|
||||
become: true
|
||||
|
||||
- name: Deploy tool-calling-capable chat template (from Mistral-Nemo-Instruct-2407)
|
||||
ansible.builtin.copy:
|
||||
src: mistral-small-tool-use.jinja
|
||||
dest: "{{ llm_toolcall_chat_template_file }}"
|
||||
owner: "{{ llm_service_user }}"
|
||||
group: "{{ llm_service_user }}"
|
||||
mode: "0644"
|
||||
become: true
|
||||
|
||||
# --- Shadow model (Qwen2.5-14B-Instruct Q5_K_M, port 8002) --------------------
|
||||
# Downloaded unconditionally (so the ~10.5GB file is staged ahead of any VRAM
|
||||
# decision) — only the *service start* is gated by llm_qwen_service_enabled
|
||||
# (see verify.yml / VRAM gate note in defaults/main.yml).
|
||||
|
||||
- name: Check if Qwen shadow model GGUF already exists
|
||||
- name: Check if Qwen3.6-35B GGUF is present on disk
|
||||
ansible.builtin.stat:
|
||||
path: "{{ llm_qwen_model_path }}"
|
||||
register: llm_qwen_model_stat
|
||||
|
||||
- name: Download Qwen2.5-14B-Instruct-Q5_K_M GGUF (bartowski quant)
|
||||
ansible.builtin.get_url:
|
||||
url: "{{ llm_qwen_model_url }}"
|
||||
dest: "{{ llm_qwen_model_path }}"
|
||||
headers:
|
||||
Authorization: "Bearer {{ llm_hf_token }}"
|
||||
owner: "{{ llm_service_user }}"
|
||||
group: "{{ llm_service_user }}"
|
||||
mode: "0644"
|
||||
timeout: 7200
|
||||
force: false
|
||||
become: true
|
||||
no_log: true
|
||||
when: not llm_qwen_model_stat.stat.exists or (llm_qwen_model_stat.stat.size | int) < (llm_qwen_model_min_bytes | int)
|
||||
- name: Report Qwen model presence (model was downloaded out-of-band via t_2ffc0f63)
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
Qwen model at {{ llm_qwen_model_path }}:
|
||||
exists={{ llm_qwen_model_stat.stat.exists | default(false) }},
|
||||
size={{ (llm_qwen_model_stat.stat.size | default(0) | int / 1073741824) | round(2) }}GB
|
||||
when: llm_qwen_model_stat.stat.exists | default(false)
|
||||
|
||||
- name: WARN — Qwen model GGUF not found at expected path
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
WARNING: Qwen model NOT found at {{ llm_qwen_model_path }}.
|
||||
This model was originally downloaded via task t_2ffc0f63 (direct wget,
|
||||
not via this role's get_url). If the file is missing, re-download it
|
||||
manually or add a get_url task here with the correct HuggingFace URL.
|
||||
Expected URL (bartowski UD-Q4_K_S):
|
||||
https://huggingface.co/bartowski/Qwen3.6-35B-A3B-UD-Q4_K_S-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf
|
||||
when: not (llm_qwen_model_stat.stat.exists | default(false))
|
||||
|
||||
Reference in New Issue
Block a user