d10255297c
- New llama-server-qwen systemd unit template, gated by llm_qwen_service_enabled (default false) - Idempotent GGUF download task (bartowski Qwen2.5-14B-Instruct-Q5_K_M, stat-guarded) - Launch flags per local-llm-64k-context-recommendation.md: ctx-size 65536, flash-attn, q8_0 KV cache, batch 2048/ubatch 512, jinja, parallel 1 - verify.yml only starts/verifies the qwen unit when llm_qwen_service_enabled=true - README: documents live VRAM gate finding (nvidia-smi 2026-08-06: Phi-4+Mistral already ~16.6/24GB, ~7.5GB free -- insufficient for Qwen weights concurrently) and options - Does NOT touch llama-server-aux (8000) or llama-server-toolcall (8001) service state
125 lines
4.7 KiB
YAML
125 lines
4.7 KiB
YAML
---
|
|
# ------------------------------------------------------------------------------
|
|
# FILE: roles/llm-inference-multimodel/tasks/models.yml
|
|
# DESCRIPTION: Phase 1 — download both GGUFs to {{ llm_models_dir }}.
|
|
# Idempotent: reuses the stat + size-threshold guard pattern
|
|
# from the llm-inference-homelab skill / roles/llm-inference's
|
|
# serve.yml, so reruns don't re-pull 8.5GB / 11.7GB files.
|
|
#
|
|
# Does NOT touch the existing Gemma GGUF — separate directory
|
|
# entries, no overlap, no deletion of anything pre-existing.
|
|
# ------------------------------------------------------------------------------
|
|
|
|
- name: Create models directory
|
|
ansible.builtin.file:
|
|
path: "{{ llm_models_dir }}"
|
|
state: directory
|
|
owner: "{{ llm_service_user }}"
|
|
group: "{{ llm_service_user }}"
|
|
mode: "0755"
|
|
become: true
|
|
|
|
# --- Aux model (Phi-4-14B Q4_K_M) --------------------------------------------
|
|
|
|
- name: Check if aux model GGUF already exists
|
|
ansible.builtin.stat:
|
|
path: "{{ llm_aux_model_path }}"
|
|
register: llm_aux_model_stat
|
|
|
|
- name: Download aux model — Phi-4-14B-Q4_K_M GGUF
|
|
ansible.builtin.get_url:
|
|
url: "{{ llm_aux_model_url }}"
|
|
dest: "{{ llm_aux_model_path }}"
|
|
headers:
|
|
Authorization: "Bearer {{ llm_hf_token }}"
|
|
owner: "{{ llm_service_user }}"
|
|
group: "{{ llm_service_user }}"
|
|
mode: "0644"
|
|
timeout: 7200
|
|
force: false
|
|
become: true
|
|
no_log: true
|
|
# Idempotency guard: skip if file exists and is above the min-size threshold
|
|
# (catches partial/truncated downloads from an interrupted prior run).
|
|
when: not llm_aux_model_stat.stat.exists or (llm_aux_model_stat.stat.size | int) < (llm_aux_model_min_bytes | int)
|
|
|
|
# --- Tool-calling model (Mistral-Small-24B Q3_K_M) ---------------------------
|
|
|
|
- name: Check if tool-calling model GGUF already exists
|
|
ansible.builtin.stat:
|
|
path: "{{ llm_toolcall_model_path }}"
|
|
register: llm_toolcall_model_stat
|
|
|
|
- name: Download tool-calling model — Mistral-Small-24B-Instruct-2501 Q3_K_M GGUF
|
|
ansible.builtin.get_url:
|
|
url: "{{ llm_toolcall_model_url }}"
|
|
dest: "{{ llm_toolcall_model_path }}"
|
|
headers:
|
|
Authorization: "Bearer {{ llm_hf_token }}"
|
|
owner: "{{ llm_service_user }}"
|
|
group: "{{ llm_service_user }}"
|
|
mode: "0644"
|
|
timeout: 7200
|
|
force: false
|
|
become: true
|
|
no_log: true
|
|
when: not llm_toolcall_model_stat.stat.exists or (llm_toolcall_model_stat.stat.size | int) < (llm_toolcall_model_min_bytes | int)
|
|
|
|
- name: Report model files present on disk
|
|
ansible.builtin.debug:
|
|
msg:
|
|
- "Aux model: {{ llm_aux_model_path }}"
|
|
- "Tool-calling model: {{ llm_toolcall_model_path }}"
|
|
|
|
# --- Tool-calling chat template override -------------------------------------
|
|
# Mistral-Small-24B-Instruct-2501's own embedded/tokenizer_config chat template
|
|
# has NO tool-call handling ([AVAILABLE_TOOLS]/[TOOL_CALLS] blocks) — confirmed
|
|
# via /props chat_template_caps.supports_tools=false against the stock
|
|
# template. Mistral-Nemo-Instruct-2407 ships a template with full tool-calling
|
|
# support and the same Mistral instruct format family, so we serve it via
|
|
# --chat-template-file instead of relying on GGUF-embedded metadata.
|
|
# See docs/validation-log.md for the investigation and probe results.
|
|
|
|
- name: Create chat templates directory
|
|
ansible.builtin.file:
|
|
path: "{{ llm_toolcall_chat_template_file | dirname }}"
|
|
state: directory
|
|
owner: "{{ llm_service_user }}"
|
|
group: "{{ llm_service_user }}"
|
|
mode: "0755"
|
|
become: true
|
|
|
|
- name: Deploy tool-calling-capable chat template (from Mistral-Nemo-Instruct-2407)
|
|
ansible.builtin.copy:
|
|
src: mistral-small-tool-use.jinja
|
|
dest: "{{ llm_toolcall_chat_template_file }}"
|
|
owner: "{{ llm_service_user }}"
|
|
group: "{{ llm_service_user }}"
|
|
mode: "0644"
|
|
become: true
|
|
|
|
# --- Shadow model (Qwen2.5-14B-Instruct Q5_K_M, port 8002) --------------------
|
|
# Downloaded unconditionally (so the ~10.5GB file is staged ahead of any VRAM
|
|
# decision) — only the *service start* is gated by llm_qwen_service_enabled
|
|
# (see verify.yml / VRAM gate note in defaults/main.yml).
|
|
|
|
- name: Check if Qwen shadow model GGUF already exists
|
|
ansible.builtin.stat:
|
|
path: "{{ llm_qwen_model_path }}"
|
|
register: llm_qwen_model_stat
|
|
|
|
- name: Download Qwen2.5-14B-Instruct-Q5_K_M GGUF (bartowski quant)
|
|
ansible.builtin.get_url:
|
|
url: "{{ llm_qwen_model_url }}"
|
|
dest: "{{ llm_qwen_model_path }}"
|
|
headers:
|
|
Authorization: "Bearer {{ llm_hf_token }}"
|
|
owner: "{{ llm_service_user }}"
|
|
group: "{{ llm_service_user }}"
|
|
mode: "0644"
|
|
timeout: 7200
|
|
force: false
|
|
become: true
|
|
no_log: true
|
|
when: not llm_qwen_model_stat.stat.exists or (llm_qwen_model_stat.stat.size | int) < (llm_qwen_model_min_bytes | int)
|