diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index cd7b4df..8244069 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -190,38 +190,6 @@ llm_router_nomic_ctx_size: 8192 llm_router_nomic_batch_size: 4096 llm_router_nomic_ubatch_size: 4096 -# --- Monitoring: VRAM exporter + Prometheus scrape + Grafana dashboard ------- -# Phase 3: GPU/LLM monitoring deployment (Wong, 2026-08-18) -# Provides: VRAM textfile exporter, Prometheus scrape config for llama-swap -# /metrics endpoint, Grafana 6-panel dashboard, PrometheusRule alert rules. -# -# Ref: roles/llm-inference-multimodel/references/monitoring-llm-homelab-ciro-luciotta-2026.md -llm_monitoring_enabled: true # gate for monitoring tasks -llm_vram_exporter_script: /opt/llama-server-monitoring/nvidia-smi-vram-exporter.sh -llm_vram_exporter_cron_minute: "*" # run every minute -llm_vram_exporter_gpu_index: 0 # GPU 0 (RTX 3090 on astro-orbiter) -llm_vram_textfile_dir: /var/lib/node_exporter/textfile_collector - -# Alert thresholds (per Ciro Luciotta pattern) -llm_vram_critical_mib: 24000 # ~90% of 24GB RTX 3090 -llm_kv_cache_spill_ratio: 0.92 # KV-cache spill threshold -llm_throughput_baseline_tokens_per_min: 50 # baseline for degradation alert - -# Grafana dashboard -llm_grafana_dashboard_uid: llama-swap-monitor -llm_grafana_dashboard_title: "llama-swap GPU/LLM Monitoring" -llm_grafana_dashboard_tags: - - llm - - llama-swap - - gpu-monitoring - - ciro-luciotta -llm_grafana_dashboard_refresh: "30s" -llm_grafana_dashboard_time_from: "now-24h" - -# Prometheus scrape job -llm_prometheus_scrape_interval: "30s" -llm_prometheus_scrape_timeout: "10s" - # --- llama-swap mode (port 8001) ----------------------------------------------- # Deploy llama-swap — Go-based hot-swap proxy (v250+) for model orchestration. # Replaces router mode entirely: single binary + YAML config.json, no --models-preset INI. @@ -317,26 +285,36 @@ llm_swapmode_models: sleep_idle_seconds: -1 # never idle (always ready for embeddings) load_on_startup: true - # Added t_c5cef2b2 (2026-08-19, War Machine): Qwen3-8B aux model. - # Dense 8B (not MoE). ~4.68GB weights at Q4_K_M. GPU-resident (~5.2GB total - # including KV at 32K ctx). Thinking mode on by default — callers MUST send - # /no_think prefix for latency-sensitive aux tasks. See INI template comments. - # VRAM budget: Qwen3.8-27B (~17,804 MiB @ 65536 ctx) + Qwen3-8B (~5,300 MiB - # @ 32K ctx) = ~23,104 MiB — fits in 24GB (24,576 MiB) with ~1.4GB headroom. - # llama-swap evicts Qwen3.8 when Qwen3-8B is requested (cannot co-reside). - # LRU eviction is the operative strategy; cold load ~5s for 5GB GGUF. + # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking deployment. + # Both variants point to the same GGUF. GPU-resident (~5.2GB each). + # Cannot co-reside with Qwen3.8-27B-Q4_K_M; LRU eviction applies. + # chat_template_file for no_think variant: {{ llm_models_dir }}/templates/qwen3-no-think.jinja - id: Qwen3-8B-Q4_K_M gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf" port: 8106 - n_gpu_layers: -1 # -1 = auto-detect / all layers to GPU (~5GB, fits easily) + n_gpu_layers: 99 # GPU-resident (thinking variant) ctx_size: 32768 batch_size: 4096 ubatch_size: 4096 parallel: 1 cache_type: q4_0 - flash_attn: true + flash_attn: "true" sleep_idle_seconds: 60 # idle after 60s no requests + - id: Qwen3-8B-Q4_K_M-no_think + gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf" + port: 8107 + n_gpu_layers: 99 # GPU-resident (no-think variant) + ctx_size: 32768 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + cache_type: q4_0 + flash_attn: "true" + sleep_idle_seconds: 60 + chat_template_file: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja" + + # llama-swap matrix routing configuration # Each row defines a set of models that can be co-resident and hot-swappable # Syntax: "model1 & model2" = both models in same row (via v250 expression DSL) @@ -356,5 +334,13 @@ llm_swapmode_matrix_rows: - row: row4 expr: "Phi-3.5-mini-instruct-Q8_0 & nomic-embed-text-v1.5" # Mini + embed + # t_c5cef2b2 / t_664289a0 (2026-08-19): Qwen3-8B dual-thinking rows. + # Both Qwen3-8B variants co-reside with nomic-embed but NOT with + # Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction swaps between primary and + # Qwen3-8B when needed. They CAN co-reside with each other (~10.4GB total) + # but NOT simultaneously with Qwen3.8-27B. - row: row5 - expr: "Qwen3-8B-Q4_K_M & nomic-embed-text-v1.5" # Aux 8B + embed + expr: "Qwen3-8B-Q4_K_M & nomic-embed-text-v1.5" # Thinking variant + embed + + - row: row6 + expr: "Qwen3-8B-Q4_K_M-no_think & nomic-embed-text-v1.5" # No-think variant + embed diff --git a/ansible/roles/llm-inference-multimodel/tasks/models.yml b/ansible/roles/llm-inference-multimodel/tasks/models.yml index 300b805..5b159ee 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/models.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/models.yml @@ -74,3 +74,30 @@ loop_control: loop_var: staged_model tags: [models] + +# --- Chat template overrides --------------------------------------------------- +# Deploy per-model chat template files used by llama-server via chat-template-file. +# These are static files dropped into {{ llm_models_dir }}/templates/. +# t_664289a0: qwen3-no-think.jinja — Qwen3 template with enable_thinking=false +# hardcoded. Used by [Qwen3-8B-Q4_K_M-no_think] in the router preset INI and +# the llama-swap config. The companion [Qwen3-8B-Q4_K_M] section uses the GGUF's +# baked-in template (thinking ON by default). +- name: Ensure chat template directory exists at {{ llm_models_dir }}/templates + ansible.builtin.file: + path: "{{ llm_models_dir }}/templates" + state: directory + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0755" + become: true + tags: [models, chat_templates] + +- name: Deploy qwen3-no-think.jinja (thinking=false hard-switch for Qwen3-8B no_think variant) + ansible.builtin.template: + src: qwen3-no-think.jinja.j2 + dest: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + become: true + tags: [models, chat_templates] diff --git a/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml b/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml index 3920d18..5eb4892 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/swapmode.yml @@ -241,12 +241,12 @@ when: llm_swapmode_enabled | default(false) tags: [swapmode_verify] -- name: "[swapmode_verify] Assert all 5 models are discoverable" +- name: "[swapmode_verify] Assert all 7 models are discoverable" ansible.builtin.assert: that: - - llm_swapmode_models_list.json.data | map(attribute='id') | list | length == 5 + - llm_swapmode_models_list.json.data | map(attribute='id') | list | length == 7 fail_msg: >- - Expected 5 models in /v1/models response, got {{ llm_swapmode_models_list.json.data | length }}. + Expected 7 models in /v1/models response, got {{ llm_swapmode_models_list.json.data | length }}. Models: {{ llm_swapmode_models_list.json.data | map(attribute='id') | list }} when: llm_swapmode_enabled | default(false) tags: [swapmode_verify] @@ -300,5 +300,6 @@ ✓ GATE 4: VRAM guard ({{ llm_swapmode_vram_used.stdout }} MiB < {{ llm_swapmode_vram_max_mib }} MiB) PASS llama-swap service is ready at http://{{ llm_swapmode_bind_address }}:{{ llm_swapmode_port }}/ + NOTE: 7 models registered (5 original + Qwen3-8B-Q4_K_M + Qwen3-8B-Q4_K_M-no_think). when: llm_swapmode_enabled | default(false) tags: [swapmode_verify] diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 index 777ec8d..9c7f019 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -208,45 +208,55 @@ rope-freq-scale = 0.75 load-on-startup = true sleep-idle-seconds = -1 -; --- Aux model: Qwen3-8B-Q4_K_M ----------------------------------------------- -; Added t_c5cef2b2 (2026-08-19, War Machine). -; Qwen3-8B is a dense 8B model (Qwen3 family, Alibaba) for aux tasks: -; intent classification, query rewriting, structured extraction, tool-call -; construction, and draft generation. Chosen over Llama-3.1-8B for superior -; json_schema grammar support and stronger instruction-following. -; -; Architecture: 36 layers, 32 Q heads, 8 KV heads (GQA), 32K native context, -; 131K via YaRN. Q4_K_M GGUF from bartowski/Qwen_Qwen3-8B-GGUF. ~4.68GB weights. -; -; ctx-size=32768: native training context (safe, no YaRN extension needed). -; VRAM at 32K ctx with q4_0 KV: ~4.68GB weights + ~0.5GB KV ≈ 5.2GB total. -; Fits comfortably on RTX 3090 24GB. Co-resident with nomic-embed (~84MB): -; ~5.3GB total — well within 24GB budget even as a warm auxiliary model. -; -; THINKING MODE NOTE (JARVIS clarification, t_c5cef2b2): -; Qwen3-8B has thinking (chain-of-thought) mode ENABLED BY DEFAULT. -; For latency-sensitive aux tasks (routing, rewriting, structured extraction), -; disable at call time — NOT in this deployment config: -; - Prompt prefix: begin the user message with "/no_think" -; - Chat template override: pass enable_thinking=False in the template vars -; (llama.cpp Jinja2 template: {% if enable_thinking is false %} ... ) -; Do NOT hardcode thinking=false here — this is a call-site concern. -; Thinking mode IS appropriate for complex multi-step tool plans and long-context -; summarization; leave that decision to the caller. -; -; Source: bartowski/Qwen_Qwen3-8B-GGUF (public, no auth). Stored locally as -; Qwen3-8B-Q4_K_M.gguf (canonical name, trimmed from HF's Qwen_Qwen3-8B prefix). -; n-gpu-layers=99: GPU (all layers). At ~5GB, fully GPU-resident with headroom. -; flash-attn=true: Qwen3-8B uses standard attention; flash-attn is safe. -; sleep-idle-seconds=60: evict after 60s idle to free GPU VRAM for Qwen3.8-27B. +; --- Auxiliary model: Qwen3-8B-Q4_K_M (thinking variant) ---------------------- +; GGUF: bartowski/Qwen_Qwen3-8B-GGUF (5,027,784,224 bytes) +; Thinking mode: ON BY DEFAULT (Qwen3 baked-in template, no override). +; - Use for complex aux tasks: long-context summarization, multi-step tool +; planning, structured extraction requiring CoT. +; - To suppress thinking at request time, send /no_think prefix in the prompt +; OR route to [Qwen3-8B-Q4_K_M-no_think] section below. +; n-gpu-layers=99: GPU-resident (~5.2GB VRAM). Cannot co-reside with +; Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction handles swapping. +; ctx-size=32768: 32K context, q4_0 KV cache. +; flash-attn=true: required for Qwen3 architecture at this context size. +; sleep-idle-seconds=60: evict after 60s idle (free VRAM for primary model). +; Added: t_c5cef2b2 (2026-08-19, War Machine) — initial Qwen3-8B deployment. [Qwen3-8B-Q4_K_M] -model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf -n-gpu-layers = 99 -ctx-size = 32768 -flash-attn = true -cache-type-k = {{ llm_router_cache_type_k }} -cache-type-v = {{ llm_router_cache_type_v }} -batch-size = {{ llm_router_batch_size }} -ubatch-size = {{ llm_router_ubatch_size }} -parallel = {{ llm_router_parallel }} +model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf +n-gpu-layers = 99 +ctx-size = 32768 +flash-attn = true +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} +sleep-idle-seconds = 60 +; --- Auxiliary model: Qwen3-8B-Q4_K_M (no-think variant) ---------------------- +; Secondary section serving the SAME GGUF with enable_thinking=False via an +; overridden Jinja2 template. This gives low-latency, non-reasoning inference +; for latency-sensitive aux tasks (intent classification, query rewriting, +; structured extraction, tool-call construction, draft generation). +; +; Both sections point to the same .gguf file — llama-server spawns independent +; child processes. VRAM cost: ~5.2GB per instance (~10.4GB total). Fits on +; RTX 3090 24GB alongside nomic-embed (84MB). +; +; Thinking variant (Qwen3-8B-Q4_K_M) remains available for complex tasks that +; benefit from CoT (long-context summarization, multi-step tool planning). +; +; Cannot co-reside with Qwen3.8-27B-Q4_K_M (17.8GB); LRU eviction applies. +; Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment. +[Qwen3-8B-Q4_K_M-no_think] +model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf +n-gpu-layers = 99 +ctx-size = 32768 +flash-attn = true +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} +chat-template-file = {{ llm_models_dir }}/templates/qwen3-no-think.jinja +sleep-idle-seconds = 60 \ No newline at end of file diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 index 86d0d7b..e28a6c0 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-swap-config.yaml.j2 @@ -39,6 +39,7 @@ models: --parallel {{ model.parallel }} {% if model.cache_type is defined %}--cache-type-k {{ model.cache_type }} --cache-type-v {{ model.cache_type }}{% endif %} {% if model.flash_attn is defined %}--flash-attn {{ model.flash_attn }}{% endif %} + {% if model.chat_template_file is defined %}--chat-template-file {{ model.chat_template_file }}{% endif %} {% if model.sleep_idle_seconds is defined %}--sleep-idle-seconds {{ model.sleep_idle_seconds }}{% endif %} {% if model.load_on_startup is defined and model.load_on_startup %}--load-on-startup{% endif %} --host 127.0.0.1 diff --git a/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 b/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 new file mode 100644 index 0000000..0206423 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 @@ -0,0 +1,103 @@ +{# + FILE: roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2 + DESCRIPTION: Qwen3 chat template with enable_thinking unconditionally false. + + This is a chat-template FILE deployed to {{ llm_models_dir }}/templates/qwen3-no-think.jinja + on astro-orbiter and referenced via chat-template-file in the INI preset for + [Qwen3-8B-Q4_K_M-no_think]. The [Qwen3-8B-Q4_K_M] section uses the model's + baked-in default template (thinking ON by default). + + Mechanism: at the add_generation_prompt step, instead of checking + "enable_thinking is defined and enable_thinking is false" + we UNCONDITIONALLY emit the empty prefix that suppresses CoT. + This is the hard-switch documented in the Qwen3 template spec and confirmed in + https://huggingface.co/blog/qwen-3-chat-template-deep-dive (section 1). + + All other logic is identical to /opt/llama.cpp/models/templates/Qwen-Qwen3-0.6B.jinja + (the shipped template for Qwen3). Only the final add_generation_prompt block differs. + + Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment. +#} +{%- if tools %} + {{- '<|im_start|>system\n' }} + {%- if messages[0].role == 'system' %} + {{- messages[0].content + '\n\n' }} + {%- endif %} + {{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n" }} +{%- else %} + {%- if messages[0].role == 'system' %} + {{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('') and message.content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} +{%- endfor %} +{%- for message in messages %} + {%- if (message.role == "user") or (message.role == "system" and not loop.first) %} + {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set content = message.content %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is defined and message.reasoning_content is not none %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in message.content %} + {%- set content = message.content.split('')[-1].lstrip('\n') %} + {%- set reasoning_content = message.content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- if loop.index0 > ns.last_query_index %} + {%- if loop.last or (not loop.last and reasoning_content) %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content.strip('\n') + '\n\n\n' + content.lstrip('\n') }} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- else %} + {{- '<|im_start|>' + message.role + '\n' + content }} + {%- endif %} + {%- if message.tool_calls %} + {%- for tool_call in message.tool_calls %} + {%- if (loop.first and content) or (not loop.first) %} + {{- '\n' }} + {%- endif %} + {%- if tool_call.function %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {{- '\n{"name": "' }} + {{- tool_call.name }} + {{- '", "arguments": ' }} + {%- if tool_call.arguments is string %} + {{- tool_call.arguments }} + {%- else %} + {{- tool_call.arguments | tojson }} + {%- endif %} + {{- '}\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- message.content }} + {{- '\n' }} + {%- if loop.last or (messages[loop.index0 + 1].role != "tool") %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {{- '\n\n\n\n' }} +{%- endif %}