Merge origin/main: sync Qwen3-8B no_think variant to Ansible repo (t_36e8ba68)
Resolved conflicts between localbafd76a(t_c5cef2b2, Qwen3-8B single-variant) and origin/main5cf4468(t_664289a0, Qwen3-8B dual-thinking deployment). Conflict resolution strategy: took origin/main version throughout — it is the authoritative result from t_664289a0 which deployed the live no_think config to astro-orbiter and already reflects the correct production state. Changes incorporated from origin/main: - defaults/main.yml: Qwen3-8B-Q4_K_M-no_think model entry (port 8107, n_gpu_layers=99, chat_template_file), row6 matrix entry, dual-thinking comment block. - templates/llama-server-router-preset.ini.j2: [Qwen3-8B-Q4_K_M] with sleep-idle-seconds=60 plus new [Qwen3-8B-Q4_K_M-no_think] section with chat-template-file directive. - templates/qwen3-no-think.jinja.j2: new file — Qwen3 template with enable_thinking=false. - tasks/models.yml: template deploy task for qwen3-no-think.jinja (chat_templates tag). - tasks/swapmode.yml: GATE 2 assert updated to 7 models. - templates/llama-swap-config.yaml.j2: chat_template_file flag support. Also pulled in monitoring defaults (llm_monitoring_enabled, VRAM exporter settings, Grafana dashboard vars, Prometheus scrape config) from origin/main monitoring branch.
This commit is contained in:
Hermes Agent service account
@@ -208,45 +208,55 @@ rope-freq-scale = 0.75
|
||||
load-on-startup = true
|
||||
sleep-idle-seconds = -1
|
||||
|
||||
; --- Aux model: Qwen3-8B-Q4_K_M -----------------------------------------------
|
||||
; Added t_c5cef2b2 (2026-08-19, War Machine).
|
||||
; Qwen3-8B is a dense 8B model (Qwen3 family, Alibaba) for aux tasks:
|
||||
; intent classification, query rewriting, structured extraction, tool-call
|
||||
; construction, and draft generation. Chosen over Llama-3.1-8B for superior
|
||||
; json_schema grammar support and stronger instruction-following.
|
||||
;
|
||||
; Architecture: 36 layers, 32 Q heads, 8 KV heads (GQA), 32K native context,
|
||||
; 131K via YaRN. Q4_K_M GGUF from bartowski/Qwen_Qwen3-8B-GGUF. ~4.68GB weights.
|
||||
;
|
||||
; ctx-size=32768: native training context (safe, no YaRN extension needed).
|
||||
; VRAM at 32K ctx with q4_0 KV: ~4.68GB weights + ~0.5GB KV ≈ 5.2GB total.
|
||||
; Fits comfortably on RTX 3090 24GB. Co-resident with nomic-embed (~84MB):
|
||||
; ~5.3GB total — well within 24GB budget even as a warm auxiliary model.
|
||||
;
|
||||
; THINKING MODE NOTE (JARVIS clarification, t_c5cef2b2):
|
||||
; Qwen3-8B has thinking (chain-of-thought) mode ENABLED BY DEFAULT.
|
||||
; For latency-sensitive aux tasks (routing, rewriting, structured extraction),
|
||||
; disable at call time — NOT in this deployment config:
|
||||
; - Prompt prefix: begin the user message with "/no_think"
|
||||
; - Chat template override: pass enable_thinking=False in the template vars
|
||||
; (llama.cpp Jinja2 template: {% if enable_thinking is false %} ... )
|
||||
; Do NOT hardcode thinking=false here — this is a call-site concern.
|
||||
; Thinking mode IS appropriate for complex multi-step tool plans and long-context
|
||||
; summarization; leave that decision to the caller.
|
||||
;
|
||||
; Source: bartowski/Qwen_Qwen3-8B-GGUF (public, no auth). Stored locally as
|
||||
; Qwen3-8B-Q4_K_M.gguf (canonical name, trimmed from HF's Qwen_Qwen3-8B prefix).
|
||||
; n-gpu-layers=99: GPU (all layers). At ~5GB, fully GPU-resident with headroom.
|
||||
; flash-attn=true: Qwen3-8B uses standard attention; flash-attn is safe.
|
||||
; sleep-idle-seconds=60: evict after 60s idle to free GPU VRAM for Qwen3.8-27B.
|
||||
; --- Auxiliary model: Qwen3-8B-Q4_K_M (thinking variant) ----------------------
|
||||
; GGUF: bartowski/Qwen_Qwen3-8B-GGUF (5,027,784,224 bytes)
|
||||
; Thinking mode: ON BY DEFAULT (Qwen3 baked-in template, no override).
|
||||
; - Use for complex aux tasks: long-context summarization, multi-step tool
|
||||
; planning, structured extraction requiring CoT.
|
||||
; - To suppress thinking at request time, send /no_think prefix in the prompt
|
||||
; OR route to [Qwen3-8B-Q4_K_M-no_think] section below.
|
||||
; n-gpu-layers=99: GPU-resident (~5.2GB VRAM). Cannot co-reside with
|
||||
; Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction handles swapping.
|
||||
; ctx-size=32768: 32K context, q4_0 KV cache.
|
||||
; flash-attn=true: required for Qwen3 architecture at this context size.
|
||||
; sleep-idle-seconds=60: evict after 60s idle (free VRAM for primary model).
|
||||
; Added: t_c5cef2b2 (2026-08-19, War Machine) — initial Qwen3-8B deployment.
|
||||
[Qwen3-8B-Q4_K_M]
|
||||
model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
|
||||
n-gpu-layers = 99
|
||||
ctx-size = 32768
|
||||
flash-attn = true
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
|
||||
n-gpu-layers = 99
|
||||
ctx-size = 32768
|
||||
flash-attn = true
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
sleep-idle-seconds = 60
|
||||
|
||||
; --- Auxiliary model: Qwen3-8B-Q4_K_M (no-think variant) ----------------------
|
||||
; Secondary section serving the SAME GGUF with enable_thinking=False via an
|
||||
; overridden Jinja2 template. This gives low-latency, non-reasoning inference
|
||||
; for latency-sensitive aux tasks (intent classification, query rewriting,
|
||||
; structured extraction, tool-call construction, draft generation).
|
||||
;
|
||||
; Both sections point to the same .gguf file — llama-server spawns independent
|
||||
; child processes. VRAM cost: ~5.2GB per instance (~10.4GB total). Fits on
|
||||
; RTX 3090 24GB alongside nomic-embed (84MB).
|
||||
;
|
||||
; Thinking variant (Qwen3-8B-Q4_K_M) remains available for complex tasks that
|
||||
; benefit from CoT (long-context summarization, multi-step tool planning).
|
||||
;
|
||||
; Cannot co-reside with Qwen3.8-27B-Q4_K_M (17.8GB); LRU eviction applies.
|
||||
; Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment.
|
||||
[Qwen3-8B-Q4_K_M-no_think]
|
||||
model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
|
||||
n-gpu-layers = 99
|
||||
ctx-size = 32768
|
||||
flash-attn = true
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
chat-template-file = {{ llm_models_dir }}/templates/qwen3-no-think.jinja
|
||||
sleep-idle-seconds = 60
|
||||
@@ -39,6 +39,7 @@ models:
|
||||
--parallel {{ model.parallel }}
|
||||
{% if model.cache_type is defined %}--cache-type-k {{ model.cache_type }} --cache-type-v {{ model.cache_type }}{% endif %}
|
||||
{% if model.flash_attn is defined %}--flash-attn {{ model.flash_attn }}{% endif %}
|
||||
{% if model.chat_template_file is defined %}--chat-template-file {{ model.chat_template_file }}{% endif %}
|
||||
{% if model.sleep_idle_seconds is defined %}--sleep-idle-seconds {{ model.sleep_idle_seconds }}{% endif %}
|
||||
{% if model.load_on_startup is defined and model.load_on_startup %}--load-on-startup{% endif %}
|
||||
--host 127.0.0.1
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
{#
|
||||
FILE: roles/llm-inference-multimodel/templates/qwen3-no-think.jinja.j2
|
||||
DESCRIPTION: Qwen3 chat template with enable_thinking unconditionally false.
|
||||
|
||||
This is a chat-template FILE deployed to {{ llm_models_dir }}/templates/qwen3-no-think.jinja
|
||||
on astro-orbiter and referenced via chat-template-file in the INI preset for
|
||||
[Qwen3-8B-Q4_K_M-no_think]. The [Qwen3-8B-Q4_K_M] section uses the model's
|
||||
baked-in default template (thinking ON by default).
|
||||
|
||||
Mechanism: at the add_generation_prompt step, instead of checking
|
||||
"enable_thinking is defined and enable_thinking is false"
|
||||
we UNCONDITIONALLY emit the empty <think></think> prefix that suppresses CoT.
|
||||
This is the hard-switch documented in the Qwen3 template spec and confirmed in
|
||||
https://huggingface.co/blog/qwen-3-chat-template-deep-dive (section 1).
|
||||
|
||||
All other logic is identical to /opt/llama.cpp/models/templates/Qwen-Qwen3-0.6B.jinja
|
||||
(the shipped template for Qwen3). Only the final add_generation_prompt block differs.
|
||||
|
||||
Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment.
|
||||
#}
|
||||
{%- if tools %}
|
||||
{{- '<|im_start|>system\n' }}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- messages[0].content + '\n\n' }}
|
||||
{%- endif %}
|
||||
{{- "# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within <tools></tools> XML tags:\n<tools>" }}
|
||||
{%- for tool in tools %}
|
||||
{{- "\n" }}
|
||||
{{- tool | tojson }}
|
||||
{%- endfor %}
|
||||
{{- "\n</tools>\n\nFor each function call, return a json object with function name and arguments within <tool_call></tool_call> XML tags:\n<tool_call>\n{\"name\": <function-name>, \"arguments\": <args-json-object>}\n</tool_call><|im_end|>\n" }}
|
||||
{%- else %}
|
||||
{%- if messages[0].role == 'system' %}
|
||||
{{- '<|im_start|>system\n' + messages[0].content + '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
|
||||
{%- for message in messages[::-1] %}
|
||||
{%- set index = (messages|length - 1) - loop.index0 %}
|
||||
{%- if ns.multi_step_tool and message.role == "user" and not(message.content.startswith('<tool_response>') and message.content.endswith('</tool_response>')) %}
|
||||
{%- set ns.multi_step_tool = false %}
|
||||
{%- set ns.last_query_index = index %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- for message in messages %}
|
||||
{%- if (message.role == "user") or (message.role == "system" and not loop.first) %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
|
||||
{%- elif message.role == "assistant" %}
|
||||
{%- set content = message.content %}
|
||||
{%- set reasoning_content = '' %}
|
||||
{%- if message.reasoning_content is defined and message.reasoning_content is not none %}
|
||||
{%- set reasoning_content = message.reasoning_content %}
|
||||
{%- else %}
|
||||
{%- if '</think>' in message.content %}
|
||||
{%- set content = message.content.split('</think>')[-1].lstrip('\n') %}
|
||||
{%- set reasoning_content = message.content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- if loop.index0 > ns.last_query_index %}
|
||||
{%- if loop.last or (not loop.last and reasoning_content) %}
|
||||
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content.strip('\n') + '\n</think>\n\n' + content.lstrip('\n') }}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- else %}
|
||||
{{- '<|im_start|>' + message.role + '\n' + content }}
|
||||
{%- endif %}
|
||||
{%- if message.tool_calls %}
|
||||
{%- for tool_call in message.tool_calls %}
|
||||
{%- if (loop.first and content) or (not loop.first) %}
|
||||
{{- '\n' }}
|
||||
{%- endif %}
|
||||
{%- if tool_call.function %}
|
||||
{%- set tool_call = tool_call.function %}
|
||||
{%- endif %}
|
||||
{{- '<tool_call>\n{"name": "' }}
|
||||
{{- tool_call.name }}
|
||||
{{- '", "arguments": ' }}
|
||||
{%- if tool_call.arguments is string %}
|
||||
{{- tool_call.arguments }}
|
||||
{%- else %}
|
||||
{{- tool_call.arguments | tojson }}
|
||||
{%- endif %}
|
||||
{{- '}\n</tool_call>' }}
|
||||
{%- endfor %}
|
||||
{%- endif %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- elif message.role == "tool" %}
|
||||
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
||||
{{- '<|im_start|>user' }}
|
||||
{%- endif %}
|
||||
{{- '\n<tool_response>\n' }}
|
||||
{{- message.content }}
|
||||
{{- '\n</tool_response>' }}
|
||||
{%- if loop.last or (messages[loop.index0 + 1].role != "tool") %}
|
||||
{{- '<|im_end|>\n' }}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{%- if add_generation_prompt %}
|
||||
{{- '<|im_start|>assistant\n' }}
|
||||
{{- '<think>\n\n</think>\n\n' }}
|
||||
{%- endif %}
|
||||
Reference in New Issue
Block a user