Merge origin/main: sync Qwen3-8B no_think variant to Ansible repo (t_36e8ba68)
Resolved conflicts between localbafd76a(t_c5cef2b2, Qwen3-8B single-variant) and origin/main5cf4468(t_664289a0, Qwen3-8B dual-thinking deployment). Conflict resolution strategy: took origin/main version throughout — it is the authoritative result from t_664289a0 which deployed the live no_think config to astro-orbiter and already reflects the correct production state. Changes incorporated from origin/main: - defaults/main.yml: Qwen3-8B-Q4_K_M-no_think model entry (port 8107, n_gpu_layers=99, chat_template_file), row6 matrix entry, dual-thinking comment block. - templates/llama-server-router-preset.ini.j2: [Qwen3-8B-Q4_K_M] with sleep-idle-seconds=60 plus new [Qwen3-8B-Q4_K_M-no_think] section with chat-template-file directive. - templates/qwen3-no-think.jinja.j2: new file — Qwen3 template with enable_thinking=false. - tasks/models.yml: template deploy task for qwen3-no-think.jinja (chat_templates tag). - tasks/swapmode.yml: GATE 2 assert updated to 7 models. - templates/llama-swap-config.yaml.j2: chat_template_file flag support. Also pulled in monitoring defaults (llm_monitoring_enabled, VRAM exporter settings, Grafana dashboard vars, Prometheus scrape config) from origin/main monitoring branch.
This commit is contained in:
Hermes Agent service account
@@ -208,45 +208,55 @@ rope-freq-scale = 0.75
|
||||
load-on-startup = true
|
||||
sleep-idle-seconds = -1
|
||||
|
||||
; --- Aux model: Qwen3-8B-Q4_K_M -----------------------------------------------
|
||||
; Added t_c5cef2b2 (2026-08-19, War Machine).
|
||||
; Qwen3-8B is a dense 8B model (Qwen3 family, Alibaba) for aux tasks:
|
||||
; intent classification, query rewriting, structured extraction, tool-call
|
||||
; construction, and draft generation. Chosen over Llama-3.1-8B for superior
|
||||
; json_schema grammar support and stronger instruction-following.
|
||||
;
|
||||
; Architecture: 36 layers, 32 Q heads, 8 KV heads (GQA), 32K native context,
|
||||
; 131K via YaRN. Q4_K_M GGUF from bartowski/Qwen_Qwen3-8B-GGUF. ~4.68GB weights.
|
||||
;
|
||||
; ctx-size=32768: native training context (safe, no YaRN extension needed).
|
||||
; VRAM at 32K ctx with q4_0 KV: ~4.68GB weights + ~0.5GB KV ≈ 5.2GB total.
|
||||
; Fits comfortably on RTX 3090 24GB. Co-resident with nomic-embed (~84MB):
|
||||
; ~5.3GB total — well within 24GB budget even as a warm auxiliary model.
|
||||
;
|
||||
; THINKING MODE NOTE (JARVIS clarification, t_c5cef2b2):
|
||||
; Qwen3-8B has thinking (chain-of-thought) mode ENABLED BY DEFAULT.
|
||||
; For latency-sensitive aux tasks (routing, rewriting, structured extraction),
|
||||
; disable at call time — NOT in this deployment config:
|
||||
; - Prompt prefix: begin the user message with "/no_think"
|
||||
; - Chat template override: pass enable_thinking=False in the template vars
|
||||
; (llama.cpp Jinja2 template: {% if enable_thinking is false %} ... )
|
||||
; Do NOT hardcode thinking=false here — this is a call-site concern.
|
||||
; Thinking mode IS appropriate for complex multi-step tool plans and long-context
|
||||
; summarization; leave that decision to the caller.
|
||||
;
|
||||
; Source: bartowski/Qwen_Qwen3-8B-GGUF (public, no auth). Stored locally as
|
||||
; Qwen3-8B-Q4_K_M.gguf (canonical name, trimmed from HF's Qwen_Qwen3-8B prefix).
|
||||
; n-gpu-layers=99: GPU (all layers). At ~5GB, fully GPU-resident with headroom.
|
||||
; flash-attn=true: Qwen3-8B uses standard attention; flash-attn is safe.
|
||||
; sleep-idle-seconds=60: evict after 60s idle to free GPU VRAM for Qwen3.8-27B.
|
||||
; --- Auxiliary model: Qwen3-8B-Q4_K_M (thinking variant) ----------------------
|
||||
; GGUF: bartowski/Qwen_Qwen3-8B-GGUF (5,027,784,224 bytes)
|
||||
; Thinking mode: ON BY DEFAULT (Qwen3 baked-in template, no override).
|
||||
; - Use for complex aux tasks: long-context summarization, multi-step tool
|
||||
; planning, structured extraction requiring CoT.
|
||||
; - To suppress thinking at request time, send /no_think prefix in the prompt
|
||||
; OR route to [Qwen3-8B-Q4_K_M-no_think] section below.
|
||||
; n-gpu-layers=99: GPU-resident (~5.2GB VRAM). Cannot co-reside with
|
||||
; Qwen3.8-27B-Q4_K_M (17.8GB). LRU eviction handles swapping.
|
||||
; ctx-size=32768: 32K context, q4_0 KV cache.
|
||||
; flash-attn=true: required for Qwen3 architecture at this context size.
|
||||
; sleep-idle-seconds=60: evict after 60s idle (free VRAM for primary model).
|
||||
; Added: t_c5cef2b2 (2026-08-19, War Machine) — initial Qwen3-8B deployment.
|
||||
[Qwen3-8B-Q4_K_M]
|
||||
model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
|
||||
n-gpu-layers = 99
|
||||
ctx-size = 32768
|
||||
flash-attn = true
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
|
||||
n-gpu-layers = 99
|
||||
ctx-size = 32768
|
||||
flash-attn = true
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
sleep-idle-seconds = 60
|
||||
|
||||
; --- Auxiliary model: Qwen3-8B-Q4_K_M (no-think variant) ----------------------
|
||||
; Secondary section serving the SAME GGUF with enable_thinking=False via an
|
||||
; overridden Jinja2 template. This gives low-latency, non-reasoning inference
|
||||
; for latency-sensitive aux tasks (intent classification, query rewriting,
|
||||
; structured extraction, tool-call construction, draft generation).
|
||||
;
|
||||
; Both sections point to the same .gguf file — llama-server spawns independent
|
||||
; child processes. VRAM cost: ~5.2GB per instance (~10.4GB total). Fits on
|
||||
; RTX 3090 24GB alongside nomic-embed (84MB).
|
||||
;
|
||||
; Thinking variant (Qwen3-8B-Q4_K_M) remains available for complex tasks that
|
||||
; benefit from CoT (long-context summarization, multi-step tool planning).
|
||||
;
|
||||
; Cannot co-reside with Qwen3.8-27B-Q4_K_M (17.8GB); LRU eviction applies.
|
||||
; Added: t_664289a0 (2026-08-19, War Machine) — dual thinking deployment.
|
||||
[Qwen3-8B-Q4_K_M-no_think]
|
||||
model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf
|
||||
n-gpu-layers = 99
|
||||
ctx-size = 32768
|
||||
flash-attn = true
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
chat-template-file = {{ llm_models_dir }}/templates/qwen3-no-think.jinja
|
||||
sleep-idle-seconds = 60
|
||||
Reference in New Issue
Block a user