diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index 91a2d7f..b50a34d 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -34,26 +34,27 @@ common_root_lv: ubuntu-lv # (t_33acbb2e) so the router can keep more than one GGUF resident on-demand # and LRU-evict when needed. # -# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad, updated t_441470b9): -# With models-max=4 and all 5 GGUFs registered, worst case is all 5 loaded simultaneously: -# Qwen3.8-27B Q4_K_M: ~23.3GB (weights ~17.1GB + KV ~6.2GB @ 128K ctx, q4_0) ← UPDATED +# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad, updated t_441470b9, updated t_c5cef2b2): +# With models-max=4 and all 6 GGUFs registered, worst case is all 6 loaded simultaneously: +# Qwen3.8-27B Q4_K_M: ~20.0GB (weights ~17.1GB + KV ~2.9GB @ 65536 ctx, q4_0) ← CORRECTED (ctx rolled back from 128K to 65536, t_c9fed26c 2026-08-18) # Phi-3.5-mini-instruct Q8_0: ~4.3GB (weights ~3.8GB + KV ~0.5GB @ 32K ctx) # Meta-Llama-3.1-8B Q4_K_M: ~5.6GB (weights ~4.6GB + KV ~0.2GB @ 8K ctx) # Qwen2.5-Coder-14B Q4_K_M: ~9.0GB (weights ~8.4GB + KV ~0.6GB @ 16K ctx) # nomic-embed-text-v1.5 Q4_K_M: ~0.09GB (~84MB, embedding only — no KV cache) -# Total worst-case: ~42.3GB >> 24GB RTX 3090 +# Qwen3-8B Q4_K_M: ~5.5GB (weights ~4.68GB + KV ~0.5GB @ 32K ctx, q4_0) +# Total worst-case: ~44.5GB >> 24GB RTX 3090 # # OOM RISK: Full co-residency is impossible on 24GB. LRU eviction prevents this -# in practice: models-max=4 means the router can REGISTER 5 models but only keeps +# in practice: models-max=4 means the router can REGISTER 6 models but only keeps # up to 4 LOADED simultaneously — the router will evict the LRU model when a new # one is needed. nomic-embed-text-v1.5 is pinned via sleep-idle-seconds=-1 and # load-on-startup=true but it uses only ~84MB, so it never meaningfully changes # the budget. In single-user homelab operation, only one generative model is active # at a time alongside the always-resident embedding model. -# Qwen3.8-27B alone uses ~23.1GB (weights+KV); co-residency with Coder (~9GB) = ~32GB > 24GB. -# LRU eviction handles this automatically — the router evicts the idle model before -# loading the new one. Ryan should be aware this means model-switching always incurs -# a ~30-60s cold-load latency when switching between Qwen3.8-27B and any other model. +# Qwen3.8-27B alone uses ~17,804 MiB (weights+KV @ 65536 ctx); co-residency +# with Coder (~9GB) = ~27GB > 24GB. LRU eviction handles this automatically. +# Ryan should be aware this means model-switching always incurs a ~30-60s +# cold-load latency when switching between Qwen3.8-27B and any other model. # Proceeding to models-max=4 as instructed; flagged for Ryan's attention. # Router --models-max override for astro-orbiter. # UPDATED (t_f5f7e9ad, 2026-08-16): Set to 2 because Qwen3.8-27B-Q4_K_M @@ -67,13 +68,16 @@ common_root_lv: ubuntu-lv # This avoids an LRU eviction race with nomic-embed at startup. # UPDATED (t_72646029, 2026-08-17): CPU offload for Coder + Llama changes the # constraint. Coder and Llama now use CPU inference (n-gpu-layers=0). GPU-resident -# VRAM: Qwen3.8 (~20,302 MiB at 128K ctx) + nomic-embed (558 MiB, pinned) plus the -# CUDA-context buffers llama.cpp 6ea215d allocates for the CPU models (~1.4-1.7GB -# each) = ~24,004 MiB steady-state, below the 24,576 MiB physical limit. +# VRAM: Qwen3.8 (~17,804 MiB at 65536 ctx) + nomic-embed (558 MiB, pinned) plus +# the CUDA-context buffers llama.cpp 6ea215d allocates for the CPU models (~1.4-1.7GB +# each) = ~20,004 MiB steady-state, below the 24,576 MiB physical limit. +# CORRECTED (t_c5cef2b2, 2026-08-19): ctx-size was rolled back from 131072 to 65536 +# (t_c9fed26c 2026-08-18). Qwen3.8 VRAM at 65536: 17,804 MiB (not 20,302 MiB). # models-max raised to 4: nomic (slot 1, pinned) + Qwen3.8 (slot 2, GPU) + -# Llama (slot 3, CPU) + Coder (slot 4, CPU). Phi (GPU, ~8.3GB) can still be -# requested but evicts Qwen3.8 due to VRAM constraint. models-max=4 -# is required so CPU-offloaded models count as loaded without evicting Qwen3.8. +# Llama (slot 3, CPU) + Coder (slot 4, CPU). Phi (GPU, ~8.3GB) and new +# Qwen3-8B (GPU, ~5.5GB) can also be requested but evict Qwen3.8 due to VRAM. +# models-max=4 is required so CPU-offloaded models count as loaded without +# evicting Qwen3.8. llm_router_models_max: 4 llm_staged_models: @@ -93,4 +97,15 @@ llm_staged_models: url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf" size_bytes: 84106624 source_repo: "nomic-ai/nomic-embed-text-v1.5-GGUF" + # Added t_c5cef2b2 (2026-08-19, War Machine): Qwen3-8B dense 8B model for + # aux tasks (routing, rewriting, structured extraction, tool-call construction). + # Source: bartowski/Qwen_Qwen3-8B-GGUF (public, no auth). HF filename is + # Qwen_Qwen3-8B-Q4_K_M.gguf; stored locally as Qwen3-8B-Q4_K_M.gguf. + # Exact size verified from HF manifest (content-length): 5,027,784,224 bytes. + # VRAM: ~4.68GB weights + ~0.5GB KV @ 32K ctx (q4_0) ≈ 5.2GB total. + # Thinking mode ON by default; use /no_think for latency-sensitive aux tasks. + - filename: "Qwen3-8B-Q4_K_M.gguf" + url: "https://huggingface.co/bartowski/Qwen_Qwen3-8B-GGUF/resolve/main/Qwen_Qwen3-8B-Q4_K_M.gguf" + size_bytes: 5027784224 + source_repo: "bartowski/Qwen_Qwen3-8B-GGUF" diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 2520b15..cd7b4df 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -317,6 +317,26 @@ llm_swapmode_models: sleep_idle_seconds: -1 # never idle (always ready for embeddings) load_on_startup: true + # Added t_c5cef2b2 (2026-08-19, War Machine): Qwen3-8B aux model. + # Dense 8B (not MoE). ~4.68GB weights at Q4_K_M. GPU-resident (~5.2GB total + # including KV at 32K ctx). Thinking mode on by default — callers MUST send + # /no_think prefix for latency-sensitive aux tasks. See INI template comments. + # VRAM budget: Qwen3.8-27B (~17,804 MiB @ 65536 ctx) + Qwen3-8B (~5,300 MiB + # @ 32K ctx) = ~23,104 MiB — fits in 24GB (24,576 MiB) with ~1.4GB headroom. + # llama-swap evicts Qwen3.8 when Qwen3-8B is requested (cannot co-reside). + # LRU eviction is the operative strategy; cold load ~5s for 5GB GGUF. + - id: Qwen3-8B-Q4_K_M + gguf_path: "{{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf" + port: 8106 + n_gpu_layers: -1 # -1 = auto-detect / all layers to GPU (~5GB, fits easily) + ctx_size: 32768 + batch_size: 4096 + ubatch_size: 4096 + parallel: 1 + cache_type: q4_0 + flash_attn: true + sleep_idle_seconds: 60 # idle after 60s no requests + # llama-swap matrix routing configuration # Each row defines a set of models that can be co-resident and hot-swappable # Syntax: "model1 & model2" = both models in same row (via v250 expression DSL) @@ -335,3 +355,6 @@ llm_swapmode_matrix_rows: - row: row4 expr: "Phi-3.5-mini-instruct-Q8_0 & nomic-embed-text-v1.5" # Mini + embed + + - row: row5 + expr: "Qwen3-8B-Q4_K_M & nomic-embed-text-v1.5" # Aux 8B + embed diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 index 5ca4d35..777ec8d 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -207,3 +207,46 @@ rope-scaling = yarn rope-freq-scale = 0.75 load-on-startup = true sleep-idle-seconds = -1 + +; --- Aux model: Qwen3-8B-Q4_K_M ----------------------------------------------- +; Added t_c5cef2b2 (2026-08-19, War Machine). +; Qwen3-8B is a dense 8B model (Qwen3 family, Alibaba) for aux tasks: +; intent classification, query rewriting, structured extraction, tool-call +; construction, and draft generation. Chosen over Llama-3.1-8B for superior +; json_schema grammar support and stronger instruction-following. +; +; Architecture: 36 layers, 32 Q heads, 8 KV heads (GQA), 32K native context, +; 131K via YaRN. Q4_K_M GGUF from bartowski/Qwen_Qwen3-8B-GGUF. ~4.68GB weights. +; +; ctx-size=32768: native training context (safe, no YaRN extension needed). +; VRAM at 32K ctx with q4_0 KV: ~4.68GB weights + ~0.5GB KV ≈ 5.2GB total. +; Fits comfortably on RTX 3090 24GB. Co-resident with nomic-embed (~84MB): +; ~5.3GB total — well within 24GB budget even as a warm auxiliary model. +; +; THINKING MODE NOTE (JARVIS clarification, t_c5cef2b2): +; Qwen3-8B has thinking (chain-of-thought) mode ENABLED BY DEFAULT. +; For latency-sensitive aux tasks (routing, rewriting, structured extraction), +; disable at call time — NOT in this deployment config: +; - Prompt prefix: begin the user message with "/no_think" +; - Chat template override: pass enable_thinking=False in the template vars +; (llama.cpp Jinja2 template: {% if enable_thinking is false %} ... ) +; Do NOT hardcode thinking=false here — this is a call-site concern. +; Thinking mode IS appropriate for complex multi-step tool plans and long-context +; summarization; leave that decision to the caller. +; +; Source: bartowski/Qwen_Qwen3-8B-GGUF (public, no auth). Stored locally as +; Qwen3-8B-Q4_K_M.gguf (canonical name, trimmed from HF's Qwen_Qwen3-8B prefix). +; n-gpu-layers=99: GPU (all layers). At ~5GB, fully GPU-resident with headroom. +; flash-attn=true: Qwen3-8B uses standard attention; flash-attn is safe. +; sleep-idle-seconds=60: evict after 60s idle to free GPU VRAM for Qwen3.8-27B. +[Qwen3-8B-Q4_K_M] +model = {{ llm_models_dir }}/Qwen3-8B-Q4_K_M.gguf +n-gpu-layers = 99 +ctx-size = 32768 +flash-attn = true +cache-type-k = {{ llm_router_cache_type_k }} +cache-type-v = {{ llm_router_cache_type_v }} +batch-size = {{ llm_router_batch_size }} +ubatch-size = {{ llm_router_ubatch_size }} +parallel = {{ llm_router_parallel }} +