feat(llm): swap astro-orbiter primary model Qwen3.6 -> Qwen3.8-27B-Q4_K_M
Ryan-directed model swap (kanban t_f5f7e9ad, 2026-08-16). Changes: - Replace [Qwen3.6-35B-A3B-UD-Q4_K_S] with [Qwen3.8-27B-Q4_K_M] in llama-server-router-preset.ini.j2 (production model slot). - Qwen3.8-27B: dense 27B VLM, Apache-2.0, Alibaba Aug 2026. Unsloth Dynamic V3.0 GGUF quantization. Q4_K_M chosen: 17,106,775,008 bytes, 17.1GB. Measured VRAM: 17,068 MiB at ctx=32768 (q4_0 KV cache). - ctx-size set to 32768 (32K) via new variable llm_router_qwen38_ctx_size. Native context is 262K; 32K chosen to maintain eviction headroom on 24GB RTX 3090. - models-max reduced 4 -> 2 in host_vars. Qwen3.8 (17.6GB) + nomic-embed (558MB) exhaust the 24GB card; no auxiliary model can co-reside with Qwen3.8. LRU eviction handles model switching with ~30-60s cold-load latency. - llm_router_expected_model_id updated to Qwen3.8-27B-Q4_K_M. - Qwen3.6 GGUF retained at /opt/models/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf (not deleted — pending stable period and explicit cleanup task). - day2_swap_qwen38.yml playbook added for Ansible idempotent redeployment. Architecture note: Qwen3.8 uses Gated DeltaNet; llama.cpp 6ea215d logs 'fused Gated Delta Net (chunked) not supported, set to disabled'. Inference works correctly on the non-fused fallback. A llama.cpp update may improve throughput on the GDN layers. Smoke test passed: model responded via router endpoint (http://10.1.71.130:8002). VRAM: 17,630 MiB (Qwen3.8) + 5,928 MiB (Llama-8B concurrent) = 23,558 MiB. Also commits accumulated but unpushed changes: - nomic-embed batch-size/rope-scaling fix (t_openviking_embed_batch) - per-model ctx-size day2 playbook (day2_per_model_ctx_size.yml) - llama-server-router.service.j2 minor update
This commit is contained in:
Hermes Agent service account
@@ -62,22 +62,21 @@ llm_allowed_source_cidr: "10.1.70.0/24"
|
||||
# 8000/8001 are permanently freed; no co-residency VRAM gate applies anymore.
|
||||
llm_qwen_service_enabled: true
|
||||
llm_qwen_port: 8002
|
||||
llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf"
|
||||
llm_qwen_model_min_bytes: 19000000000 # guard threshold; complete file ~20GB
|
||||
llm_qwen_model_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf"
|
||||
llm_qwen_model_min_bytes: 17000000000 # guard threshold; complete file ~17.1GB
|
||||
llm_qwen_ctx_size: 65536
|
||||
llm_qwen_parallel: 1
|
||||
llm_qwen_gpu_layers: 99
|
||||
llm_qwen_batch_size: 2048
|
||||
llm_qwen_ubatch_size: 512
|
||||
llm_qwen_batch_size: 4096
|
||||
llm_qwen_ubatch_size: 4096
|
||||
llm_qwen_service_name: llama-server-qwen
|
||||
llm_qwen_model_id: Qwen3.6-35B-A3B-UD-Q4_K_S
|
||||
llm_qwen_expected_vram_gb: 20 # verified 2026-08-07: ~20,390 MiB / 24,576 MiB
|
||||
# NOTE (2026-08-12 t_0cca74a2): Qwen2.5-14B-Instruct-1M was superseded by
|
||||
# Qwen3.6-35B-A3B-UD-Q4_K_S (task t_2ffc0f63, 2026-08-07). Defaults updated
|
||||
# to reflect the current production model. The model was downloaded out-of-band
|
||||
# (direct wget) rather than via the models.yml get_url pattern.
|
||||
# llm_qwen_model_url is intentionally not set — see models.yml WARN task for
|
||||
# the HuggingFace URL if a re-download is ever needed.
|
||||
llm_qwen_model_id: Qwen3.8-27B-Q4_K_M
|
||||
llm_qwen_expected_vram_gb: 17 # Q4_K_M = 17.1GB weights + ~6GB KV @ 65536 ctx = ~23GB max
|
||||
# NOTE (2026-08-16 t_f5f7e9ad): Qwen3.6-35B-A3B-UD-Q4_K_S superseded by
|
||||
# Qwen3.8-27B-Q4_K_M per Ryan's direction. Qwen3.8-27B is a dense 27B VLM
|
||||
# (Apache-2.0, Alibaba, Aug 2026) quantized by Unsloth Dynamic V3.0.
|
||||
# Q4_K_M: 17,106,775,008 bytes. Downloaded out-of-band via wget.
|
||||
# llm_qwen_model_url: https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf
|
||||
|
||||
# --- Staged GGUF models (data-driven, idempotent staging) --------------------
|
||||
# Additional GGUFs to ensure are present in llm_models_dir, alongside the
|
||||
@@ -122,14 +121,14 @@ llm_router_models_max: 1 # CRITICAL: RTX 3090 24GB,
|
||||
llm_router_ctx_size: 65536 # 64K — must match production (Hermes floor)
|
||||
llm_router_parallel: 1
|
||||
llm_router_gpu_layers: 99
|
||||
llm_router_batch_size: 2048
|
||||
llm_router_ubatch_size: 512
|
||||
llm_router_batch_size: 4096
|
||||
llm_router_ubatch_size: 4096
|
||||
llm_router_cache_type_k: q4_0 # required to fit 64K KV in 24GB
|
||||
llm_router_cache_type_v: q4_0
|
||||
llm_router_flash_attn: "auto"
|
||||
llm_router_bind_address: "{{ llm_bind_address }}" # 10.1.71.130
|
||||
llm_router_allowed_source_cidr: "{{ llm_allowed_source_cidr }}" # 10.1.70.0/24
|
||||
llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S" # verified at Gate 1
|
||||
llm_router_expected_model_id: "Qwen3.8-27B-Q4_K_M" # verified at Gate 1
|
||||
llm_router_vram_max_mib: 23000 # Gate 3: fail if exceeded under load
|
||||
|
||||
# --- Router preset mode (--models-preset INI) ---------------------------------
|
||||
@@ -160,8 +159,23 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}"
|
||||
llm_router_coder_ctx_size: 16384
|
||||
llm_router_coder_flash_attn: "true"
|
||||
llm_router_preset_path: /opt/llama-server-router-preset.ini
|
||||
# Qwen3.8-27B: ctx=32768 (32K). Measured VRAM: 17,068 MiB at 32K vs 17,804 MiB at 64K.
|
||||
# Using 32K to leave more headroom during LRU eviction transitions on the 24GB RTX 3090.
|
||||
# Native context of Qwen3.8-27B is 262,144 tokens; 32K is sufficient for Hermes.
|
||||
llm_router_qwen38_ctx_size: 32768
|
||||
# nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83
|
||||
# No flash_attn or KV cache params — embedding models use bidirectional forward pass,
|
||||
# No flash_attn or KV cache params - embedding models use bidirectional forward pass,
|
||||
# not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it
|
||||
# always warm at negligible VRAM cost (~84MB).
|
||||
llm_router_nomic_ctx_size: 8192
|
||||
# FIX (2026-08-14, t_openviking_embed_batch): batch-size/ubatch-size were
|
||||
# previously omitted from this section entirely, so llama-server silently
|
||||
# defaulted the physical batch (ubatch-size) to 512 tokens. Embedding requests
|
||||
# cannot be split across ubatches in llama.cpp, so any OpenViking chunk over
|
||||
# ~512 tokens (observed 2000-3400 tokens/chunk from openviking-config's
|
||||
# embedding.dense chunking) hard-failed with "input (N tokens) is too large to
|
||||
# process. increase the physical batch size" - this fed OpenViking's circuit
|
||||
# breaker into a permanent fail/re-enqueue loop. 4096 covers the observed max
|
||||
# comfortably while staying under ctx-size=8192.
|
||||
llm_router_nomic_batch_size: 4096
|
||||
llm_router_nomic_ubatch_size: 4096
|
||||
|
||||
@@ -50,18 +50,39 @@
|
||||
; waiting for the first request. — War Machine.
|
||||
; ------------------------------------------------------------------------------
|
||||
|
||||
; --- Production model: Qwen3.6-35B-A3B-UD-Q4_K_S ----------------------------
|
||||
; Primary model ID: Qwen3.6-35B-A3B-UD-Q4_K_S (unchanged from --models-dir)
|
||||
; ~20GB, primary Hermes production LLM. Context: 64K with q4_0 KV cache.
|
||||
[Qwen3.6-35B-A3B-UD-Q4_K_S]
|
||||
model = {{ llm_models_dir }}/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf
|
||||
n-gpu-layers = {{ llm_router_gpu_layers }}
|
||||
ctx-size = {{ llm_router_ctx_size }}
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
; --- Production model: Qwen3.8-27B-Q4_K_M ------------------------------------
|
||||
; Swapped from Qwen3.6-35B-A3B-UD-Q4_K_S by War Machine (t_f5f7e9ad, 2026-08-16).
|
||||
; Ryan-directed swap. Qwen3.8-27B is a dense 27B VLM (Apache-2.0) from Alibaba,
|
||||
; released Aug 2026. GGUF quantized by Unsloth Dynamic V3.0 (preview).
|
||||
; Q4_K_M chosen: 17.1GB weights — fits RTX 3090 (24GB) with ~7GB headroom for
|
||||
; KV cache at ctx=65536 (q4_0 KV). Smaller than prior Qwen3.6 at ~20GB.
|
||||
; Native context: 262,144 tokens. Running at 65536 (Hermes floor) for now;
|
||||
; can be raised later if needed.
|
||||
; VRAM footprint (measured 2026-08-16): 17,068 MiB at ctx=32768 with q4_0 KV;
|
||||
; 17,804 MiB at ctx=65536. Using 32768 (32K) to give more eviction headroom
|
||||
; on the 24GB RTX 3090 (nomic-embed 558MB always resident; total ~17.6GB).
|
||||
; Native context is 262,144 tokens; 32K is sufficient for Hermes usage.
|
||||
; Architecture note: Qwen3.8 uses Gated DeltaNet; llama.cpp 6ea215d logs
|
||||
; "fused Gated Delta Net (chunked) not supported, set to disabled" — falls
|
||||
; back to non-fused implementation. Inference works correctly but may be
|
||||
; slower on the GDN layers. An updated llama.cpp may improve throughput.
|
||||
; load-on-startup NOT set (loads on first request, ~30-60s cold load).
|
||||
; With models-max=2 in host_vars, nomic-embed occupies slot 1 (pinned),
|
||||
; and the generative slot (slot 2) is Qwen3.8 on first request. Auxiliary
|
||||
; models (Phi, Llama, Coder) evict Qwen3.8 when requested; Qwen3.8 evicts
|
||||
; them in turn. One cold-load (~30-60s) per switch between Qwen3.8 and
|
||||
; auxiliary models is expected and acceptable. In practice, once Hermes
|
||||
; config.yaml references Qwen3.8 as primary, it stays resident.
|
||||
; Primary model ID: Qwen3.8-27B-Q4_K_M
|
||||
[Qwen3.8-27B-Q4_K_M]
|
||||
model = {{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf
|
||||
n-gpu-layers = {{ llm_router_gpu_layers }}
|
||||
ctx-size = {{ llm_router_qwen38_ctx_size }}
|
||||
cache-type-k = {{ llm_router_cache_type_k }}
|
||||
cache-type-v = {{ llm_router_cache_type_v }}
|
||||
batch-size = {{ llm_router_batch_size }}
|
||||
ubatch-size = {{ llm_router_ubatch_size }}
|
||||
parallel = {{ llm_router_parallel }}
|
||||
|
||||
; --- Auxiliary model: Phi-3.5-mini-instruct-Q8_0 ----------------------------
|
||||
; Primary model ID: Phi-3.5-mini-instruct-Q8_0 (unchanged from --models-dir)
|
||||
@@ -147,10 +168,35 @@ parallel = {{ llm_router_parallel }}
|
||||
; for embedding inference and may be silently ignored or cause warnings; omit.
|
||||
; Source: nomic-ai/nomic-embed-text-v1.5-GGUF (public, no auth needed)
|
||||
; Added 2026-08-13 (t_34b96e83) — War Machine.
|
||||
;
|
||||
; FIXED (2026-08-14, t_openviking_embed_batch): the original section omitted
|
||||
; batch-size/ubatch-size, so llama-server defaulted the PHYSICAL batch
|
||||
; (ubatch-size) to 512 tokens. For embedding requests llama.cpp cannot split
|
||||
; a single input across ubatches, so any OpenViking chunk over ~512 tokens
|
||||
; large chunk over ~512 tokens (observed 2000-3400 tokens/chunk) failed hard with "input (N tokens) is too
|
||||
; large to process. increase the physical batch size (current batch size:
|
||||
; 512)". This tripped OpenViking's circuit breaker into an infinite
|
||||
; fail/re-enqueue loop. Fix: set batch-size/ubatch-size to 4096 (comfortably
|
||||
; over the observed max chunk size and under ctx-size=8192).
|
||||
;
|
||||
; FOLLOW-UP FINDING (2026-08-14, same task): after the batch-size fix landed,
|
||||
; logs showed a SECOND, separate problem: llama.cpp capped the effective
|
||||
; context to 2048 regardless of ctx-size=8192 ("n_ctx_seq (8192) > n_ctx_train
|
||||
; (2048)" / "capping"). This is expected per the nomic-embed-text-v1.5-GGUF
|
||||
; model card: the base GGUF's native RoPE training context is 2048; the
|
||||
; original HF model reaches its benchmarked 8192-token context via Dynamic
|
||||
; NTK-Aware RoPE scaling, which llama.cpp does not implement — so llama.cpp
|
||||
; defaults to 2048 unless YaRN scaling is explicitly requested. Model card
|
||||
; prescribes: --rope-scaling yarn --rope-freq-scale 0.75 alongside -c 8192.
|
||||
; Added rope-scaling/rope-freq-scale below to actually reach 8192.
|
||||
[nomic-embed-text-v1.5]
|
||||
model = {{ llm_models_dir }}/nomic-embed-text-v1.5-Q4_K_M.gguf
|
||||
embedding = true
|
||||
n-gpu-layers = {{ llm_router_gpu_layers }}
|
||||
ctx-size = {{ llm_router_nomic_ctx_size }}
|
||||
batch-size = {{ llm_router_nomic_batch_size }}
|
||||
ubatch-size = {{ llm_router_nomic_ubatch_size }}
|
||||
rope-scaling = yarn
|
||||
rope-freq-scale = 0.75
|
||||
load-on-startup = true
|
||||
sleep-idle-seconds = -1
|
||||
|
||||
@@ -19,8 +19,10 @@ ExecStart={{ llm_binary_path }} \
|
||||
--host {{ llm_router_bind_address }} \
|
||||
--port {{ llm_router_port }} \
|
||||
--n-gpu-layers {{ llm_router_gpu_layers }} \
|
||||
{% if not (llm_router_preset_enabled | default(false)) %}
|
||||
--ctx-size {{ llm_router_ctx_size }} \
|
||||
--flash-attn {{ llm_router_flash_attn }} \
|
||||
{% endif %}
|
||||
--cache-type-k {{ llm_router_cache_type_k }} \
|
||||
--cache-type-v {{ llm_router_cache_type_v }} \
|
||||
--batch-size {{ llm_router_batch_size }} \
|
||||
|
||||
Reference in New Issue
Block a user