From a2994bf55d3ccfdf5d429200f13c31d1360df0c7 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Sun, 16 Aug 2026 22:39:12 -0500 Subject: [PATCH] feat(astro-orbiter): bump Qwen3.8-27B ctx-size 32768->131072 (128K) [t_441470b9] --- ansible/host_vars/astro-orbiter/vars.yml | 9 +++-- ansible/playbooks/day2_qwen38_ctx128k.yml | 38 +++++++++++++++++++ .../defaults/main.yml | 10 +++-- .../llama-server-router-preset.ini.j2 | 10 +++-- 4 files changed, 56 insertions(+), 11 deletions(-) create mode 100644 ansible/playbooks/day2_qwen38_ctx128k.yml diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index 30f1f36..b31ee1e 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -34,14 +34,14 @@ common_root_lv: ubuntu-lv # (t_33acbb2e) so the router can keep more than one GGUF resident on-demand # and LRU-evict when needed. # -# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad): +# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad, updated t_441470b9): # With models-max=4 and all 5 GGUFs registered, worst case is all 5 loaded simultaneously: -# Qwen3.8-27B Q4_K_M: ~23.1GB (weights ~17.1GB + KV ~6GB @ 64K ctx, q4_0) +# Qwen3.8-27B Q4_K_M: ~23.3GB (weights ~17.1GB + KV ~6.2GB @ 128K ctx, q4_0) ← UPDATED # Phi-3.5-mini-instruct Q8_0: ~4.3GB (weights ~3.8GB + KV ~0.5GB @ 32K ctx) # Meta-Llama-3.1-8B Q4_K_M: ~5.6GB (weights ~4.6GB + KV ~0.2GB @ 8K ctx) # Qwen2.5-Coder-14B Q4_K_M: ~9.0GB (weights ~8.4GB + KV ~0.6GB @ 16K ctx) # nomic-embed-text-v1.5 Q4_K_M: ~0.09GB (~84MB, embedding only — no KV cache) -# Total worst-case: ~42.1GB >> 24GB RTX 3090 +# Total worst-case: ~42.3GB >> 24GB RTX 3090 # # OOM RISK: Full co-residency is impossible on 24GB. LRU eviction prevents this # in practice: models-max=4 means the router can REGISTER 5 models but only keeps @@ -65,6 +65,9 @@ common_root_lv: ubuntu-lv # auxiliary models evict it on demand, and vice versa). # NOTE: Qwen3.8 does NOT have load-on-startup — it loads on first request. # This avoids an LRU eviction race with nomic-embed at startup. +# UPDATED (t_441470b9, 2026-08-16): ctx bumped to 131072 (128K). Measured +# VRAM: 20,282 MiB at 131072 ctx. nomic-embed 558 MiB always resident -> +# ~20.8GB total, ~3.2GB headroom. models-max=2 unchanged (same constraint). llm_router_models_max: 2 llm_staged_models: diff --git a/ansible/playbooks/day2_qwen38_ctx128k.yml b/ansible/playbooks/day2_qwen38_ctx128k.yml new file mode 100644 index 0000000..fdcfc0f --- /dev/null +++ b/ansible/playbooks/day2_qwen38_ctx128k.yml @@ -0,0 +1,38 @@ +--- +# ------------------------------------------------------------------------------ +# Playbook: day2_qwen38_ctx128k.yml +# Purpose: Bump Qwen3.8-27B-Q4_K_M ctx-size from 32768 to 131072 (128K) +# on astro-orbiter's production router (port 8002). +# +# What this playbook does: +# 1. Renders the updated llama-server-router-preset.ini.j2 (now with +# llm_router_qwen38_ctx_size: 131072) to /opt/llama-server-router-preset.ini. +# 2. Restarts llama-server-router.service. +# 3. Verifies the router loads Qwen3.8-27B at ctx=131072 in status.args. +# +# Context: +# - Empirical VRAM test (t_4455a44c): 131072 ctx = 20,282 MiB Qwen3.8 +# + 558 MiB nomic-embed = ~20.8GB total; ~3.2GB headroom on 24GB RTX 3090. +# Co-resident with nomic-embed: comfortably fits. +# - Ryan approved this deployment. +# - Semaphore SSH gap for astro-orbiter still applies (t_730f9584 / t_33acbb2e); +# running direct CLI Ansible per standing exception. +# +# Run: +# cd /home/hermes/git/homelab/ansible +# env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook \ +# -i inventory.yml \ +# playbooks/day2_qwen38_ctx128k.yml +# +# Task reference: t_441470b9 — War Machine, 2026-08-16 +# ------------------------------------------------------------------------------ +- name: Bump Qwen3.8-27B ctx-size to 131072 on astro-orbiter + hosts: astro-orbiter + become: true + vars: + llm_router_preset_enabled: true + llm_router_qwen38_ctx_size: 131072 + + roles: + - role: llm-inference-multimodel + tags: [preset, systemd, verify] diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 89fde29..4e85e4e 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -159,10 +159,12 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}" llm_router_coder_ctx_size: 16384 llm_router_coder_flash_attn: "true" llm_router_preset_path: /opt/llama-server-router-preset.ini -# Qwen3.8-27B: ctx=32768 (32K). Measured VRAM: 17,068 MiB at 32K vs 17,804 MiB at 64K. -# Using 32K to leave more headroom during LRU eviction transitions on the 24GB RTX 3090. -# Native context of Qwen3.8-27B is 262,144 tokens; 32K is sufficient for Hermes. -llm_router_qwen38_ctx_size: 32768 +# Qwen3.8-27B: ctx=131072 (128K). Bumped from 32768 -> 131072 per Ryan approval (t_441470b9, 2026-08-16). +# Measured VRAM: 20,282 MiB at 131072 ctx (empirically tested in t_4455a44c); nomic-embed 558 MiB +# always resident -> ~20.8GB total, ~3.2GB headroom on 24GB RTX 3090. Comfortably safe. +# Prior value was 32768 (17,068 MiB) — bumping 4x for genuine 128K context. +# Native context of Qwen3.8-27B is 262,144 tokens; 128K is a practical production ceiling. +llm_router_qwen38_ctx_size: 131072 # nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83 # No flash_attn or KV cache params - embedding models use bidirectional forward pass, # not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 index 1181628..dc2d354 100644 --- a/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2 @@ -58,10 +58,11 @@ ; KV cache at ctx=65536 (q4_0 KV). Smaller than prior Qwen3.6 at ~20GB. ; Native context: 262,144 tokens. Running at 65536 (Hermes floor) for now; ; can be raised later if needed. -; VRAM footprint (measured 2026-08-16): 17,068 MiB at ctx=32768 with q4_0 KV; -; 17,804 MiB at ctx=65536. Using 32768 (32K) to give more eviction headroom -; on the 24GB RTX 3090 (nomic-embed 558MB always resident; total ~17.6GB). -; Native context is 262,144 tokens; 32K is sufficient for Hermes usage. +; VRAM footprint (empirically tested, t_4455a44c 2026-08-16): +; ctx=32768: 17,068 MiB; ctx=65536: 17,804 MiB; ctx=131072: 20,282 MiB. +; BUMPED to 131072 (128K) per Ryan approval (t_441470b9, 2026-08-16). +; nomic-embed always resident at 558 MiB -> total ~20.8GB, ~3.2GB headroom. +; Native context is 262,144 tokens; 128K is the production ceiling. ; Architecture note: Qwen3.8 uses Gated DeltaNet; llama.cpp 6ea215d logs ; "fused Gated Delta Net (chunked) not supported, set to disabled" — falls ; back to non-fused implementation. Inference works correctly but may be @@ -73,6 +74,7 @@ ; them in turn. One cold-load (~30-60s) per switch between Qwen3.8 and ; auxiliary models is expected and acceptable. In practice, once Hermes ; config.yaml references Qwen3.8 as primary, it stays resident. +; ctx-size raised to 131072 (128K) per Ryan approval (t_441470b9, 2026-08-16). ; Primary model ID: Qwen3.8-27B-Q4_K_M [Qwen3.8-27B-Q4_K_M] model = {{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf