feat(astro-orbiter): bump Qwen3.8-27B ctx-size 32768->131072 (128K) [t_441470b9]

ansible/host_vars/astro-orbiter/vars.yml

@@ -34,14 +34,14 @@ common_root_lv: ubuntu-lv
 # (t_33acbb2e) so the router can keep more than one GGUF resident on-demand
 # and LRU-evict when needed.
 #
-# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad):
+# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad, updated t_441470b9):
 # With models-max=4 and all 5 GGUFs registered, worst case is all 5 loaded simultaneously:
-#   Qwen3.8-27B Q4_K_M:               ~23.1GB (weights ~17.1GB + KV ~6GB @ 64K ctx, q4_0)
+#   Qwen3.8-27B Q4_K_M:               ~23.3GB (weights ~17.1GB + KV ~6.2GB @ 128K ctx, q4_0) ← UPDATED
 #   Phi-3.5-mini-instruct Q8_0:       ~4.3GB  (weights ~3.8GB + KV ~0.5GB @ 32K ctx)
 #   Meta-Llama-3.1-8B Q4_K_M:        ~5.6GB  (weights ~4.6GB + KV ~0.2GB @ 8K ctx)
 #   Qwen2.5-Coder-14B Q4_K_M:        ~9.0GB  (weights ~8.4GB + KV ~0.6GB @ 16K ctx)
 #   nomic-embed-text-v1.5 Q4_K_M:    ~0.09GB (~84MB, embedding only — no KV cache)
-#   Total worst-case:                 ~42.1GB  >>  24GB RTX 3090
+#   Total worst-case:                 ~42.3GB  >>  24GB RTX 3090
 #
 # OOM RISK: Full co-residency is impossible on 24GB. LRU eviction prevents this
 # in practice: models-max=4 means the router can REGISTER 5 models but only keeps
@@ -65,6 +65,9 @@ common_root_lv: ubuntu-lv
 # auxiliary models evict it on demand, and vice versa).
 # NOTE: Qwen3.8 does NOT have load-on-startup — it loads on first request.
 # This avoids an LRU eviction race with nomic-embed at startup.
+# UPDATED (t_441470b9, 2026-08-16): ctx bumped to 131072 (128K). Measured
+# VRAM: 20,282 MiB at 131072 ctx. nomic-embed 558 MiB always resident ->
+# ~20.8GB total, ~3.2GB headroom. models-max=2 unchanged (same constraint).
 llm_router_models_max: 2
 
 llm_staged_models:

ansible/playbooks/day2_qwen38_ctx128k.yml

@@ -0,0 +1,38 @@
+---
+# ------------------------------------------------------------------------------
+# Playbook: day2_qwen38_ctx128k.yml
+# Purpose:  Bump Qwen3.8-27B-Q4_K_M ctx-size from 32768 to 131072 (128K)
+#           on astro-orbiter's production router (port 8002).
+#
+# What this playbook does:
+#   1. Renders the updated llama-server-router-preset.ini.j2 (now with
+#      llm_router_qwen38_ctx_size: 131072) to /opt/llama-server-router-preset.ini.
+#   2. Restarts llama-server-router.service.
+#   3. Verifies the router loads Qwen3.8-27B at ctx=131072 in status.args.
+#
+# Context:
+#   - Empirical VRAM test (t_4455a44c): 131072 ctx = 20,282 MiB Qwen3.8
+#     + 558 MiB nomic-embed = ~20.8GB total; ~3.2GB headroom on 24GB RTX 3090.
+#     Co-resident with nomic-embed: comfortably fits.
+#   - Ryan approved this deployment.
+#   - Semaphore SSH gap for astro-orbiter still applies (t_730f9584 / t_33acbb2e);
+#     running direct CLI Ansible per standing exception.
+#
+# Run:
+#   cd /home/hermes/git/homelab/ansible
+#   env -u ANSIBLE_VAULT_PASSWORD_FILE ansible-playbook \
+#     -i inventory.yml \
+#     playbooks/day2_qwen38_ctx128k.yml
+#
+# Task reference: t_441470b9 — War Machine, 2026-08-16
+# ------------------------------------------------------------------------------
+- name: Bump Qwen3.8-27B ctx-size to 131072 on astro-orbiter
+  hosts: astro-orbiter
+  become: true
+  vars:
+    llm_router_preset_enabled: true
+    llm_router_qwen38_ctx_size: 131072
+
+  roles:
+    - role: llm-inference-multimodel
+      tags: [preset, systemd, verify]

ansible/roles/llm-inference-multimodel/defaults/main.yml

@@ -159,10 +159,12 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}"
 llm_router_coder_ctx_size: 16384
 llm_router_coder_flash_attn: "true"
 llm_router_preset_path: /opt/llama-server-router-preset.ini
-# Qwen3.8-27B: ctx=32768 (32K). Measured VRAM: 17,068 MiB at 32K vs 17,804 MiB at 64K.
-# Using 32K to leave more headroom during LRU eviction transitions on the 24GB RTX 3090.
-# Native context of Qwen3.8-27B is 262,144 tokens; 32K is sufficient for Hermes.
-llm_router_qwen38_ctx_size: 32768
+# Qwen3.8-27B: ctx=131072 (128K). Bumped from 32768 -> 131072 per Ryan approval (t_441470b9, 2026-08-16).
+# Measured VRAM: 20,282 MiB at 131072 ctx (empirically tested in t_4455a44c); nomic-embed 558 MiB
+# always resident -> ~20.8GB total, ~3.2GB headroom on 24GB RTX 3090. Comfortably safe.
+# Prior value was 32768 (17,068 MiB) — bumping 4x for genuine 128K context.
+# Native context of Qwen3.8-27B is 262,144 tokens; 128K is a practical production ceiling.
+llm_router_qwen38_ctx_size: 131072
 # nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83
 # No flash_attn or KV cache params - embedding models use bidirectional forward pass,
 # not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it

ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2

@@ -58,10 +58,11 @@
 ; KV cache at ctx=65536 (q4_0 KV). Smaller than prior Qwen3.6 at ~20GB.
 ; Native context: 262,144 tokens. Running at 65536 (Hermes floor) for now;
 ; can be raised later if needed.
-; VRAM footprint (measured 2026-08-16): 17,068 MiB at ctx=32768 with q4_0 KV;
-;   17,804 MiB at ctx=65536. Using 32768 (32K) to give more eviction headroom
-;   on the 24GB RTX 3090 (nomic-embed 558MB always resident; total ~17.6GB).
-;   Native context is 262,144 tokens; 32K is sufficient for Hermes usage.
+; VRAM footprint (empirically tested, t_4455a44c 2026-08-16):
+;   ctx=32768:  17,068 MiB; ctx=65536: 17,804 MiB; ctx=131072: 20,282 MiB.
+;   BUMPED to 131072 (128K) per Ryan approval (t_441470b9, 2026-08-16).
+;   nomic-embed always resident at 558 MiB -> total ~20.8GB, ~3.2GB headroom.
+;   Native context is 262,144 tokens; 128K is the production ceiling.
 ; Architecture note: Qwen3.8 uses Gated DeltaNet; llama.cpp 6ea215d logs
 ; "fused Gated Delta Net (chunked) not supported, set to disabled" — falls
 ; back to non-fused implementation. Inference works correctly but may be
@@ -73,6 +74,7 @@
 ; them in turn. One cold-load (~30-60s) per switch between Qwen3.8 and
 ; auxiliary models is expected and acceptable. In practice, once Hermes
 ; config.yaml references Qwen3.8 as primary, it stays resident.
+; ctx-size raised to 131072 (128K) per Ryan approval (t_441470b9, 2026-08-16).
 ; Primary model ID: Qwen3.8-27B-Q4_K_M
 [Qwen3.8-27B-Q4_K_M]
 model              = {{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf