rblundon/homelab
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(astro-orbiter): bump Qwen3.8-27B ctx-size 32768->131072 (128K) [t_441470b9]

Browse Source
This commit is contained in:
Hermes Agent service account
2026-08-16 22:39:12 -05:00
parent 7b44a41da3
commit a2994bf55d
4 changed files with 56 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ansible/roles/llm-inference-multimodel/defaults/main.yml
View File

@@ -159,10 +159,12 @@ llm_router_phi_flash_attn: "{{ llm_router_flash_attn }}"
llm_router_coder_ctx_size: 16384
llm_router_coder_flash_attn: "true"
llm_router_preset_path: /opt/llama-server-router-preset.ini
# Qwen3.8-27B: ctx=32768 (32K). Measured VRAM: 17,068 MiB at 32K vs 17,804 MiB at 64K.
# Using 32K to leave more headroom during LRU eviction transitions on the 24GB RTX 3090.
# Native context of Qwen3.8-27B is 262,144 tokens; 32K is sufficient for Hermes.
llm_router_qwen38_ctx_size: 32768
# Qwen3.8-27B: ctx=131072 (128K). Bumped from 32768 -> 131072 per Ryan approval (t_441470b9, 2026-08-16).
# Measured VRAM: 20,282 MiB at 131072 ctx (empirically tested in t_4455a44c); nomic-embed 558 MiB
# always resident -> ~20.8GB total, ~3.2GB headroom on 24GB RTX 3090. Comfortably safe.
# Prior value was 32768 (17,068 MiB) — bumping 4x for genuine 128K context.
# Native context of Qwen3.8-27B is 262,144 tokens; 128K is a practical production ceiling.
llm_router_qwen38_ctx_size: 131072
# nomic-embed-text-v1.5: embedding model, ctx-size=8192 per task t_34b96e83
# No flash_attn or KV cache params - embedding models use bidirectional forward pass,
# not autoregressive KV cache. load-on-startup=true / sleep-idle-seconds=-1 keep it