From 782cbe33d1d2a76609b0d5dc2b9c936a5f763a86 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Wed, 5 Aug 2026 12:14:11 -0500 Subject: [PATCH] llm-inference: size ctx-size/parallel for aux task offload Previous ctx-size=8192/parallel=4 gave 2048 tokens/slot, too small for context compression inputs (observed live rejection at 3826 tokens). Measured VRAM on astro-orbiter (RTX 3090 24GB): weights ~17GB resident, ~294KiB/token pool-wide for KV cache+buffers at prior sizing. New: ctx-size=16384, parallel=2 -> 8192 tokens/slot (matches model's native n_ctx_train max). Projected VRAM ~21.8GB, ~2.7GB headroom. Applied directly via ansible-playbook (Semaphore currently broken -- fix tracked separately). --- ansible/roles/llm-inference/defaults/main.yml | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/ansible/roles/llm-inference/defaults/main.yml b/ansible/roles/llm-inference/defaults/main.yml index 007ddbf..5369df7 100644 --- a/ansible/roles/llm-inference/defaults/main.yml +++ b/ansible/roles/llm-inference/defaults/main.yml @@ -21,9 +21,28 @@ llm_hf_model: google/gemma-2-27b-it # llama-server serve llm_serve_port: 8000 llm_serve_host: "0.0.0.0" -llm_max_model_len: 8192 +# NOTE (2026-08-05): --ctx-size is llama.cpp's TOTAL KV cache pool, divided +# evenly across --parallel slots (per-slot context = ctx-size / parallel). +# Previous 8192/4=2048 tokens-per-slot was too small for aux task offload +# (context compression) and caused live rejections: "request (3826 tokens) +# exceeds the available context size (2048 tokens)". +# Sized against measured VRAM on astro-orbiter (RTX 3090, 24576MiB total): +# - Weights (Q4_K_M, 27B) ~16998MiB resident. +# - At ctx-size=8192/parallel=4, total llama-server VRAM = 19404MiB +# (nvidia-smi), i.e. ~2406MiB for KV cache + compute buffers at 8192 +# total context tokens -> ~294KiB/token (pool-wide, incl. buffers). +# - Model n_ctx_train=8192 is the native max; per-slot context beyond +# this degrades coherence, so per-slot should cap at 8192. +# - New sizing: ctx-size=16384, parallel=2 -> 8192 tokens/slot (native +# max, covers compression's multi-thousand-token inputs with margin). +# Projected VRAM: 16998 + (~294KiB/token * 16384) ≈ 21.8GB used, +# leaving ~2.7GB headroom on the 24GB card. +# - parallel=2 (not 4) trades some concurrency for correct per-slot +# context; 2 concurrent aux-task requests is enough headroom before +# the known "3+ simultaneous compressions" GPU bottleneck kicks in. +llm_max_model_len: 16384 llm_gpu_layers: 99 # offload all layers to GPU -llm_parallel_slots: 4 # concurrent request slots +llm_parallel_slots: 2 # concurrent request slots -> 8192 tokens/slot (ctx-size / parallel) llm_gguf_dir: /home/jarvis/models llm_gguf_path: /home/jarvis/models/gemma-2-27b-it-Q4_K_M.gguf