From f3a5687adf499bae3cd2a35bf25b9e8f13d363bf Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 31 Aug 2026 18:29:22 -0500 Subject: [PATCH] hindsight: cap RETAIN_MAX_COMPLETION_TOKENS for vLLM's 8192 ctx Default 64000 exceeded vLLM Qwen2.5-32B-Instruct-AWQ's --max-model-len 8192, causing every retain call to 500 with 'max_tokens=64000 cannot be greater than max_model_len=8192'. llama-swap's Qwen3.8-27B ran at ctx=65536 so this never surfaced before the vLLM cutover. Lowered to 4096. --- cluster/applications/hindsight/values.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cluster/applications/hindsight/values.yaml b/cluster/applications/hindsight/values.yaml index 0e5776f..1df216d 100644 --- a/cluster/applications/hindsight/values.yaml +++ b/cluster/applications/hindsight/values.yaml @@ -73,6 +73,16 @@ api: HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8000/v1" HINDSIGHT_API_LLM_PROVIDER: "openai" HINDSIGHT_API_LLM_MODEL: "Qwen2.5-32B-Instruct-AWQ" + # vLLM's Qwen2.5-32B-Instruct-AWQ is deployed with --max-model-len 8192 + # (roles/deploy-vllm/defaults/main.yml vllm_max_model_len; llama-swap's + # Qwen3.8-27B ran at ctx=65536, matching Hindsight's default + # RETAIN_MAX_COMPLETION_TOKENS=64000). Discovered via a live retain 500: + # "max_tokens=64000 cannot be greater than max_model_len=8192" — every + # retain call was requesting a 64K-token completion budget the new + # backend physically cannot serve. Lowered to 4096, comfortably under + # 8192 with room for the ~3000-char (RETAIN_CHUNK_SIZE) input chunk plus + # prompt overhead. Verified with a live hindsight_retain call post-fix. + HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS: "4096" # DO NOT set HINDSIGHT_API_EMBEDDINGS_* here (t_e6facb19, 2026-08-31 # attempted this, reverted after a production incident — see below). #