diff --git a/cluster/applications/hindsight/values.yaml b/cluster/applications/hindsight/values.yaml index db83f29..d295984 100644 --- a/cluster/applications/hindsight/values.yaml +++ b/cluster/applications/hindsight/values.yaml @@ -75,23 +75,29 @@ api: # MODEL SWAP (t_r1d32b_swap, 2026-09-01): astro-orbiter's vLLM primary # model changed from Qwen2.5-32B-Instruct-AWQ to # DeepSeek-R1-Distill-Qwen-32B-AWQ (single-model deployment; nomic-embed - # and Qwen3-8B-AWQ both disabled on that host). Same endpoint - # (http://astro-orbiter:8000/v1), same API key (vLLM instance/port - # unchanged) -- only the served model name changed. DeepSeek-R1 emits - # a ... reasoning trace before its final answer; - # Hindsight's fact-extraction prompt parsing has not been specifically - # audited against this reasoning-model output shape -- flag for - # follow-up if extraction quality regresses (see README note below). + # and Qwen3-8B-AWQ both disabled on that host). Superseded below. + # + # MODEL SWAP #2 (t_gemma4_swap, 2026-09-01): DeepSeek-R1-Distill-Qwen-32B + # retired after confirming its tool_choice="auto" reliability is a + # known, documented DeepSeek-R1-distillation limitation (upstream + # GitHub-confirmed: trained on pure reasoning traces, no function- + # calling data) -- not relevant to Hindsight's pure-text extraction + # use case, but disqualifying for agent-facing Hermes profiles, which + # drove the swap. Replaced with Gemma 4 26B A4B (Google, Apache 2.0, + # US-origin). Same endpoint (http://astro-orbiter:8000/v1), same API + # key -- only the served model name changed. Gemma 4 does NOT emit a + # reasoning trace by default (confirmed live) -- simpler completion + # parsing than DeepSeek-R1's always-on blocks. HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8000/v1" HINDSIGHT_API_LLM_PROVIDER: "openai" - HINDSIGHT_API_LLM_MODEL: "DeepSeek-R1-Distill-Qwen-32B-AWQ" - # vLLM's max_model_len is now 32768 (up from the prior 8192 role - # default) -- more headroom than before, but DeepSeek-R1 reasoning - # traces consume completion tokens before the final answer even - # appears, so the completion cap is left unchanged pending live - # verification that 4096 is enough for a real retain to complete - # (rather than truncating mid-). Raise if extraction responses - # come back truncated/empty. + HINDSIGHT_API_LLM_MODEL: "Gemma-4-26B-A4B-it-AWQ" + # vLLM's max_model_len is now 65536 (up from DeepSeek's 32768, up from + # the original 8192 role default). Gemma 4's native context is 256K; + # 65536 is astro-orbiter's configured ceiling, comfortably above + # Hermes's 64K floor. Completion cap left at 4096 pending live + # verification -- Gemma 4 doesn't burn tokens on unwanted reasoning + # traces the way DeepSeek-R1 did, so 4096 should have MORE effective + # headroom for the actual extraction output than it did before. HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS: "4096" # DO NOT set HINDSIGHT_API_EMBEDDINGS_* here (t_e6facb19, 2026-08-31 # attempted this, reverted after a production incident — see below). @@ -130,8 +136,9 @@ api: # Kept at 1 post-cutover: vLLM's single-process-per-model design is also # effectively serial for a single generative model instance under this # GPU's VRAM budget (KV cache sized tight against the 24GB card at - # max_model_len=32768 for DeepSeek-R1-Distill-Qwen-32B-AWQ, t_r1d32b_swap - # 2026-09-01 — previously 8192 for Qwen2.5-32B-Instruct-AWQ). + # max_model_len=65536 for Gemma-4-26B-A4B-it-AWQ, t_gemma4_swap + # 2026-09-01 — previously 32768 for DeepSeek-R1-Distill-Qwen-32B-AWQ, + # previously 8192 for Qwen2.5-32B-Instruct-AWQ). HINDSIGHT_API_LLM_MAX_CONCURRENT: "1" HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT: "1" # Client + per-request timeout. Default is 120s; a 29K-token retain runs