From 39c5fdca69281cd0d654c9740fc323c1f41afa3f Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 31 Aug 2026 19:06:11 -0500 Subject: [PATCH] hindsight: cut over LLM endpoint to vLLM (t_5508360a) Companion commit to 6bfcc76 (vllm permanent residency cutover). Now that llama-swap is stopped+disabled and vLLM is the permanent serving layer: - HINDSIGHT_API_LLM_BASE_URL: llama-swap :8001 -> vLLM :8000 - HINDSIGHT_API_LLM_MODEL: Qwen3.8-27B-Q4_K_M -> Qwen2.5-32B-Instruct-AWQ - Added HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS=4096 (vLLM's max_model_len=8192 is far below the 64000 Hindsight retain default; uncapped this 400s on long retains -- found in t_e6facb19's validation) - ExternalSecret HINDSIGHT_API_LLM_API_KEY source: 1Password 'nous' item (unused fallback) -> 1Password 'vllm' item's api-key (vLLM validates its bearer token, unlike llama-swap which accepted anything) Embeddings intentionally UNCHANGED -- Hindsight stays on its bundled local 384-dim embedder (HINDSIGHT_API_EMBEDDINGS_PROVIDER still unset). Switching to vLLM's nomic-embed (768-dim) is a separate, destructive, explicitly-approved-only migration (~1300 rows across ~20 banks) -- out of scope here, not part of the dashboard's 'stop llama-swap / start vLLM' instruction. --- .../hindsight/externalsecret.yaml | 27 +++++++------- cluster/applications/hindsight/values.yaml | 36 +++++++++---------- 2 files changed, 32 insertions(+), 31 deletions(-) diff --git a/cluster/applications/hindsight/externalsecret.yaml b/cluster/applications/hindsight/externalsecret.yaml index 7a3aa56..2dd934b 100644 --- a/cluster/applications/hindsight/externalsecret.yaml +++ b/cluster/applications/hindsight/externalsecret.yaml @@ -23,9 +23,11 @@ # 1Password item "hindsight" fields (all CONCEALED): # - postgres-password (32 chars, letters+digits only, URL-safe) # - HINDSIGHT_API_MCP_AUTH_TOKEN (48 chars urlsafe bearer token) -# - HINDSIGHT_API_LLM_API_KEY (literal "local-placeholder" — astro-orbiter -# does not validate; must rotate if OpenRouter -# hosted fallback is ever enabled) +# HINDSIGHT_API_LLM_API_KEY now sourced from 1Password item "vllm" (vault +# mk-labs), field "api-key" -- vLLM DOES validate its bearer token (unlike +# llama-swap, which never checked one), so this must be a real matching +# secret post-cutover (t_5508360a, 2026-08-31). See values.yaml api.env +# comment and roles/deploy-vllm/README.md for the full cutover writeup. # ============================================================================ apiVersion: external-secrets.io/v1beta1 kind: ExternalSecret @@ -75,15 +77,16 @@ spec: remoteRef: key: hindsight property: HINDSIGHT_API_MCP_AUTH_TOKEN - # 1Password item "nous" (vault mk-labs), field "api-key" (Ryan-provisioned, - # decision 4). REVERTED to this source (t_e6facb19, 2026-08-31) after the - # vLLM cutover attempt was reverted -- llama-swap doesn't validate the - # API key at all, so this value is effectively unused by the live LLM - # path, but restoring the original source keeps this file's intent - # honest (Nous fallback item, not vLLM's real auth key) until a real - # vLLM cutover is safe to attempt again. See values.yaml's api.env - # comment for the full incident writeup. + # CUT OVER to vLLM's real auth key (t_5508360a, 2026-08-31): llama-swap + # is now stopped+disabled on astro-orbiter (dashboard-approved breaking + # change) and Hindsight's LLM path points at vLLM :8000 + # (Qwen2.5-32B-Instruct-AWQ), which DOES validate its API key (unlike + # llama-swap, which ignored it). Source is the same 1Password item + # deploy-vllm's Ansible role reads at deploy time + # (op://mk-labs/vllm/api-key) -- keeps exactly one source of truth for + # the vLLM auth key across the Ansible-deployed server and this + # Kubernetes-deployed consumer. - secretKey: HINDSIGHT_API_LLM_API_KEY remoteRef: - key: nous + key: vllm property: api-key diff --git a/cluster/applications/hindsight/values.yaml b/cluster/applications/hindsight/values.yaml index 462a0ab..40b04c2 100644 --- a/cluster/applications/hindsight/values.yaml +++ b/cluster/applications/hindsight/values.yaml @@ -60,26 +60,24 @@ existingSecret: hindsight-credentials # ---------------------------------------------------------------------------- api: env: - # REVERTED (t_e6facb19, 2026-08-31) -- see full incident writeup below. - # Attempted cutover to vLLM (:8000, Qwen2.5-32B-Instruct-AWQ), validated - # working in isolation (health, /v1/chat/completions, and a live - # hindsight_retain + recall round-trip all succeeded), but reverted - # after discovering vLLM cannot safely stay resident on this 24GB card - # AT THE SAME TIME as llama-swap needs to serve its own generative - # models: with vLLM's two processes loaded, every llama-swap - # /v1/chat/completions request (Qwen3.8-27B, Qwen3-8B aux models) failed - # with "upstream command exited prematurely" (an OOM at llama-server - # spawn time). That would break the 21 Hermes agent profiles' aux-model - # tasks and OpenViking's VLM -- a much larger blast radius than - # Hindsight alone. Since Hindsight's LLM endpoint must be reachable - # continuously (not just in a brief validation window), and vLLM cannot - # be continuously resident without starving llama-swap, pointing - # Hindsight at vLLM is unsafe until a real architectural fix lands (see - # roles/deploy-vllm README "Known Gaps"). Restored to llama-swap - # (astro-orbiter:8001, Qwen3.8-27B-Q4_K_M) -- the pre-task working state. - HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8001/v1" + # CUT OVER to vLLM (t_5508360a, 2026-08-31): dashboard explicitly + # approved "stop and disable llama-swap and start vLLM" as a breaking + # change. llama-swap is now stopped+disabled on astro-orbiter; vLLM + # (Qwen2.5-32B-Instruct-AWQ, :8000) is the permanent, boot-persistent + # replacement. This exact config (base URL, model name, retry-safe low + # concurrency) was validated end-to-end in t_e6facb19's shadow window + # (health, /v1/chat/completions, live hindsight_retain+recall + # round-trip) before that task reverted it pending this decision -- + # now re-applied for real. See roles/deploy-vllm/README.md "Critical + # architectural finding" + "Consumer cutover status" for the full history. + HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8000/v1" HINDSIGHT_API_LLM_PROVIDER: "openai" - HINDSIGHT_API_LLM_MODEL: "Qwen3.8-27B-Q4_K_M" + HINDSIGHT_API_LLM_MODEL: "Qwen2.5-32B-Instruct-AWQ" + # vLLM's max_model_len=8192 (role default, see deploy-vllm/defaults) is + # far below the 64000 default Hindsight's retain path assumes. Cap + # explicitly or long retains 400 with a context-length error (found + + # fixed in t_e6facb19's validation window). + HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS: "4096" # DO NOT set HINDSIGHT_API_EMBEDDINGS_* here (t_e6facb19, 2026-08-31 # attempted this, reverted after a production incident — see below). #