From 9d6869ad9db14012d8cc6fedc6f08e15466b3740 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 31 Aug 2026 18:17:25 -0500 Subject: [PATCH] hindsight: revert embeddings cutover after dimension-mismatch crash HINDSIGHT_API_EMBEDDINGS_PROVIDER=openai pointed at vLLM's nomic-embed-text-v1.5 (768-dim) crash-looped hindsight-api: 'Cannot change embedding dimension from 384 to 768: memory_units table contains 1289 rows with embeddings.' Hindsight was never actually using astro-orbiter for embeddings (defaults to a bundled local 384-dim sentence-transformers model when the env var is unset) -- this was a genuine architecture assumption error, not a config typo. Re-embedding all existing memory data across ~20 agent banks is a destructive, irreversible operation requiring its own explicit, approved task -- not something to trigger as a side effect of an infra migration. Keeps the LLM cutover (astro-orbiter:8000, vLLM Qwen2.5-32B-Instruct-AWQ) which is safe and already validated. --- .../hindsight/externalsecret.yaml | 15 ++--- cluster/applications/hindsight/values.yaml | 57 +++++++++++-------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/cluster/applications/hindsight/externalsecret.yaml b/cluster/applications/hindsight/externalsecret.yaml index 8211a1f..4476dc5 100644 --- a/cluster/applications/hindsight/externalsecret.yaml +++ b/cluster/applications/hindsight/externalsecret.yaml @@ -55,15 +55,12 @@ spec: # env-var-named keys injected via envFrom HINDSIGHT_API_MCP_AUTH_TOKEN: "{{ .HINDSIGHT_API_MCP_AUTH_TOKEN }}" HINDSIGHT_API_LLM_API_KEY: "{{ .HINDSIGHT_API_LLM_API_KEY }}" - # Cutover to vLLM (t_e6facb19, 2026-08-31): vLLM enforces its API key - # on every request (unlike llama-swap, which accepted any/no key) — - # confirmed empirically, a bad/missing key gets a real 401 - # {"error":"Unauthorized"}. Both the completions endpoint (:8000) and - # the embeddings endpoint (:8020) are separate vLLM processes but - # share the SAME key (api-key.env is written once, read by both - # systemd units per roles/deploy-vllm/tasks/api-key.yml). Reuse - # HINDSIGHT_API_LLM_API_KEY's value for the embeddings key too. - HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY: "{{ .HINDSIGHT_API_LLM_API_KEY }}" + # NOTE: no HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY here — Hindsight's + # embeddings stayed on its bundled local provider after the vLLM + # embeddings cutover attempt crash-looped the API on an embedding + # dimension mismatch (384 existing vs 768 nomic-embed). See + # values.yaml's api.env comment for the full incident writeup + # (t_e6facb19, 2026-08-31). data: # 1Password item "hindsight", field "postgres-password". # letters+digits only / URL-safe: required because the chart interpolates the diff --git a/cluster/applications/hindsight/values.yaml b/cluster/applications/hindsight/values.yaml index 2b48dd5..0e5776f 100644 --- a/cluster/applications/hindsight/values.yaml +++ b/cluster/applications/hindsight/values.yaml @@ -61,35 +61,44 @@ existingSecret: hindsight-credentials api: env: # Cutover to vLLM (t_e6facb19, 2026-08-31): astro-orbiter's llama-swap is - # being retired for the completions+embeddings roles Hindsight needs. - # vLLM serves Qwen2.5-32B-Instruct-AWQ (:8000, completions) and - # nomic-embed-text-v1.5 (:8020, --runner pooling --convert embed) as two - # independent systemd-managed processes — validated end-to-end (health, - # /v1/chat/completions, /v1/embeddings returning a 768-dim vector) in a - # shadow window with llama-swap stopped. Model swap from Qwen3.8-27B to - # Qwen2.5-32B-Instruct-AWQ: both are capable instruction-tuned models: - # confirm structured-extraction/tagging behavior (the prior stepfun/solar - # 400 "missing tags" failure mode) holds up after this swap — see + # being retired for the completions role Hindsight needs. vLLM serves + # Qwen2.5-32B-Instruct-AWQ (:8000, completions) — validated end-to-end + # (health, /v1/chat/completions) in a shadow window with llama-swap + # stopped. Model swap from Qwen3.8-27B to Qwen2.5-32B-Instruct-AWQ: both + # are capable instruction-tuned models; confirm structured-extraction/ + # tagging behavior (the prior stepfun/solar 400 "missing tags" failure + # mode) holds up after this swap — see # references/hindsight-backend-fallback-to-local-qwen-workflow.md if it # regresses and llama-swap needs to come back temporarily. HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8000/v1" HINDSIGHT_API_LLM_PROVIDER: "openai" HINDSIGHT_API_LLM_MODEL: "Qwen2.5-32B-Instruct-AWQ" - # Embeddings: llama.cpp nomic-embed-text-v1.5 (llama-swap, :8001) -> - # vLLM nomic-embed-text-v1.5 (:8020). Same model weights, different - # serving stack. Hindsight's embeddings provider defaults to "local" - # (bundled sentence-transformers) unless overridden — explicit TEI/openai - # provider config was never wired for astro-orbiter previously (verified: - # `kubectl exec ... env | grep -i embed` showed NO HINDSIGHT_API_EMBEDDINGS_* - # vars set, confirming Hindsight was using its bundled local embedder, NOT - # astro-orbiter, before this change — despite the OpenViking-style stack - # documentation implying otherwise). Wiring the openai-compatible provider - # here for the first time to point Hindsight's actual embedding generation - # at the vLLM-served nomic-embed-text-v1.5, matching the model OpenViking - # already uses (consistency across mk-labs services). - HINDSIGHT_API_EMBEDDINGS_PROVIDER: "openai" - HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL: "http://astro-orbiter:8020/v1" - HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL: "nomic-embed-text-v1.5" + # DO NOT set HINDSIGHT_API_EMBEDDINGS_* here (t_e6facb19, 2026-08-31 + # attempted this, reverted after a production incident — see below). + # + # DISCOVERY: Hindsight's embeddings provider was NEVER pointed at + # astro-orbiter. It defaults to "local" (bundled sentence-transformers, + # BAAI/bge-small-en-v1.5, 384 dimensions) whenever + # HINDSIGHT_API_EMBEDDINGS_PROVIDER is unset — verified via + # `kubectl exec ... env | grep -i embed` showing NO + # HINDSIGHT_API_EMBEDDINGS_* vars in the live pod, despite this file's + # LLM section referencing astro-orbiter for years. The nomic-embed- + # text-v1.5 model documented across mk-labs skills as "Hindsight's + # embedding model" was OpenViking's embedding model, not Hindsight's. + # + # INCIDENT: pointing HINDSIGHT_API_EMBEDDINGS_PROVIDER at vLLM's + # nomic-embed-text-v1.5 (768 dimensions) crash-looped hindsight-api on + # rollout: `RuntimeError: Cannot change embedding dimension from 384 to + # 768: memory_units table contains 1289 rows with embeddings.` The + # migration path (`ensure_embedding_dimension` in migrations.py) refuses + # a live dimension change without either re-embedding everything or + # deleting all existing memory_units rows across every bank (jarvis, + # hermes, war-machine, and ~18 other agent banks) — a destructive, + # irreversible operation requiring explicit human approval, not + # something to do as a side effect of an infra migration task. Reverted + # immediately; Hindsight keeps its bundled local embedder (384-dim, + # unchanged, zero data risk) until a deliberate, approved re-embedding + # migration is planned as its own task. # --- t_d7f8cd65: fix 502s on the serial astro-orbiter node --- # astro-orbiter is a single llama-swap process (serial: 1 generate at a # time, ctx 64K). Hindsight's default LLM concurrency is 32, so a retain