# ============================================================================ # Hindsight — helm values (Phase C). Consumed by the ArgoCD Application source 1 # via `helm.valueFiles: ["$values/values.yaml"]` (openviking multi-source pattern). # # Design decisions (all verified against chart v0.9.1 + rendered output): # - Chart is the single source for the app (api, control-plane, services, # probes, ingress). We do NOT hand-roll Deployments/Services. # - Postgres is EXTERNAL (separate Deployment in this dir, firecrawl pattern) # => postgresql.enabled: false, external.* points at hindsight-postgres:5432. # - Secrets come from 1Password via ExternalSecret => existingSecret: # hindsight-credentials. The chart does envFrom(secretRef) so # HINDSIGHT_API_LLM_API_KEY / HINDSIGHT_API_MCP_AUTH_TOKEN are injected # automatically; POSTGRES_PASSWORD is a secretKeyRef that K8s expands into # HINDSIGHT_API_DATABASE_URL (verified with a live envFrom test pod). # - LLM is the Nous free-tier inference endpoint # (https://inference-api.nousresearch.com/v1), model # `upstage/solar-pro4:free` (tool-calling, verified reflect). Fallback # (documented, NOT deployed): `stepfun/step-3.7-flash:free`. API key via existingSecret # envFrom (hindsight-credentials / HINDSIGHT_API_LLM_API_KEY), sourced from # 1Password `nous` item per decision 4. # - Ingress is driven through the chart's NATIVE ingress template (approved # plan: "Ingress driven through values.yaml"). api.service.port=8888, # controlPlane.service.port=3000. # - Image tag defaults to .Values.version (root) when api.image.tag is unset, # so version: "0.9.1" pins the API image to 0.9.1. # ============================================================================ version: "0.9.1" # ---------------------------------------------------------------------------- # External PostgreSQL (chart's bundled postgresql is disabled). # password is the K8s env expansion `$(POSTGRES_PASSWORD)` — the chart defines # POSTGRES_PASSWORD as a secretKeyRef (hindsight-credentials / postgres-password) # earlier in the same container, so K8s substitutes it at container start. # ---------------------------------------------------------------------------- postgresql: enabled: false external: host: hindsight-postgres port: 5432 username: hindsight database: hindsight password: $(POSTGRES_PASSWORD) # ExternalSecret (from 1Password) that the chart injects via envFrom(secretRef). # Keys it must expose: postgres-password, HINDSIGHT_API_LLM_API_KEY, # HINDSIGHT_API_MCP_AUTH_TOKEN. See externalsecret.yaml in this dir. existingSecret: hindsight-credentials # ---------------------------------------------------------------------------- # API container environment (explicit env entries; the chart renders this map # to individual env vars). LLM points at the Nous free-tier inference endpoint # (https://inference-api.nousresearch.com/v1), model upstage/solar-pro4:free # (tool-calling, verified retain/recall). stepfun/step-3.7-flash was swapped out # because it rejects Hindsight's tagged fact-extraction (BadRequestError 400 # 'missing tags'); solar-pro4 is the previously-verified-good Nous model for the # extract/retain path (tasks t_e3375410 / t_d0dffc3d). # HINDSIGHT_API_LLM_API_KEY is NOT set here — it comes from the existingSecret # via envFrom (1Password `nous` item). # ---------------------------------------------------------------------------- api: env: # Cutover to vLLM (t_e6facb19, 2026-08-31): astro-orbiter's llama-swap is # being retired for the completions role Hindsight needs. vLLM serves # Qwen2.5-32B-Instruct-AWQ (:8000, completions) — validated end-to-end # (health, /v1/chat/completions) in a shadow window with llama-swap # stopped. Model swap from Qwen3.8-27B to Qwen2.5-32B-Instruct-AWQ: both # are capable instruction-tuned models; confirm structured-extraction/ # tagging behavior (the prior stepfun/solar 400 "missing tags" failure # mode) holds up after this swap — see # references/hindsight-backend-fallback-to-local-qwen-workflow.md if it # regresses and llama-swap needs to come back temporarily. HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8000/v1" HINDSIGHT_API_LLM_PROVIDER: "openai" HINDSIGHT_API_LLM_MODEL: "Qwen2.5-32B-Instruct-AWQ" # vLLM's Qwen2.5-32B-Instruct-AWQ is deployed with --max-model-len 8192 # (roles/deploy-vllm/defaults/main.yml vllm_max_model_len; llama-swap's # Qwen3.8-27B ran at ctx=65536, matching Hindsight's default # RETAIN_MAX_COMPLETION_TOKENS=64000). Discovered via a live retain 500: # "max_tokens=64000 cannot be greater than max_model_len=8192" — every # retain call was requesting a 64K-token completion budget the new # backend physically cannot serve. Lowered to 4096, comfortably under # 8192 with room for the ~3000-char (RETAIN_CHUNK_SIZE) input chunk plus # prompt overhead. Verified with a live hindsight_retain call post-fix. HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS: "4096" # DO NOT set HINDSIGHT_API_EMBEDDINGS_* here (t_e6facb19, 2026-08-31 # attempted this, reverted after a production incident — see below). # # DISCOVERY: Hindsight's embeddings provider was NEVER pointed at # astro-orbiter. It defaults to "local" (bundled sentence-transformers, # BAAI/bge-small-en-v1.5, 384 dimensions) whenever # HINDSIGHT_API_EMBEDDINGS_PROVIDER is unset — verified via # `kubectl exec ... env | grep -i embed` showing NO # HINDSIGHT_API_EMBEDDINGS_* vars in the live pod, despite this file's # LLM section referencing astro-orbiter for years. The nomic-embed- # text-v1.5 model documented across mk-labs skills as "Hindsight's # embedding model" was OpenViking's embedding model, not Hindsight's. # # INCIDENT: pointing HINDSIGHT_API_EMBEDDINGS_PROVIDER at vLLM's # nomic-embed-text-v1.5 (768 dimensions) crash-looped hindsight-api on # rollout: `RuntimeError: Cannot change embedding dimension from 384 to # 768: memory_units table contains 1289 rows with embeddings.` The # migration path (`ensure_embedding_dimension` in migrations.py) refuses # a live dimension change without either re-embedding everything or # deleting all existing memory_units rows across every bank (jarvis, # hermes, war-machine, and ~18 other agent banks) — a destructive, # irreversible operation requiring explicit human approval, not # something to do as a side effect of an infra migration task. Reverted # immediately; Hindsight keeps its bundled local embedder (384-dim, # unchanged, zero data risk) until a deliberate, approved re-embedding # migration is planned as its own task. # --- t_d7f8cd65: fix 502s on the serial astro-orbiter node --- # astro-orbiter is a single llama-swap process (serial: 1 generate at a # time, ctx 64K). Hindsight's default LLM concurrency is 32, so a retain # burst hits the node with N parallel calls -> the node rejects/times out # the extras -> hindsight-api surfaces APITimeoutError as 502. astro-orbiter # is the ONLY LLM endpoint (all ops route there), so cap the whole pool to # 1 and pin retain to 1 as well. The upstream chart exposes these as native # semaphore config (HINDSIGHT_API_*_MAX_CONCURRENT); no code change needed. # Kept at 1 post-cutover: vLLM's single-process-per-model design is also # effectively serial for a single Qwen2.5-32B instance under this GPU's # VRAM budget (KV cache sized for low concurrency at max_model_len=8192). HINDSIGHT_API_LLM_MAX_CONCURRENT: "1" HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT: "1" # Client + per-request timeout. Default is 120s; a 29K-token retain runs # ~29s and under load a single long retain can reach ~90s. Raise to 600s to # cover the longest round-trip so the serial call never times out the client # (keep >= ingress proxy-read-timeout below). Per-op retain timeout pins the # retain path explicitly; the global timeout covers reflect/consolidation. HINDSIGHT_API_LLM_TIMEOUT: "600" HINDSIGHT_API_RETAIN_LLM_TIMEOUT: "600" # ---------------------------------------------------------------------------- # Ingress via the chart's native template. # /health,/v1,/mcp,/ext -> api:8888 (longest-prefix wins in nginx) # / -> controlPlane:3000 # TLS secret hindsight-tls provisioned by the letsencrypt-prod issuer. # ---------------------------------------------------------------------------- ingress: enabled: true className: "nginx" annotations: cert-manager.io/cluster-issuer: "letsencrypt-prod" # Raised read/send timeout so a slow agentic reflect / long single retain # (up to ~90s under load on the serial astro-orbiter node; client timeout # is 600s per t_d7f8cd65) can complete before nginx cuts the connection. # Raised 300 -> 600 (t_d7f8cd65) to cover the longest retain round-trip. nginx.ingress.kubernetes.io/proxy-read-timeout: "600" nginx.ingress.kubernetes.io/proxy-send-timeout: "600" hosts: - host: cosmic-rewind.local.mk-labs.cloud paths: - path: /health pathType: Prefix service: api - path: /v1 pathType: Prefix service: api - path: /mcp pathType: Prefix service: api - path: /ext pathType: Prefix service: api - path: / pathType: Prefix service: controlPlane tls: - hosts: - cosmic-rewind.local.mk-labs.cloud secretName: hindsight-tls