diff --git a/cluster/applications/hindsight/values.yaml b/cluster/applications/hindsight/values.yaml index 09c897d..9227866 100644 --- a/cluster/applications/hindsight/values.yaml +++ b/cluster/applications/hindsight/values.yaml @@ -66,6 +66,23 @@ api: HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8001/v1" HINDSIGHT_API_LLM_PROVIDER: "openai" HINDSIGHT_API_LLM_MODEL: "Qwen3.8-27B-Q4_K_M" + # --- t_d7f8cd65: fix 502s on the serial astro-orbiter node --- + # astro-orbiter is a single llama-swap process (serial: 1 generate at a + # time, ctx 64K). Hindsight's default LLM concurrency is 32, so a retain + # burst hits the node with N parallel calls -> the node rejects/times out + # the extras -> hindsight-api surfaces APITimeoutError as 502. astro-orbiter + # is the ONLY LLM endpoint (all ops route there), so cap the whole pool to + # 1 and pin retain to 1 as well. The upstream chart exposes these as native + # semaphore config (HINDSIGHT_API_*_MAX_CONCURRENT); no code change needed. + HINDSIGHT_API_LLM_MAX_CONCURRENT: "1" + HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT: "1" + # Client + per-request timeout. Default is 120s; a 29K-token retain runs + # ~29s and under load a single long retain can reach ~90s. Raise to 600s to + # cover the longest round-trip so the serial call never times out the client + # (keep >= ingress proxy-read-timeout below). Per-op retain timeout pins the + # retain path explicitly; the global timeout covers reflect/consolidation. + HINDSIGHT_API_LLM_TIMEOUT: "600" + HINDSIGHT_API_RETAIN_LLM_TIMEOUT: "600" # ---------------------------------------------------------------------------- # Ingress via the chart's native template. @@ -78,10 +95,12 @@ ingress: className: "nginx" annotations: cert-manager.io/cluster-issuer: "letsencrypt-prod" - # Raised read/send timeout so a slow agentic reflect (~132s on solar-pro4) - # can complete before nginx cuts the connection (was 60s default -> 504). - nginx.ingress.kubernetes.io/proxy-read-timeout: "300" - nginx.ingress.kubernetes.io/proxy-send-timeout: "300" + # Raised read/send timeout so a slow agentic reflect / long single retain + # (up to ~90s under load on the serial astro-orbiter node; client timeout + # is 600s per t_d7f8cd65) can complete before nginx cuts the connection. + # Raised 300 -> 600 (t_d7f8cd65) to cover the longest retain round-trip. + nginx.ingress.kubernetes.io/proxy-read-timeout: "600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "600" hosts: - host: cosmic-rewind.local.mk-labs.cloud paths: