From 2cc9370f3d66e89aeb5d7ebeb9036a1bd9af1c7f Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 31 Aug 2026 18:15:22 -0500 Subject: [PATCH] deploy-vllm: add embedding-mode support, cut over Hindsight to vLLM (t_e6facb19) - vllm.service.j2: branch on role==embedding for --runner pooling --convert embed, --no-enable-prefix-caching, per-model trust_remote_code toggle (needed for nomic-embed-text-v1.5's custom NomicBertModel code), and enforce_eager toggle (needed to avoid CUDA graph capture OOM when co-resident with another vLLM process on this 24GB card). - tasks/verify.yml: split completions vs embedding smoke tests -- embedding-mode instances don't serve /v1/completions. Assert a non-empty embedding vector, not just HTTP 200. - host_vars/astro-orbiter: enable nomic-embed-text-v1.5 (port 8020), lower primary model's gpu_memory_utilization 0.95->0.90 + add enforce_eager after finding 0.95 crash-looped 6-7x before stabilizing with co-resident nomic-embed (real fix, confirmed via NRestarts=0 after clean stop/start, not luck). - Hindsight (values.yaml + externalsecret.yaml): cut LLM + embeddings over to vLLM (:8000, :8020), wire the previously-unset HINDSIGHT_API_EMBEDDINGS_* env vars for the first time, and swap the API key secret source from the Nous fallback item to vllm/api-key (vLLM enforces real auth, llama-swap did not). - README: document the embedding-mode branch, VRAM findings, and a genuine architecture gap -- vLLM's one-model-per-process design cannot replace llama-swap's 5-model LRU roster on this 24GB card, so 21 Hermes profiles' aux-model consumers (Qwen3-8B-no_think, Phi-3.5-mini, Meta-Llama-3.1-8B, Qwen2.5-Coder-14B) and OpenViking's VLM stay on llama-swap. Full teardown (t_6dff1ecc) needs a human decision on the aux-model strategy before it can proceed. --- ansible/host_vars/astro-orbiter/vars.yml | 54 +++++++++ ansible/roles/deploy-vllm/README.md | 105 +++++++++++++++++- ansible/roles/deploy-vllm/defaults/main.yml | 4 + ansible/roles/deploy-vllm/tasks/verify.yml | 60 ++++++++-- .../deploy-vllm/templates/vllm.service.j2 | 14 +++ .../hindsight/externalsecret.yaml | 23 +++- cluster/applications/hindsight/values.yaml | 37 +++++- 7 files changed, 275 insertions(+), 22 deletions(-) diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index b50a34d..1ff07a6 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -109,3 +109,57 @@ llm_staged_models: size_bytes: 5027784224 source_repo: "bartowski/Qwen_Qwen3-8B-GGUF" +# --- deploy-vllm role: vllm_models override (t_e6facb19, 2026-08-31) -------- +# Ansible's hash_behaviour is "replace" (see ansible.cfg) — a host_vars list +# variable REPLACES the role default list wholesale, it does not deep-merge. +# This is therefore a full copy of roles/deploy-vllm/defaults/main.yml's +# vllm_models with ONE change: nomic-embed-text-v1.5.enabled flipped to true, +# now that vllm.service.j2 has an embedding-mode branch (--runner pooling +# --convert embed --trust-remote-code) tested end-to-end in a shadow window. +# Primary (Qwen2.5-32B-Instruct-AWQ) and aux (Qwen3-8B-AWQ) entries are +# unchanged from role defaults — reproduced here only because the whole list +# must be redefined together. Keep this in sync with defaults/main.yml if the +# role's model roster changes. +vllm_models: + - id: "Qwen2.5-32B-Instruct-AWQ" + hf_repo: "Qwen/Qwen2.5-32B-Instruct-AWQ" + role: primary + quantization: awq + port: 8000 + max_model_len: 8192 + # 0.95 (role default) OOM'd during CUDA graph capture once nomic-embed + # (role: embedding, ~814MiB actual, not the nominal 300MB) is co-resident + # on the same 24GB card (t_e6facb19, 2026-08-31): KV cache allocation + # succeeded (14,720 tokens) but graph capture needed ~20MiB more than the + # 0.95 budget left after nomic's share. Two independent, permanent + # co-residents (unlike t_ca1af9fb's shadow-window test, which had the + # whole 24GB free) need either a lower utilization ceiling or no graph + # capture. enforce_eager avoids the whole cudagraph capture memory spike + # entirely — small throughput cost, no OOM risk, safer for a fixed + # multi-process VRAM budget than tuning utilization percentages by hand. + # Even WITH enforce_eager, 0.95 left only ~847MiB genuinely free out of + # 24576MiB total (23,729MiB used) and both services crash-looped 6-7x + # during warmup/KV-cache sizing before stabilizing — too fragile for a + # permanent two-process co-residency. Lowered to 0.90 for real headroom + # (~1.6GiB free), confirmed clean single-attempt start with no retries. + gpu_memory_utilization: 0.90 + enforce_eager: true + enabled: true + - id: "Qwen3-8B-AWQ" + hf_repo: "Qwen/Qwen3-8B-AWQ" + role: aux + quantization: awq + port: 8010 + max_model_len: 32768 + gpu_memory_utilization: 0.15 + enabled: false + - id: "nomic-embed-text-v1.5" + hf_repo: "nomic-ai/nomic-embed-text-v1.5" + role: embedding + quantization: none + port: 8020 + max_model_len: 2048 + gpu_memory_utilization: 0.05 + trust_remote_code: true + enabled: true + diff --git a/ansible/roles/deploy-vllm/README.md b/ansible/roles/deploy-vllm/README.md index da9976a..656558a 100644 --- a/ansible/roles/deploy-vllm/README.md +++ b/ansible/roles/deploy-vllm/README.md @@ -82,11 +82,6 @@ supports (`--quantization awq`) and which fit the VRAM budget: ## Known Gaps / Follow-ups -- The `nomic-embed-text-v1.5` entry in `vllm_models` is present but the - `vllm.service.j2` template does not yet branch for embedding-mode flags - (`--task embed`). Do not flip `enabled: true` on it without first adding - that branch and testing `/v1/embeddings` — this is what Hindsight retain - actually depends on, so get it right before cutover. - Quarterly API key rotation is documented (`/etc/vllm/API_KEY_ROTATION.md` on the target, rendered by `tasks/api-key.yml`) but not automated — no cron job exists to force rotation on a schedule. Consider a follow-up cron task @@ -94,6 +89,106 @@ supports (`--quantization awq`) and which fit the VRAM budget: - `vllm_service_enabled` defaults to `false` deliberately — see "Deliberate staging-first default" above. Flip together with the cutover step, not before. +- **vLLM cannot replace llama-swap's full model roster on this card (t_e6facb19, 2026-08-31).** + This role only wires two of llama-swap's five served models: the primary + completions model (Qwen2.5-32B-Instruct-AWQ, replacing Qwen3.8-27B) and + the embedding model (nomic-embed-text-v1.5). llama-swap ALSO serves + Qwen3-8B-Q4_K_M(-no_think), Phi-3.5-mini-instruct-Q8_0, + Meta-Llama-3.1-8B-Instruct-Q4_K_M, and Qwen2.5-Coder-14B-Instruct-Q4_K_M — + 21 Hermes agent profiles' `custom_providers` reference these model IDs for + aux tasks (skills_hub, approval, mcp, title_generation, profile_describer, + compression). vLLM 0.28 serves **one model per process**; running 5-6 + separate vLLM processes concurrently does not fit a 24GB card (each + process reserves its own CUDA context + weights + KV cache, unlike + llama-swap's matrix DSL which time-shares one GPU across LRU-evicted + processes). **Full llama-swap teardown (t_6dff1ecc) cannot proceed until + either:** (a) the aux-model consumers are migrated to a different backend + (Anthropic, or a smaller local llama.cpp router kept alongside vLLM), or + (b) vLLM gains a comparable multi-model time-sharing mode. This is a + genuine architecture gap, not a missing role feature — flagging for a + human decision on the aux-model strategy before teardown can be + unconditionally safe. + +## Embedding-mode support (t_e6facb19, 2026-08-31) + +`vllm.service.j2` now branches on `role: embedding` entries in `vllm_models`: +adds `--runner pooling --convert embed` (vLLM's embedding-serving flags — +see https://docs.vllm.ai/en/latest/models/pooling_models/embed/) and +`--no-enable-prefix-caching` (prefix caching is a completions-only +optimization; irrelevant and safely disabled for pooling). An additional +per-model `trust_remote_code: true` toggle renders `--trust-remote-code` +when set — required for `nomic-ai/nomic-embed-text-v1.5`, which ships +custom `NomicBertModel` modeling code on its HF repo. + +**Verification does NOT run `/v1/completions` against embedding-mode +instances** (they don't serve that endpoint — a completions request 400s +immediately). `tasks/verify.yml` splits `vllm_enabled_models` by `role` and +runs the appropriate smoke test per group: completions models get the +`/v1/completions` "capital of France" test; embedding models get a real +`/v1/embeddings` POST with an `ansible.builtin.assert` on a non-empty +`data[0].embedding` array (not just HTTP 200 — an empty/malformed vector +would still 200). + +**Critical VRAM finding: co-resident completions + embedding vLLM processes +need MORE headroom than either alone, and CUDA graph capture is the failure +mode, not KV cache sizing.** Enabling `nomic-embed-text-v1.5` alongside the +primary Qwen2.5-32B model at the role-default `gpu_memory_utilization: 0.95` +crash-looped repeatedly: +- First failure: `torch.OutOfMemoryError` during `capture_model()` (CUDA + graph capture) — KV cache sizing itself succeeded (14,720 tokens + allocated), but graph capture needed ~20MiB more than the 0.95 budget had + left once nomic's embedding process (814MiB actual, not the nominal + ~300MB estimate in the model roster table) claimed its share. +- Fix attempt 1: added a per-model `enforce_eager: true` template branch + (`--enforce-eager` skips CUDA graph capture entirely) — this stopped the + graph-capture OOM but the combined processes still landed at only + ~847MiB genuinely free out of 24,576MiB, and both services crash-looped + 6-7 times during warmup before finally stabilizing (each attempt leaves + transient VRAM that the next attempt fights over, extending time-to-stable + well past a single health-check retry window). +- Fix attempt 2 (final, verified stable): lowered the primary model's + `gpu_memory_utilization` from 0.95 to **0.90** (host_vars override) in + addition to `enforce_eager: true`. Result: clean single-attempt start for + both services, `NRestarts=0`, ~2GB genuinely free (22,577MiB used / + 24,576MiB total). Confirmed via `systemctl show -p NRestarts` after + a full stop/start cycle — 0.95 was NOT a fluke of Restart=always masking + the underlying fragility; 0.90 is a real, reproducible fix. +- **Takeaway for future multi-process vLLM VRAM budgeting on this host:** + do not just check "does it eventually come up" — check `NRestarts` and + free VRAM headroom after a clean stop/start. A model that "works" after + 6 crash-loop retries is not production-stable; the retries themselves are + evidence the utilization ceiling is too tight for the actual (not + nominal) footprint of co-resident processes. + +## Consumer cutover status (t_e6facb19, 2026-08-31) + +**Cut over (validated end-to-end):** +- Hindsight (`cluster/applications/hindsight/values.yaml` + + `externalsecret.yaml`): `HINDSIGHT_API_LLM_BASE_URL` → vLLM `:8000` + (Qwen2.5-32B-Instruct-AWQ), plus newly-wired `HINDSIGHT_API_EMBEDDINGS_*` + env vars pointing at vLLM `:8020` (nomic-embed-text-v1.5). Both endpoints + require vLLM's real API key (unlike llama-swap, which accepted + any/no key) — ExternalSecret now reads `op://mk-labs/vllm/api-key` + (item "vllm") instead of the prior Nous fallback item, and reuses the same + key value for `HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY` (both vLLM + endpoints share one key file per `tasks/api-key.yml`). + +**NOT cut over — genuine scope gap requiring a human decision, see +"Known Gaps" above:** the 21 Hermes agent profiles' aux-model +`custom_providers` entries (Qwen3-8B-no_think, Phi-3.5-mini, Meta-Llama-3.1-8B, +Qwen2.5-Coder-14B) still point at llama-swap `:8001` — vLLM has no +equivalent multi-model serving mode on this 24GB card. llama-swap MUST stay +up to serve these until that gap is resolved. This is why the teardown task +(t_6dff1ecc) remains blocked even after this task's completion — see the +comment posted there. + +- OpenViking (`cluster/platform/openviking/values.yaml`): still points at + llama-swap `:8001` (`Meta-Llama-3.1-8B-Instruct-Q4_K_M` VLM + nomic-embed + for dense embeddings). Left unchanged — its VLM model has no vLLM + equivalent staged, and migrating only its embedding path while leaving its + VLM on llama-swap would still require llama-swap up, providing zero + teardown benefit. Flagged, not touched, per the same aux-model gap above. + ## Validation Log (2026-08-31, t_ca1af9fb) diff --git a/ansible/roles/deploy-vllm/defaults/main.yml b/ansible/roles/deploy-vllm/defaults/main.yml index c65c36e..50f48e0 100644 --- a/ansible/roles/deploy-vllm/defaults/main.yml +++ b/ansible/roles/deploy-vllm/defaults/main.yml @@ -55,6 +55,10 @@ vllm_models: port: 8020 max_model_len: 2048 gpu_memory_utilization: 0.05 + # NomicBertModel ships custom modeling code on the HF repo (rotary/ALiBi + # variant) — vLLM needs --trust-remote-code to load it, same requirement + # as sentence-transformers/llama.cpp. Wired into vllm.service.j2 (t_e6facb19). + trust_remote_code: true enabled: false # --- systemd --------------------------------------------------------------- diff --git a/ansible/roles/deploy-vllm/tasks/verify.yml b/ansible/roles/deploy-vllm/tasks/verify.yml index 67158a1..0bfa481 100644 --- a/ansible/roles/deploy-vllm/tasks/verify.yml +++ b/ansible/roles/deploy-vllm/tasks/verify.yml @@ -71,7 +71,12 @@ loop_control: label: "{{ item.item.id }}" -- name: Run a live completion smoke test against each enabled instance +- name: Split enabled models into completion-serving vs embedding for the right smoke test + ansible.builtin.set_fact: + vllm_completion_models: "{{ vllm_enabled_models | rejectattr('role', 'equalto', 'embedding') | list }}" + vllm_embedding_models: "{{ vllm_enabled_models | selectattr('role', 'equalto', 'embedding') | list }}" + +- name: Run a live completion smoke test against each completion-serving instance ansible.builtin.uri: url: "http://127.0.0.1:{{ item.port }}/v1/completions" method: POST @@ -86,7 +91,7 @@ temperature: 0 timeout: 60 status_code: 200 - loop: "{{ vllm_enabled_models }}" + loop: "{{ vllm_completion_models }}" loop_control: label: "{{ item.id }}" register: vllm_completion_test @@ -99,12 +104,50 @@ loop_control: label: "{{ item.item.id }}" -- name: Check journalctl for the primary unit is free of ERROR/Traceback since last start +# Embedding-mode vLLM instances (--runner pooling --convert embed) do NOT +# serve /v1/completions — only /v1/embeddings (and /pooling). A completions +# smoke test against one 400s immediately. Verify with a real vector request +# instead, and assert the response actually contains a non-empty float vector +# (not just HTTP 200 — an empty/malformed embedding would still 200). +- name: Run a live embeddings smoke test against each embedding-mode instance + ansible.builtin.uri: + url: "http://127.0.0.1:{{ item.port }}/v1/embeddings" + method: POST + headers: + Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}" + Content-Type: "application/json" + body_format: json + body: + model: "{{ item.id }}" + input: "The capital of France is Paris." + timeout: 60 + status_code: 200 + return_content: true + loop: "{{ vllm_embedding_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_embedding_test + no_log: true + +- name: Assert embeddings smoke test returned a non-empty float vector + ansible.builtin.assert: + that: + - (item.json.data[0].embedding | length) > 0 + fail_msg: "/v1/embeddings on port {{ item.item.port }} did not return a non-empty embedding vector" + success_msg: "/v1/embeddings confirmed {{ item.item.id }} returns a {{ item.json.data[0].embedding | length }}-dim vector" + loop: "{{ vllm_embedding_test.results }}" + loop_control: + label: "{{ item.item.id }}" + +- name: Check journalctl for each enabled unit is free of ERROR/Traceback since last start ansible.builtin.shell: | set -o pipefail - journalctl -u vllm.service --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true + journalctl -u {{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }} --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true args: executable: /bin/bash + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" register: vllm_journal_errors changed_when: false become: true @@ -112,6 +155,9 @@ - name: Report journalctl scan result ansible.builtin.debug: msg: >- - {{ 'journalctl clean — no error/traceback lines in the last 10 minutes' - if vllm_journal_errors.stdout | trim | length == 0 - else 'WARNING — journalctl lines matched error/traceback: ' + vllm_journal_errors.stdout }} + {{ item.item.id ~ ': journalctl clean — no error/traceback lines in the last 10 minutes' + if item.stdout | trim | length == 0 + else item.item.id ~ ' WARNING — journalctl lines matched error/traceback: ' ~ item.stdout }} + loop: "{{ vllm_journal_errors.results }}" + loop_control: + label: "{{ item.item.id }}" diff --git a/ansible/roles/deploy-vllm/templates/vllm.service.j2 b/ansible/roles/deploy-vllm/templates/vllm.service.j2 index 70daf6d..4d23bb3 100644 --- a/ansible/roles/deploy-vllm/templates/vllm.service.j2 +++ b/ansible/roles/deploy-vllm/templates/vllm.service.j2 @@ -34,6 +34,16 @@ ExecStart={{ vllm_venv_path }}/bin/python -m vllm.entrypoints.openai.api_server --served-model-name {{ item.id }} \ --host {{ vllm_serve_host }} \ --port {{ item.port }} \ +{% if item.role == 'embedding' %} + --runner pooling \ + --convert embed \ +{% endif %} +{% if item.trust_remote_code is defined and item.trust_remote_code %} + --trust-remote-code \ +{% endif %} +{% if item.enforce_eager is defined and item.enforce_eager %} + --enforce-eager \ +{% endif %} {% if item.quantization is defined and item.quantization != 'none' %} --quantization {{ item.quantization }} \ {% endif %} @@ -41,7 +51,11 @@ ExecStart={{ vllm_venv_path }}/bin/python -m vllm.entrypoints.openai.api_server --max-model-len {{ item.max_model_len }} \ --dtype {{ vllm_dtype }} \ --api-key ${VLLM_API_KEY} \ +{% if item.role != 'embedding' %} --enable-prefix-caching +{% else %} + --no-enable-prefix-caching +{% endif %} Restart={{ vllm_restart_policy }} RestartSec=10 diff --git a/cluster/applications/hindsight/externalsecret.yaml b/cluster/applications/hindsight/externalsecret.yaml index 1813803..8211a1f 100644 --- a/cluster/applications/hindsight/externalsecret.yaml +++ b/cluster/applications/hindsight/externalsecret.yaml @@ -55,6 +55,15 @@ spec: # env-var-named keys injected via envFrom HINDSIGHT_API_MCP_AUTH_TOKEN: "{{ .HINDSIGHT_API_MCP_AUTH_TOKEN }}" HINDSIGHT_API_LLM_API_KEY: "{{ .HINDSIGHT_API_LLM_API_KEY }}" + # Cutover to vLLM (t_e6facb19, 2026-08-31): vLLM enforces its API key + # on every request (unlike llama-swap, which accepted any/no key) — + # confirmed empirically, a bad/missing key gets a real 401 + # {"error":"Unauthorized"}. Both the completions endpoint (:8000) and + # the embeddings endpoint (:8020) are separate vLLM processes but + # share the SAME key (api-key.env is written once, read by both + # systemd units per roles/deploy-vllm/tasks/api-key.yml). Reuse + # HINDSIGHT_API_LLM_API_KEY's value for the embeddings key too. + HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY: "{{ .HINDSIGHT_API_LLM_API_KEY }}" data: # 1Password item "hindsight", field "postgres-password". # letters+digits only / URL-safe: required because the chart interpolates the @@ -69,11 +78,15 @@ spec: remoteRef: key: hindsight property: HINDSIGHT_API_MCP_AUTH_TOKEN - # 1Password item "nous" (vault mk-labs), field "api-key" (Ryan-provisioned, - # decision 4). Replaces the prior "local-placeholder" value — astro-orbiter - # did not validate; now wired to the Nous free-tier inference API. Materialized - # Secret key name stays HINDSIGHT_API_LLM_API_KEY (env-var-named, envFrom). + # Cutover to vLLM (t_e6facb19, 2026-08-31): astro-orbiter's llama-swap + # (which needed no real auth) is being retired for Hindsight's + # completions+embeddings roles. vLLM enforces a real API key — read the + # SAME key roles/deploy-vllm's api-key.yml phase writes to + # /etc/vllm/api-key.env on astro-orbiter, sourced from 1Password + # op://mk-labs/vllm/api-key (item "vllm", field "api-key", vault mk-labs). + # Replaces the prior "nous" item's api-key (Nous free-tier fallback, + # no longer the active LLM backend once this cutover lands). - secretKey: HINDSIGHT_API_LLM_API_KEY remoteRef: - key: nous + key: vllm property: api-key diff --git a/cluster/applications/hindsight/values.yaml b/cluster/applications/hindsight/values.yaml index 9227866..2b48dd5 100644 --- a/cluster/applications/hindsight/values.yaml +++ b/cluster/applications/hindsight/values.yaml @@ -60,12 +60,36 @@ existingSecret: hindsight-credentials # ---------------------------------------------------------------------------- api: env: - # Restore (2026-08-29, t_e0e6f7ca): astro-orbiter back online; move LLM back - # to local Qwen3.8-27B-Q4_K_M on llama-swap. Nous free tier returns 400 - # 'missing tags' on Hindsight structured fact-extraction (retain broken). - HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8001/v1" + # Cutover to vLLM (t_e6facb19, 2026-08-31): astro-orbiter's llama-swap is + # being retired for the completions+embeddings roles Hindsight needs. + # vLLM serves Qwen2.5-32B-Instruct-AWQ (:8000, completions) and + # nomic-embed-text-v1.5 (:8020, --runner pooling --convert embed) as two + # independent systemd-managed processes — validated end-to-end (health, + # /v1/chat/completions, /v1/embeddings returning a 768-dim vector) in a + # shadow window with llama-swap stopped. Model swap from Qwen3.8-27B to + # Qwen2.5-32B-Instruct-AWQ: both are capable instruction-tuned models: + # confirm structured-extraction/tagging behavior (the prior stepfun/solar + # 400 "missing tags" failure mode) holds up after this swap — see + # references/hindsight-backend-fallback-to-local-qwen-workflow.md if it + # regresses and llama-swap needs to come back temporarily. + HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8000/v1" HINDSIGHT_API_LLM_PROVIDER: "openai" - HINDSIGHT_API_LLM_MODEL: "Qwen3.8-27B-Q4_K_M" + HINDSIGHT_API_LLM_MODEL: "Qwen2.5-32B-Instruct-AWQ" + # Embeddings: llama.cpp nomic-embed-text-v1.5 (llama-swap, :8001) -> + # vLLM nomic-embed-text-v1.5 (:8020). Same model weights, different + # serving stack. Hindsight's embeddings provider defaults to "local" + # (bundled sentence-transformers) unless overridden — explicit TEI/openai + # provider config was never wired for astro-orbiter previously (verified: + # `kubectl exec ... env | grep -i embed` showed NO HINDSIGHT_API_EMBEDDINGS_* + # vars set, confirming Hindsight was using its bundled local embedder, NOT + # astro-orbiter, before this change — despite the OpenViking-style stack + # documentation implying otherwise). Wiring the openai-compatible provider + # here for the first time to point Hindsight's actual embedding generation + # at the vLLM-served nomic-embed-text-v1.5, matching the model OpenViking + # already uses (consistency across mk-labs services). + HINDSIGHT_API_EMBEDDINGS_PROVIDER: "openai" + HINDSIGHT_API_EMBEDDINGS_OPENAI_BASE_URL: "http://astro-orbiter:8020/v1" + HINDSIGHT_API_EMBEDDINGS_OPENAI_MODEL: "nomic-embed-text-v1.5" # --- t_d7f8cd65: fix 502s on the serial astro-orbiter node --- # astro-orbiter is a single llama-swap process (serial: 1 generate at a # time, ctx 64K). Hindsight's default LLM concurrency is 32, so a retain @@ -74,6 +98,9 @@ api: # is the ONLY LLM endpoint (all ops route there), so cap the whole pool to # 1 and pin retain to 1 as well. The upstream chart exposes these as native # semaphore config (HINDSIGHT_API_*_MAX_CONCURRENT); no code change needed. + # Kept at 1 post-cutover: vLLM's single-process-per-model design is also + # effectively serial for a single Qwen2.5-32B instance under this GPU's + # VRAM budget (KV cache sized for low concurrency at max_model_len=8192). HINDSIGHT_API_LLM_MAX_CONCURRENT: "1" HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT: "1" # Client + per-request timeout. Default is 120s; a 29K-token retain runs