Clean up duplicate OPENAI_BASE_URL env vars; keep only one

Set OPENAI_API_BASE_URL (config.py default) to our local vLLM endpoint.
Open WebUI was using hardcoded 'https://api.openai.com/v1' for model discovery.

ansible/host_vars/astro-orbiter/vars.yml

@@ -34,26 +34,27 @@ common_root_lv: ubuntu-lv
 # (t_33acbb2e) so the router can keep more than one GGUF resident on-demand
 # and LRU-evict when needed.
 #
-# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad, updated t_441470b9):
+# VRAM NOTE (t_33acbb2e, updated t_55c164f5, updated t_34b96e83, updated t_f5f7e9ad, updated t_441470b9, updated t_c5cef2b2):
-# With models-max=4 and all 5 GGUFs registered, worst case is all 5 loaded simultaneously:
+# With models-max=4 and all 6 GGUFs registered, worst case is all 6 loaded simultaneously:
-#   Qwen3.8-27B Q4_K_M:               ~23.3GB (weights ~17.1GB + KV ~6.2GB @ 128K ctx, q4_0) ← UPDATED
+#   Qwen3.8-27B Q4_K_M:               ~20.0GB (weights ~17.1GB + KV ~2.9GB @ 65536 ctx, q4_0) ← CORRECTED (ctx rolled back from 128K to 65536, t_c9fed26c 2026-08-18)
 #   Phi-3.5-mini-instruct Q8_0:       ~4.3GB  (weights ~3.8GB + KV ~0.5GB @ 32K ctx)
 #   Meta-Llama-3.1-8B Q4_K_M:        ~5.6GB  (weights ~4.6GB + KV ~0.2GB @ 8K ctx)
 #   Qwen2.5-Coder-14B Q4_K_M:        ~9.0GB  (weights ~8.4GB + KV ~0.6GB @ 16K ctx)
 #   nomic-embed-text-v1.5 Q4_K_M:    ~0.09GB (~84MB, embedding only — no KV cache)
-#   Total worst-case:                 ~42.3GB  >>  24GB RTX 3090
+#   Qwen3-8B Q4_K_M:                 ~5.5GB  (weights ~4.68GB + KV ~0.5GB @ 32K ctx, q4_0)
+#   Total worst-case:                 ~44.5GB  >>  24GB RTX 3090
 #
 # OOM RISK: Full co-residency is impossible on 24GB. LRU eviction prevents this
-# in practice: models-max=4 means the router can REGISTER 5 models but only keeps
+# in practice: models-max=4 means the router can REGISTER 6 models but only keeps
 # up to 4 LOADED simultaneously — the router will evict the LRU model when a new
 # one is needed. nomic-embed-text-v1.5 is pinned via sleep-idle-seconds=-1 and
 # load-on-startup=true but it uses only ~84MB, so it never meaningfully changes
 # the budget. In single-user homelab operation, only one generative model is active
 # at a time alongside the always-resident embedding model.
-# Qwen3.8-27B alone uses ~23.1GB (weights+KV); co-residency with Coder (~9GB) = ~32GB > 24GB.
+# Qwen3.8-27B alone uses ~17,804 MiB (weights+KV @ 65536 ctx); co-residency
-# LRU eviction handles this automatically — the router evicts the idle model before
+# with Coder (~9GB) = ~27GB > 24GB. LRU eviction handles this automatically.
-# loading the new one. Ryan should be aware this means model-switching always incurs
+# Ryan should be aware this means model-switching always incurs a ~30-60s
-# a ~30-60s cold-load latency when switching between Qwen3.8-27B and any other model.
+# cold-load latency when switching between Qwen3.8-27B and any other model.
 # Proceeding to models-max=4 as instructed; flagged for Ryan's attention.
 # Router --models-max override for astro-orbiter.
 # UPDATED (t_f5f7e9ad, 2026-08-16): Set to 2 because Qwen3.8-27B-Q4_K_M
@@ -67,13 +68,16 @@ common_root_lv: ubuntu-lv
 # This avoids an LRU eviction race with nomic-embed at startup.
 # UPDATED (t_72646029, 2026-08-17): CPU offload for Coder + Llama changes the
 # constraint. Coder and Llama now use CPU inference (n-gpu-layers=0). GPU-resident
-# VRAM: Qwen3.8 (~20,302 MiB at 128K ctx) + nomic-embed (558 MiB, pinned) plus the
+# VRAM: Qwen3.8 (~17,804 MiB at 65536 ctx) + nomic-embed (558 MiB, pinned) plus
-# CUDA-context buffers llama.cpp 6ea215d allocates for the CPU models (~1.4-1.7GB
+# the CUDA-context buffers llama.cpp 6ea215d allocates for the CPU models (~1.4-1.7GB
-# each) = ~24,004 MiB steady-state, below the 24,576 MiB physical limit.
+# each) = ~20,004 MiB steady-state, below the 24,576 MiB physical limit.
+# CORRECTED (t_c5cef2b2, 2026-08-19): ctx-size was rolled back from 131072 to 65536
+# (t_c9fed26c 2026-08-18). Qwen3.8 VRAM at 65536: 17,804 MiB (not 20,302 MiB).
 # models-max raised to 4: nomic (slot 1, pinned) + Qwen3.8 (slot 2, GPU) +
-# Llama (slot 3, CPU) + Coder (slot 4, CPU). Phi (GPU, ~8.3GB) can still be
+# Llama (slot 3, CPU) + Coder (slot 4, CPU). Phi (GPU, ~8.3GB) and new
-# requested but evicts Qwen3.8 due to VRAM constraint. models-max=4
+# Qwen3-8B (GPU, ~5.5GB) can also be requested but evict Qwen3.8 due to VRAM.
-# is required so CPU-offloaded models count as loaded without evicting Qwen3.8.
+# models-max=4 is required so CPU-offloaded models count as loaded without
+# evicting Qwen3.8.
 llm_router_models_max: 4
 
 llm_staged_models:
@@ -93,4 +97,139 @@ llm_staged_models:
     url: "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5-GGUF/resolve/main/nomic-embed-text-v1.5.Q4_K_M.gguf"
     size_bytes: 84106624
     source_repo: "nomic-ai/nomic-embed-text-v1.5-GGUF"
+  # Added t_c5cef2b2 (2026-08-19, War Machine): Qwen3-8B dense 8B model for
+  # aux tasks (routing, rewriting, structured extraction, tool-call construction).
+  # Source: bartowski/Qwen_Qwen3-8B-GGUF (public, no auth). HF filename is
+  # Qwen_Qwen3-8B-Q4_K_M.gguf; stored locally as Qwen3-8B-Q4_K_M.gguf.
+  # Exact size verified from HF manifest (content-length): 5,027,784,224 bytes.
+  # VRAM: ~4.68GB weights + ~0.5GB KV @ 32K ctx (q4_0) ≈ 5.2GB total.
+  # Thinking mode ON by default; use /no_think for latency-sensitive aux tasks.
+  - filename: "Qwen3-8B-Q4_K_M.gguf"
+    url: "https://huggingface.co/bartowski/Qwen_Qwen3-8B-GGUF/resolve/main/Qwen_Qwen3-8B-Q4_K_M.gguf"
+    size_bytes: 5027784224
+    source_repo: "bartowski/Qwen_Qwen3-8B-GGUF"
+
+# --- deploy-vllm role: vllm_models override (t_r1d32b_swap, 2026-09-01) -----
+# Ansible's hash_behaviour is "replace" (see ansible.cfg) — a host_vars list
+# variable REPLACES the role default list wholesale, it does not deep-merge.
+#
+# SWAP (Ryan direction, 2026-09-01): Qwen2.5-32B-Instruct-AWQ retired,
+# replaced with DeepSeek-R1-Distill-Qwen-32B-AWQ, max_model_len=32768.
+# "Single model only" — nomic-embed-text-v1.5 (embedding, :8020) and
+# Qwen3-8B-AWQ (aux, :8010, already disabled) are BOTH disabled here.
+# DeepSeek gets the full 24GB card to itself. Nothing in production
+# consumed nomic-embed at the time of this swap (Hindsight uses its own
+# bundled 384-dim embedder; OpenViking pointed at the old llama-swap
+# endpoint, already stopped) — confirmed with Ryan before disabling.
+#
+# Model choice: casperhansen/deepseek-r1-distill-qwen-32b-awq — same
+# quantizer/toolchain (AutoAWQ) as the outgoing Qwen2.5-32B-Instruct-AWQ,
+# widely used, 4-bit GEMM AWQ, ~19.3GB on disk (4 safetensors shards).
+# Architecture: Qwen2ForCausalLM (DeepSeek-R1 distilled onto Qwen2.5-32B
+# base) — same vLLM code path as the outgoing model, no new serving
+# support needed. Native max_position_embeddings=131072; we cap at 32768
+# per the task's explicit max-model-len requirement.
+#
+# VRAM math: ~19.3GB weights (4-bit AWQ) + KV cache at 32768 ctx (GQA,
+# 8 KV heads, 128 head_dim, 64 layers, fp16 KV by default) ≈ 19.3GB +
+# ~4GB KV+overhead ≈ 23.3GB — tight but the FULL 24GB card is now
+# available (no co-resident nomic-embed/Qwen3-8B taking a share, unlike
+# the outgoing Qwen2.5-32B config). gpu_memory_utilization=0.95 (role
+# default) + enforce_eager retained as the proven-stable mitigation from
+# t_e6facb19/t_ca1af9fb (avoids CUDA graph capture VRAM spike; this host's
+# only validated way to avoid crash-loop-to-stabilize behavior on this
+# card). If 0.95 OOMs at 32768 ctx once tested live, drop to 0.90 next
+# (documented fallback, same pattern as the outgoing model).
+#
+# DeepSeek-R1 output note: reasoning traces stream in <think> tags before
+# the final answer — this is expected R1-distill behavior, not a bug.
+# Model card recommends temperature 0.5-0.7 (not 0, not vLLM's greedy
+# default) to avoid repetition/incoherence; not set here (server-side
+# default), left to be set client-side per the model card's guidance —
+# flagging for whoever wires this into Hermes profile configs next.
+vllm_models:
+  - id: "Gemma-4-26B-A4B-it-AWQ"
+    hf_repo: "cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit"
+    role: primary
+    # NO quantization field set (unlike the AutoAWQ-quantized DeepSeek/
+    # Qwen2.5 models above) — live test (2026-09-01) found this repo's
+    # config.json declares quant_method: "compressed-tensors" (llm-compressor
+    # tool output, not classic AutoAWQ), even though the repo name says
+    # "AWQ-4bit". Passing --quantization awq explicitly caused a hard
+    # pydantic ValidationError at every single startup attempt: "Quantization
+    # method specified in the model config (compressed-tensors) does not
+    # match the quantization method specified in the `quantization` argument
+    # (awq)." vLLM auto-detects the quant method correctly from the model's
+    # own config.json when --quantization is omitted — confirmed fix, clean
+    # start. Lesson: don't trust a HF repo's naming convention ("...-AWQ...")
+    # for the `quantization:` field here — check config.json's quant_method.
+    port: 8000
+    # Ryan direction (2026-09-01, t_gemma4_swap): DeepSeek-R1-Distill-Qwen-32B
+    # retired after confirming its `auto` tool-choice reliability is a known,
+    # documented DeepSeek-R1-distillation limitation (trained on pure
+    # reasoning traces, no function-calling data — GitHub-confirmed upstream,
+    # not a vLLM config gap). Replaced with Gemma 4 26B A4B (Google,
+    # Apache 2.0, US-origin — matches Ryan's standing model-origin
+    # preference, unlike Qwen/DeepSeek). Chose MoE (26B A4B, 3.8B active)
+    # over the dense 31B variant: ~3.7GB smaller on-disk AWQ footprint
+    # (17.2GB vs 20.9GB) buys more KV-cache headroom on this tight 24GB
+    # card, and decode should be faster (memory-bandwidth-bound on active
+    # params, not total params). Tradeoff accepted: MoE scores lower than
+    # dense on the Tau2 tool-use benchmark (68.2% vs 76.9%) but still beats
+    # every other size in the family except the 31B on most reasoning
+    # benchmarks. Model choice: cyankiwi/gemma-4-26B-A4B-it-AWQ-4bit —
+    # AutoAWQ 4-bit group_size=32, MoE expert layers (gate/up/down/router)
+    # explicitly excluded from quantization ("ignore" list in config.json)
+    # per standard llm-compressor MoE quant practice — only the dense
+    # attention/projection layers are 4-bit, experts stay higher precision.
+    # Native architecture: Gemma4ForConditionalGeneration (registered
+    # natively in this host's installed vLLM 0.28.0 — vllm/model_executor/
+    # models/registry.py line 415 — no plugin/trust-remote-code needed).
+    # Native max_position_embeddings: 262144 (256K) — Hermes's 64K floor is
+    # comfortably covered without any context-extension trick.
+    max_model_len: 65536
+    # VRAM math (not yet live-validated — see swap validation log below
+    # once run): AWQ weights ~17.2GB on disk (dense attn 4-bit + MoE
+    # experts higher-precision, per config.json's compressed-tensors
+    # ignore list). Starting the KV cache dtype at int4_per_token_head
+    # from the outset (rather than fp16 -> fp8 -> int4 trial-and-error like
+    # the DeepSeek swap) since that same escalation pattern is expected to
+    # repeat on this VRAM-constrained card for any 20+ GB model at >32K ctx.
+    kv_cache_dtype: int4_per_token_head
+    gpu_memory_utilization: 0.95
+    enforce_eager: true
+    # Native tool-calling + reasoning support (no `hermes` workaround
+    # needed, unlike DeepSeek-R1-Distill): Gemma4EngineToolParser and
+    # Gemma4ParserReasoningAdapter are both registered natively in this
+    # host's vLLM 0.28.0 (vllm/tool_parsers/__init__.py,
+    # vllm/reasoning/__init__.py) — purpose-built for this model's actual
+    # output format, not a same-family approximation.
+    enable_auto_tool_choice: true
+    tool_call_parser: gemma4
+    reasoning_parser: gemma4
+    enabled: true
+  - id: "Qwen3-8B-AWQ"
+    hf_repo: "Qwen/Qwen3-8B-AWQ"
+    role: aux
+    quantization: awq
+    port: 8010
+    max_model_len: 32768
+    gpu_memory_utilization: 0.15
+    enforce_eager: true
+    enabled: false   # single-model deployment — see swap note above
+  - id: "nomic-embed-text-v1.5"
+    hf_repo: "nomic-ai/nomic-embed-text-v1.5"
+    role: embedding
+    quantization: none
+    port: 8020
+    max_model_len: 2048
+    gpu_memory_utilization: 0.05
+    trust_remote_code: true
+    enabled: false   # single-model deployment — see swap note above
+
+# --- deploy-vllm role: boot persistence (unchanged) -------------------------
+# Still permanent/boot-persistent — same policy as the outgoing Qwen2.5-32B
+# deployment (t_5508360a), just now serving one model instead of two.
+vllm_service_enabled: true
+vllm_service_state: started
 

ansible/playbooks/day1_deploy_vllm.yml

@@ -0,0 +1,18 @@
+# ------------------------------------------------------------------------------
+# FILE: playbooks/day1_deploy_vllm.yml
+# Deploy vLLM to a target host via roles/deploy-vllm.
+#
+# Staging run (deploy + validate WITHOUT touching production traffic):
+#   ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml --limit astro-orbiter
+#
+# Cutover run (once staging is validated and Ryan/JARVIS approve flipping
+# traffic — starts and enables the systemd unit(s), runs Phase 5 verification):
+#   ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml \
+#     --limit astro-orbiter --extra-vars "vllm_service_state=started"
+# ------------------------------------------------------------------------------
+- name: Deploy vLLM inference serving stack
+  hosts: astro-orbiter
+  become: false
+  gather_facts: true
+  roles:
+    - deploy-vllm

ansible/playbooks/day3_deploy_qwen38_ctx131k.yml

@@ -0,0 +1,22 @@
+---
+# Playbook: day3_deploy_qwen38_ctx131k.yml
+# Purpose: Deploy Qwen3.8-27B-Q4_K_M ctx-size 65536 -> 131072 to astro-orbiter
+#         via llama-swap config re-render + restart.
+#
+# The git change to defaults/main.yml (line 235: ctx_size: 131072) is already staged.
+# This playbook renders /etc/llama-swap/config.yaml from the updated defaults
+# and restarts llama-swap to load the new ctx-size.
+#
+# Run:
+#   cd /home/hermes/git/homelab/ansible
+#   ansible-playbook -i inventory.yml playbooks/day3_deploy_qwen38_ctx131k.yml
+#
+- name: Deploy Qwen3.8 ctx-size 131072 to astro-orbiter
+  hosts: astro-orbiter
+  become: true
+  vars:
+    llm_swapmode_enabled: true
+
+  roles:
+    - role: llm-inference-multimodel
+      tags: [swapmode_config, swapmode_systemd, swapmode_verify]

ansible/roles/common/defaults/main.yml

@@ -22,6 +22,7 @@ common_packages:
   - net-tools
   - dnsutils
   - lvm2
+  - cloud-guest-utils
 
 # ------------------------------------------------------------------------------
 # LVM root volume expansion

ansible/roles/deploy-vllm/README.md

@@ -0,0 +1,515 @@
+# deploy-vllm
+
+Idempotent Ansible role that deploys a vLLM OpenAI-compatible inference
+server. Written for astro-orbiter (RTX 3090, 24GB VRAM, 64GB RAM, Ubuntu
+24.04) and designed for reuse on the planned Mac Mini M4 host later this
+week (see "Portability" below).
+
+Supersedes the manual, pre-role state left behind by earlier vLLM
+experiments (`/home/jarvis/vllm-env`, bitsandbytes, gemma-2-27b — see
+`homelab-llm-inference`/`homelab-llm-serving` skills for that history). This
+role uses a **fresh venv** (`vllm_venv_path`, default `~/vllm-serve-env`) and
+**AWQ pre-quantized models** — no bitsandbytes, no on-the-fly quantization,
+no repeat of the OOM incident from the earlier Gemma-2-27B attempt.
+
+## Phases
+
+| Phase | File | What it does |
+|---|---|---|
+| 1 | `tasks/dependencies.yml` | System Python 3.10+, dedicated venv, `pip install vllm>=0.5.0`, verifies `nvidia-smi` and `torch.cuda.is_available()` |
+| 2 | `tasks/models.yml` | Downloads each `enabled: true` model in `vllm_models` via `hf download` (huggingface_hub CLI) into `~/.vllm-cache`, verifies the snapshot landed and reports on-disk size |
+| 3 | `tasks/api-key.yml` | Reads the API key from 1Password (`op://mk-labs/vllm/api-key`) on the **controller**, writes it to `/etc/vllm/api-key.env` (root:root, 0600) on the target |
+| 4 | `tasks/systemd.yml` | Renders and installs one systemd unit per enabled model (`vllm.service` for the `role: primary` model, `vllm-<id>.service` for others) |
+| 5 | `tasks/verify.yml` | Only runs when `vllm_service_state=started`. Waits for `/health` (up to 5 min — torch.compile warmup), checks `/v1/models`, runs a live completion, scans `journalctl` for errors |
+
+Run all phases: `ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml --limit astro-orbiter`
+Run one phase: `--tags vllm-dependencies` / `vllm-models` / `vllm-api-key` / `vllm-systemd` / `vllm-verify`
+
+## Deliberate staging-first default
+
+`vllm_service_state` defaults to `stopped`. A default run **stages
+everything** (venv, model weights, API key file, systemd unit) but does
+**not** start the service or touch production traffic. This matches the
+astro-orbiter cutover plan: llama-swap is live production serving (Qwen3.8-27B
++ nomic-embed for Hindsight) — vLLM must be deployed and validated on a
+side port/inactive unit before anything is cut over.
+
+To start and validate:
+
+```bash
+ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml \
+  --limit astro-orbiter --extra-vars "vllm_service_state=started"
+```
+
+This starts the systemd unit(s), enables them, and runs Phase 5 verification
+(health, `/v1/models`, live completion, clean journalctl).
+
+**Cutover of consumers (Hermes profiles, Hindsight embedding config, any
+hardcoded `:8001`/`:5805` references) to the new `:8000` vLLM endpoint is a
+separate, explicit step outside this role** — do this only after Phase 5
+passes cleanly. Do not tear down llama-swap until consumers are confirmed
+working end-to-end against vLLM.
+
+## Model roster (`vllm_models` in defaults/main.yml)
+
+vLLM 0.5.x-0.28.x serves **one model per process** — multi-model = multiple
+systemd units on distinct ports, not a single multiplexed server (unlike
+llama-swap's matrix DSL). Today's phase enables only the primary model;
+flip `enabled: true` on the others as VRAM allows (see "Phased Strategy"):
+
+**⚠️ Table below reflects the ORIGINAL Qwen2.5-32B deployment. As of
+2026-09-01 (t_r1d32b_swap) the primary model is
+`DeepSeek-R1-Distill-Qwen-32B-AWQ`, single-model only (nomic-embed also
+disabled) — see the "SUPERSEDED" section further down for current state.**
+
+| id | hf_repo | role | port | quant | enabled |
+|---|---|---|---|---|---|
+| Qwen2.5-32B-Instruct-AWQ | Qwen/Qwen2.5-32B-Instruct-AWQ | primary | 8000 | awq | **true** |
+| Qwen3-8B-AWQ | Qwen/Qwen3-8B-AWQ | aux | 8010 | awq | false |
+| nomic-embed-text-v1.5 | nomic-ai/nomic-embed-text-v1.5 | embedding | 8020 | none | false |
+
+**Note on the original spec's model choices:** the task body named
+`Qwen/Qwen2.5-32B-Instruct` and `Qwen/Qwen3-8B-Instruct` (bf16, unquantized).
+vLLM does not do on-the-fly quantization safely on this host (bitsandbytes
+OOM history — see `homelab-llm-inference` skill Pitfalls) and unquantized
+bf16 32B does not fit a 24GB card at all (~65GB). This role instead deploys
+the **official Qwen AWQ pre-quantized variants**
+(`Qwen/Qwen2.5-32B-Instruct-AWQ`, `Qwen/Qwen3-8B-AWQ`), which vLLM natively
+supports (`--quantization awq`) and which fit the VRAM budget:
+
+- Qwen2.5-32B-Instruct-AWQ: ~19.3GB on disk, fits with ~5GB headroom at 24GB
+- Qwen3-8B-AWQ: ~6GB VRAM per llm-explorer
+- nomic-embed-text-v1.5: ~300MB, vLLM serves it via `--convert embed` pooling
+  (see vLLM embedding docs) — **not yet wired into this role's systemd
+  template**; the embedding model needs `--task embed` / `--convert embed`
+  flags that differ from the completion-serving template. Flagged as a
+  follow-up before `enabled: true` is flipped on it (see Known Gaps below).
+
+## Known Gaps / Follow-ups
+
+- Quarterly API key rotation is documented (`/etc/vllm/API_KEY_ROTATION.md`
+  on the target, rendered by `tasks/api-key.yml`) but not automated — no cron
+  job exists to force rotation on a schedule. Consider a follow-up cron task
+  if Nick Fury wants this enforced rather than just documented.
+- `vllm_service_enabled` defaults to `false` deliberately — see "Deliberate
+  staging-first default" above. Flip together with the cutover step, not
+  before.
+- **vLLM cannot replace llama-swap's full model roster on this card — see
+  "Critical architectural finding" section below for the full incident.**
+  Short version: vLLM's one-model-per-process design plus llama-swap's own
+  VRAM needs exceed this 24GB card's capacity when both must serve real
+  models simultaneously. Full llama-swap teardown (t_6dff1ecc) cannot
+  proceed until a human decides the aux-model + VRAM strategy.
+
+## Embedding-mode support (t_e6facb19, 2026-08-31)
+
+`vllm.service.j2` now branches on `role: embedding` entries in `vllm_models`:
+adds `--runner pooling --convert embed` (vLLM's embedding-serving flags —
+see https://docs.vllm.ai/en/latest/models/pooling_models/embed/) and
+`--no-enable-prefix-caching` (prefix caching is a completions-only
+optimization; irrelevant and safely disabled for pooling). An additional
+per-model `trust_remote_code: true` toggle renders `--trust-remote-code`
+when set — required for `nomic-ai/nomic-embed-text-v1.5`, which ships
+custom `NomicBertModel` modeling code on its HF repo.
+
+**Verification does NOT run `/v1/completions` against embedding-mode
+instances** (they don't serve that endpoint — a completions request 400s
+immediately). `tasks/verify.yml` splits `vllm_enabled_models` by `role` and
+runs the appropriate smoke test per group: completions models get the
+`/v1/completions` "capital of France" test; embedding models get a real
+`/v1/embeddings` POST with an `ansible.builtin.assert` on a non-empty
+`data[0].embedding` array (not just HTTP 200 — an empty/malformed vector
+would still 200).
+
+**Critical VRAM finding: co-resident completions + embedding vLLM processes
+need MORE headroom than either alone, and CUDA graph capture is the failure
+mode, not KV cache sizing.** Enabling `nomic-embed-text-v1.5` alongside the
+primary Qwen2.5-32B model at the role-default `gpu_memory_utilization: 0.95`
+crash-looped repeatedly:
+- First failure: `torch.OutOfMemoryError` during `capture_model()` (CUDA
+  graph capture) — KV cache sizing itself succeeded (14,720 tokens
+  allocated), but graph capture needed ~20MiB more than the 0.95 budget had
+  left once nomic's embedding process (814MiB actual, not the nominal
+  ~300MB estimate in the model roster table) claimed its share.
+- Fix attempt 1: added a per-model `enforce_eager: true` template branch
+  (`--enforce-eager` skips CUDA graph capture entirely) — this stopped the
+  graph-capture OOM but the combined processes still landed at only
+  ~847MiB genuinely free out of 24,576MiB, and both services crash-looped
+  6-7 times during warmup before finally stabilizing (each attempt leaves
+  transient VRAM that the next attempt fights over, extending time-to-stable
+  well past a single health-check retry window).
+- Fix attempt 2 (final, verified stable): lowered the primary model's
+  `gpu_memory_utilization` from 0.95 to **0.90** (host_vars override) in
+  addition to `enforce_eager: true`. Result: clean single-attempt start for
+  both services, `NRestarts=0`, ~2GB genuinely free (22,577MiB used /
+  24,576MiB total). Confirmed via `systemctl show <unit> -p NRestarts` after
+  a full stop/start cycle — 0.95 was NOT a fluke of Restart=always masking
+  the underlying fragility; 0.90 is a real, reproducible fix.
+- **Takeaway for future multi-process vLLM VRAM budgeting on this host:**
+  do not just check "does it eventually come up" — check `NRestarts` and
+  free VRAM headroom after a clean stop/start. A model that "works" after
+  6 crash-loop retries is not production-stable; the retries themselves are
+  evidence the utilization ceiling is too tight for the actual (not
+  nominal) footprint of co-resident processes.
+
+## Consumer cutover status (t_e6facb19, 2026-08-31)
+
+**Attempted, then REVERTED — Hindsight LLM cutover.** Hindsight's
+`HINDSIGHT_API_LLM_BASE_URL` was pointed at vLLM `:8000`
+(Qwen2.5-32B-Instruct-AWQ) and validated working in isolation: health,
+`/v1/chat/completions`, and a live `hindsight_retain` + recall round-trip
+all succeeded (after also fixing `HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS`,
+which defaulted to 64000 — exceeding vLLM's `max_model_len=8192` — down to
+4096). **Reverted anyway**, because of a severe discovery documented in the
+next section: vLLM cannot stay resident on this card without starving
+llama-swap, and Hindsight's LLM endpoint needs continuous availability, not
+just a validation window. Restored to `http://astro-orbiter:8001/v1`
+(llama-swap, Qwen3.8-27B-Q4_K_M) — the pre-task working state.
+
+**NOT cut over — embeddings.** Hindsight was discovered to have NEVER used
+astro-orbiter for embeddings — it defaults to a bundled local
+`BAAI/bge-small-en-v1.5` (384-dim) embedder whenever
+`HINDSIGHT_API_EMBEDDINGS_PROVIDER` is unset, which was always the case here.
+Pointing it at vLLM's `nomic-embed-text-v1.5` (768-dim) crash-looped the pod:
+`RuntimeError: Cannot change embedding dimension from 384 to 768:
+memory_units table contains 1289 rows with embeddings.` Re-embedding all
+existing memory data across ~20 agent banks is destructive and irreversible
+— reverted immediately, left as a separate, explicitly-approved future task.
+
+**NOT cut over — 21 Hermes agent profiles' aux models + OpenViking VLM.**
+See "Critical architectural finding" below — this was never attempted once
+the VRAM collision was discovered, would have made things categorically
+worse.
+
+## Critical architectural finding: vLLM CANNOT be continuously resident alongside llama-swap on this 24GB card (t_e6facb19, 2026-08-31)
+
+After validating vLLM's two processes (Qwen2.5-32B-Instruct-AWQ + nomic-embed-
+text-v1.5, ~22.8GB combined) work correctly in isolation, this role's
+`vllm_service_enabled`/`vllm_service_state` were flipped to `true`/`started`
+as host_vars overrides to make the deployment permanent (per the task's
+"enable for boot" requirement) — llama-swap was then restarted alongside
+vLLM to preserve its own consumers. **Result: llama-swap could no longer
+load ANY of its own generative models.** Every `/v1/chat/completions`
+request against `Qwen3.8-27B-Q4_K_M` or the `Qwen3-8B` aux models failed
+with `{"error":"unspecific error: upstream command exited prematurely",
+"src":"llama-swap"}` — llama-server's own OOM at spawn time, only ~1.8GB
+free on a 24GB card once vLLM's ~22.8GB was already claimed.
+
+**Confirmed by direct A/B test, not inference:** identical
+`Qwen3.8-27B-Q4_K_M` chat completion request returned HTTP 500 with vLLM's
+two processes running, then HTTP 200 with a real completion within seconds
+of `systemctl stop vllm.service vllm-nomic-embed-text-v1.5.service` — same
+llama-swap process, same request, only the GPU memory pressure changed.
+
+**This is a hard architectural collision, not a tunable-parameter problem.**
+llama-swap needs ~18-20GB for its own primary model (Qwen3.8-27B-Q4_K_M);
+vLLM's two processes need ~22.8GB even with `enforce_eager` and a lowered
+`gpu_memory_utilization`. The two together need more VRAM than a 24GB card
+has once both hold real models resident — there is no `gpu_memory_utilization`
+value that resolves this while both stacks serve real production models
+simultaneously.
+
+**Consequence — reverted the boot-persistence flip.** `vllm_service_enabled`
+and `vllm_service_state` are back to role defaults (`false`/`stopped`) in
+`host_vars/astro-orbiter/vars.yml`. vLLM stays staged (venv, model weights,
+systemd units all in place) and can be started for a brief shadow-validation
+window (same pattern as t_ca1af9fb's original Phase 5), but is NOT safe to
+leave resident in production alongside llama-swap.
+
+**Path forward — requires a human decision, not more role tuning:**
+1. Full llama-swap teardown (t_6dff1ecc) BEFORE vLLM gets permanent
+   residency — but that breaks the 21 agent profiles' aux-model tasks and
+   OpenViking's VLM unless those consumers are migrated to a different
+   backend first (Anthropic API, a second smaller local box, or a
+   redesigned single-process serving strategy that covers all the models
+   vLLM and llama-swap currently split between them).
+2. Accept vLLM as a shadow-only / on-demand stack (manually started for
+   specific validated windows, stopped otherwise) and do NOT attempt
+   permanent Hindsight cutover — keeps llama-swap as the sole continuous
+   production serving layer, matching the pre-task state.
+3. A hardware change (larger GPU, or a second GPU) — out of scope for this
+   task, flagging for Ryan's awareness if the aux-model consumer set is
+   expected to grow.
+
+Comment posted on t_6dff1ecc with this finding — the teardown task remains
+correctly blocked; this task's completion does NOT unblock it, because full
+cutover to vLLM is not achievable within this card's VRAM budget as
+currently scoped.
+
+## RESOLVED (t_5508360a, 2026-08-31/09-01): Dashboard decision applied — llama-swap retired, vLLM permanent, 2 of 3 models
+
+Human decision (dashboard, kanban t_5508360a): **"stop and disable llama-swap
+and start vLLM and its 3 models"** — explicit approval, "I understand this is
+a breaking change." Chose path 1 from the three options above: retire
+llama-swap, give vLLM permanent residency, accept that the 21 Hermes
+profiles' aux-model consumers lose their llama-swap aux roster (Qwen3-8B,
+Phi-3.5-mini, Meta-Llama-3.1-8B, Qwen2.5-Coder-14B — all 4 gone) in exchange
+for vLLM's stack. Mid-run the dashboard added a course-correction: **"Don't
+try to load all 3 models concurrently on first deploy. Start with
+Qwen2.5-32B only"** — received after the 3-model attempt below had already
+run and self-corrected to the same 2-model end state, so no further action
+needed, but noted for the record.
+
+**Executed:**
+1. `sudo systemctl stop llama-swap && sudo systemctl disable llama-swap` on
+   astro-orbiter — confirmed inactive+disabled, VRAM dropped to 9MiB/24576MiB
+   (from 20.6GB in production use).
+2. Flipped `vllm_service_enabled`/`vllm_service_state` to `true`/`started` in
+   `host_vars/astro-orbiter/vars.yml` — vLLM is now the permanent,
+   boot-persistent serving layer (was shadow-only/staged before this task).
+3. **Attempted the literal "3 models" instruction** — flipped
+   `Qwen3-8B-AWQ.enabled` to `true` too. **Does not fit.** With the 24GB
+   card's usable 23.55GiB budget consumed by Qwen2.5-32B-Instruct-AWQ
+   (~18.6GB weights) + nomic-embed-text-v1.5 (~0.8GB actual), only ~1.25GiB
+   remained free — below the 3.53GiB floor `gpu_memory_utilization=0.15`
+   requires for Qwen3-8B-AWQ even with `enforce_eager`. Confirmed via
+   `journalctl`: identical `ValueError: Free memory on device cuda:0
+   (1.25/23.55 GiB) on startup is less than desired GPU memory utilization`
+   on all 7 consecutive systemd restart attempts — not the transient
+   CUDA-graph-capture crash-loop t_e6facb19 solved with enforce_eager, a hard
+   ceiling. Stopped + disabled `vllm-Qwen3-8B-AWQ.service`, reverted
+   `enabled: false` in host_vars with a full writeup in the comment block.
+4. Re-ran `day1_deploy_vllm.yml --extra-vars vllm_service_state=started`
+   with the corrected 2-model config: **clean idempotent pass, changed=0** on
+   both remaining models, Phase 5 verification passed (`/health` 200 on both
+   `:8000` and `:8020`, `/v1/models` correct, live completion + live
+   embeddings smoke tests both passed), `NRestarts=0` on both services.
+5. **Cut over Hindsight's LLM endpoint** (the other production consumer):
+   `HINDSIGHT_API_LLM_BASE_URL` llama-swap `:8001` → vLLM `:8000`,
+   `HINDSIGHT_API_LLM_MODEL` → `Qwen2.5-32B-Instruct-AWQ`, added
+   `HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS=4096` (vLLM's
+   `max_model_len=8192` vs Hindsight's 64000 default), and switched the
+   ExternalSecret's `HINDSIGHT_API_LLM_API_KEY` source from the unused
+   `nous` 1Password item to `vllm`'s real `api-key` (vLLM validates its
+   bearer token; llama-swap never did). Committed to
+   `cluster/applications/hindsight/{values.yaml,externalsecret.yaml}`,
+   pushed, ArgoCD synced, confirmed the new pod logged `Connection verified:
+   openai/Qwen2.5-32B-Instruct-AWQ` on boot.
+6. **Live end-to-end verification**, not inference: a real
+   `POST /v1/default/banks/war-machine/memories` retain call against the
+   production Hindsight endpoint returned `HTTP 200` with genuine
+   fact-extraction token usage (3257 in / 245 out), and a subsequent
+   `POST .../memories/recall` returned real semantically-ranked results
+   including the just-retained memory.
+
+**Final production state on astro-orbiter (verified live):**
+- `vllm.service` (Qwen2.5-32B-Instruct-AWQ, :8000): active, enabled, boot-persistent
+- `vllm-nomic-embed-text-v1.5.service` (:8020): active, enabled, boot-persistent
+- `vllm-Qwen3-8B-AWQ.service` (:8010): inactive, disabled — does not fit, see above
+- `llama-swap.service`: inactive, disabled (unit files left in place —
+  full removal is t_6dff1ecc's job, tracked separately)
+- VRAM: ~22.6GB/24.576GB in steady-state use, no crash-looping
+
+**What this means for t_6dff1ecc (teardown) and the 21 aux-model profiles:**
+llama-swap is now stopped+disabled — t_6dff1ecc's actual teardown steps
+(remove systemd unit files, wipe caches) are now safe to execute and
+unblocked from a "live production" standpoint. However, this trades away
+the aux-model roster: the 21 Hermes profiles' aux-model tasks (skills_hub,
+approval, mcp, title_generation, profile_describer, compression) that
+used to route to llama-swap's Qwen3-8B/Phi-3.5-mini/Meta-Llama/Coder
+models now have **zero local aux-model backend** — Qwen3-8B-AWQ doesn't
+fit vLLM's VRAM budget either. This was accepted explicitly by the
+dashboard ("I understand this is a breaking change") — no further local
+aux-model migration was authorized or attempted in this task. If those 21
+profiles need a replacement aux-model path, that is separate, new,
+explicitly-scoped follow-up work, not implied by this decision.
+
+## SUPERSEDED (t_r1d32b_swap, 2026-09-01): Qwen2.5-32B-Instruct-AWQ retired, replaced with DeepSeek-R1-Distill-Qwen-32B-AWQ, single-model deployment
+
+Ryan direction: "Swap Qwen2.5-32B for DeepSeek-R1-Distill-Qwen-32B,
+max-model-len 32768. Single model only." Confirmed with Ryan that "single
+model only" includes disabling `nomic-embed-text-v1.5` (:8020) as well —
+nothing in production consumed it (Hindsight uses its own bundled 384-dim
+embedder; OpenViking pointed at the retired llama-swap endpoint). DeepSeek
+gets the entire 24GB card.
+
+**Model choice:** `casperhansen/deepseek-r1-distill-qwen-32b-awq` — same
+AutoAWQ toolchain/quant style as the outgoing Qwen2.5-32B-Instruct-AWQ,
+widely-used community quant, `Qwen2ForCausalLM` architecture (DeepSeek-R1
+reasoning distilled onto a Qwen2.5-32B base) — no new vLLM code path
+required. Native `max_position_embeddings: 131072`; capped at 32768 per
+the task's explicit requirement.
+
+**Executed:**
+1. Stopped + disabled `vllm-nomic-embed-text-v1.5.service` (single-model
+   requirement), freed its ~19GB Qwen2.5-32B model cache on disk (30GB
+   free → 48GB free) to make room for DeepSeek's ~19.3GB download.
+2. Replaced `vllm_models` in `host_vars/astro-orbiter/vars.yml`: primary
+   entry now `DeepSeek-R1-Distill-Qwen-32B-AWQ`, aux (`Qwen3-8B-AWQ`) and
+   embedding (`nomic-embed-text-v1.5`) both `enabled: false`.
+3. Staged the model via `--tags vllm-models` (idempotent `hf download`,
+   ~19GB, confirmed via `du -sh` and snapshot-dir stat).
+4. **Three rounds of live VRAM-fit debugging** before a stable config was
+   found (documented inline in host_vars comments) — worth recording here
+   since the failure mode is non-obvious and will recur for future
+   32B-class models at high context on this 24GB card:
+   - **Round 1 (fp16 KV, gpu_memory_utilization 0.90/0.95/0.98):** vLLM's
+     own pre-flight check reported 18.17GiB weights + 8.0GiB KV cache
+     needed at 32768 ctx fp16 = 26.17GB — mathematically impossible on a
+     24GB card at ANY utilization percentage. Crash-looped every attempt.
+   - **Round 2 (`--kv-cache-dtype fp8`):** halved nominal KV cache to
+     ~4.0-4.3GiB, should fit with ~1GB margin. Still OOM'd — small
+     (~50-150MB) `cudaMalloc` failures during FlashInfer kernel warmup,
+     consistently, even when vLLM's own pre-flight math said it should
+     fit. Root cause: real GPU usage during warmup kernel compilation
+     exceeds what upfront profiling/reservation accounts for by roughly
+     ~1GB (unaccounted FlashInfer/sampler warmup workspace buffers).
+     Tried both the percentage knob AND vLLM's own suggested
+     `--kv-cache-memory-bytes` exact value — same failure either way,
+     confirming the gap wasn't a rounding/estimation error in the
+     percentage math, it was a real missing ~1GB of margin.
+   - **Round 3 (`--kv-cache-dtype int4_per_token_head`, fixed): SUCCESS.**
+     Switching from 8-bit to 4-bit KV cache roughly halves the KV
+     footprint again (~2GiB instead of ~4-4.3GiB), buying back enough
+     real headroom to absorb the unaccounted warmup overhead. Clean
+     single-attempt start, `NRestarts=0`, steady-state VRAM 23.2GB/24.576GB.
+5. **Full Ansible verify phase (`--tags vllm-api-key,vllm-verify`)**
+   passed: systemd unit active, `/health` 200, `/v1/models` returns
+   `DeepSeek-R1-Distill-Qwen-32B-AWQ` with `max_model_len: 32768`, live
+   `/v1/completions` smoke test HTTP 200, clean restart + re-run of
+   `--tags vllm-systemd` confirmed idempotent (`changed=0`,
+   `NRestarts=0`, same `ActiveEnterTimestamp` — no unnecessary restart).
+6. **Manual end-to-end generation test**, not inference: a real
+   `/v1/chat/completions` call ("What is 12*8?") returned a genuine
+   DeepSeek-R1 reasoning trace in `<think>` tags followed by the correct
+   answer (96) with correct step-by-step arithmetic shown — confirms the
+   model is not just health-check-alive but actually reasoning correctly.
+
+**Role/template changes (reusable for future models on this host):**
+- Added `kv_cache_dtype` (renders `--kv-cache-dtype`) and
+  `kv_cache_memory_bytes` (renders `--kv-cache-memory-bytes`) as new
+  optional per-model fields in `vllm.service.j2` — both are `{% if
+  ... is defined %}` guarded, no effect on models that don't set them.
+
+**Final production state on astro-orbiter (verified live, 2026-09-01):**
+- `vllm.service` (DeepSeek-R1-Distill-Qwen-32B-AWQ, :8000, `max_model_len:
+  32768`, `kv_cache_dtype: int4_per_token_head`): active, enabled,
+  boot-persistent, single model on the card
+- `vllm-nomic-embed-text-v1.5.service` (:8020): inactive, disabled
+- `vllm-Qwen3-8B-AWQ.service` (:8010): inactive, disabled (unchanged from prior state)
+- `llama-swap.service`: inactive, disabled (unchanged from prior state)
+- VRAM: ~23.2GB/24.576GB steady-state, no crash-looping, `NRestarts=0`
+
+**Not done in this task (flagging, not implied by this swap):**
+- Hindsight's `HINDSIGHT_API_LLM_MODEL` / `HINDSIGHT_API_LLM_BASE_URL`
+  cluster config still references `Qwen2.5-32B-Instruct-AWQ` — that model
+  is now gone from the card. Hindsight's LLM calls to astro-orbiter will
+  fail model-not-found until that GitOps config is updated to point at
+  `DeepSeek-R1-Distill-Qwen-32B-AWQ`. Not touched here — task scope was
+  the astro-orbiter model swap itself, cluster consumer cutover is a
+  separate, explicit follow-up (same boundary respected in the prior
+  t_5508360a section: this role does not own cluster-side config).
+- DeepSeek-R1's reasoning output uses `<think>` tags and the model card
+  recommends temperature 0.5-0.7 (not greedy/0) — neither is enforced
+  server-side; any consumer wiring this model into a Hermes profile or
+  application should account for both when parsing responses.
+
+## Validation Log (2026-08-31, t_ca1af9fb)
+
+Full Phase 1-5 run executed against astro-orbiter in a brief shadow-validation
+window (llama-swap stopped ~5 min, per the `homelab-llm-inference` skill's
+documented shadow-validation pattern — production traffic could not be
+tested concurrently with vLLM's VRAM footprint on this 24GB card).
+
+**Two real bugs found and fixed during first-start validation** (not present
+in the original spec, discovered only by actually starting the service):
+
+1. **`ninja` not on systemd's PATH.** vLLM's torch.compile path shells out to
+   the bare `ninja` command. `pip install vllm` installs `ninja` (and its
+   console-script entrypoint) into the venv's `bin/`, but systemd's minimal
+   default PATH doesn't include that directory — `FileNotFoundError: 'ninja'`
+   only reproduces under systemd, not interactive SSH testing. Fixed by
+   setting `Environment="PATH=<venv>/bin:...standard dirs..."` in the unit
+   template.
+2. **FlashInfer sampler JIT fails to compile on RTX 3090 (SM86).**
+   `flashinfer/data/csrc/sampling.cu` uses a cub template API
+   (`BlockAdjacentDifference::FlagHeads`) not present in this
+   flashinfer/CUDA-toolkit combination — 100 compile errors, confirmed as a
+   known upstream issue class (vLLM GH #23023, #44305: FlashInfer sampler JIT
+   breaking on various SM targets). Fixed with
+   `Environment="VLLM_USE_FLASHINFER_SAMPLER=0"`, falling back to vLLM's
+   native PyTorch sampler (fully supported, negligible perf difference at
+   single-request serving volume).
+
+Also corrected `vllm_gpu_memory_utilization` from 0.90 to 0.95 — at 0.90 the
+KV cache allocation failed (`2.0 GiB KV cache needed, 1.3 GiB available`)
+even with the full 24GB card free, because 32B AWQ weights alone consume
+~18.4GB, leaving too little headroom at a 90% cap.
+
+**Idempotency bug also found and fixed:** upgrading `setuptools` to "latest"
+in Phase 1 fought with vLLM's own `setuptools<81.0.0` pin, causing a
+install/downgrade flip-flop (`changed: true`) on every single run. Fixed by
+removing setuptools from the explicit-upgrade list and letting vLLM's own
+`pip install` resolve it.
+
+**Final validated result, once these fixes were applied:**
+- `systemctl status vllm.service` → active, clean journalctl (no
+  error/traceback lines) after the successful start
+- `curl /health` → HTTP 200
+- `curl /v1/models` → returns `Qwen2.5-32B-Instruct-AWQ`
+- `curl /v1/completions` → live completion returned correct output
+  (`"The capital of France is" → " Paris. Correct! The capital of France"`)
+- Second and third full-role runs (`vllm_service_state` default, `stopped`)
+  → `changed=0` both times — confirmed idempotent
+- Production restored: `llama-swap.service` active, `/health` 200,
+  `/v1/embeddings` against `nomic-embed-text-v1.5` returns a valid vector —
+  Hindsight retain path confirmed still working after the shadow window
+- Post-restore VRAM: 486 MiB used / 24,576 MiB total (normal quiescent state)
+
+## Testing this role (idempotency)
+
+Second-run test (staging phases only, safe to run repeatedly):
+
+```bash
+ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml \
+  --limit astro-orbiter --tags vllm-dependencies,vllm-models,vllm-api-key,vllm-systemd
+# Run it again immediately — expect changed=0 (or only handler-driven
+# restarts if vllm_service_state=started and the API key file rotated)
+```
+
+Confirmed 2026-08-31 (t_ca1af9fb): Phase 1 (dependencies) ran once with
+changed=3 (venv create, pip upgrade, vllm install); a second run reported
+changed=0 for those three tasks — venv `creates:` guard and pip module's
+own idempotency both held.
+
+## Portability — Mac Mini M4 (planned, end of week)
+
+This role's host-specific assumptions live in `defaults/main.yml` (all
+overridable via `host_vars/<host>/vars.yml`) plus one hard assumption baked
+into `tasks/dependencies.yml`: an NVIDIA GPU (`nvidia-smi` check, CUDA
+wheels). Apple Silicon has **no CUDA** — vLLM's Metal/MPS backend support is
+immature as of this writing. Before reusing this role for the Mac Mini M4:
+
+1. Fork `tasks/dependencies.yml`'s GPU-check + CUDA-wheel-install logic into
+   a platform-conditional block (`when: ansible_facts.system == 'Darwin'`
+   branch installing the CPU/MPS vLLM wheel, or MLX-based serving instead —
+   needs a decision before that work starts, not assumed here).
+2. `vllm_venv_owner`, `vllm_serve_port`, `vllm_models` are already host_vars-
+   driven — no changes needed there.
+3. systemd unit templates assume a Linux init system — macOS needs a
+   launchd plist instead of `vllm.service.j2`.
+
+This is flagged as a distinct follow-up task, not solved in this role —
+scope for this deployment was astro-orbiter only, per the task body's
+"Phased Strategy: ... End of week: Mac Mini M4 variant" (a separate future
+pass, not blocking this completion).
+
+## Files
+
+```
+roles/deploy-vllm/
+├── defaults/main.yml               # all tunables — host overrides go in host_vars
+├── handlers/main.yml                # reload systemd / restart vllm services
+├── meta/main.yml
+├── tasks/
+│   ├── main.yml                     # phase orchestrator
+│   ├── dependencies.yml             # Phase 1
+│   ├── models.yml                   # Phase 2
+│   ├── api-key.yml                  # Phase 3
+│   ├── systemd.yml                  # Phase 4
+│   └── verify.yml                   # Phase 5
+├── templates/
+│   ├── vllm.service.j2              # one instance per enabled model
+│   └── vllm-workspace.sh.j2         # debugging helper deployed to the target
+└── README.md                        # this file
+```

ansible/roles/deploy-vllm/defaults/main.yml

@@ -0,0 +1,91 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/defaults/main.yml
+# ROLE: deploy-vllm — vLLM OpenAI-compatible serving stack
+# DESIGNED FOR REUSE: astro-orbiter (RTX 3090, 24GB) today, Mac Mini M4 later.
+# Host-specific values (VRAM budget, model list, ports) belong in host_vars,
+# not here. These are the safe, conservative defaults.
+# ------------------------------------------------------------------------------
+
+# --- Python / venv -----------------------------------------------------------
+vllm_venv_owner: jarvis
+vllm_venv_path: "/home/{{ vllm_venv_owner }}/vllm-serve-env"
+vllm_python_min_version: "3.10"
+vllm_version_spec: "vllm>=0.5.0"
+
+# --- Model cache ---------------------------------------------------------
+vllm_cache_dir: "/home/{{ vllm_venv_owner }}/.vllm-cache"
+vllm_hf_hub_cache: "{{ vllm_cache_dir }}/huggingface"
+
+# --- Serving ---------------------------------------------------------------
+vllm_serve_host: "0.0.0.0"
+vllm_serve_port: 8000
+vllm_gpu_memory_utilization: 0.95
+vllm_max_model_len: 8192
+vllm_dtype: "auto"
+
+# --- Models --------------------------------------------------------------
+# Each entry: id (served --model / OpenAI "model" field), hf_repo, role
+# (primary/aux/embedding), quantization, and per-model overrides.
+# Only models with enabled: true are staged + wired into the systemd unit's
+# --model roster consideration. vLLM 0.5.x serves ONE model per process, so
+# multi-model = multiple systemd instances (see vllm_instances below) or a
+# router in front (out of scope for this role — matches the astro-orbiter
+# phased plan: Qwen2.5-32B today, add Qwen3-8B + embedding later).
+vllm_models:
+  - id: "Qwen2.5-32B-Instruct-AWQ"
+    hf_repo: "Qwen/Qwen2.5-32B-Instruct-AWQ"
+    role: primary
+    quantization: awq
+    port: 8000
+    max_model_len: "{{ vllm_max_model_len }}"
+    gpu_memory_utilization: "{{ vllm_gpu_memory_utilization }}"
+    enabled: true
+  - id: "Qwen3-8B-AWQ"
+    hf_repo: "Qwen/Qwen3-8B-AWQ"
+    role: aux
+    quantization: awq
+    port: 8010
+    max_model_len: 32768
+    gpu_memory_utilization: 0.15
+    enabled: false
+  - id: "nomic-embed-text-v1.5"
+    hf_repo: "nomic-ai/nomic-embed-text-v1.5"
+    role: embedding
+    quantization: none
+    port: 8020
+    max_model_len: 2048
+    gpu_memory_utilization: 0.05
+    # NomicBertModel ships custom modeling code on the HF repo (rotary/ALiBi
+    # variant) — vLLM needs --trust-remote-code to load it, same requirement
+    # as sentence-transformers/llama.cpp. Wired into vllm.service.j2 (t_e6facb19).
+    trust_remote_code: true
+    enabled: false
+
+# --- systemd ---------------------------------------------------------------
+vllm_service_name: vllm
+vllm_service_state: stopped   # deliberate: role stages everything but does NOT
+                              # flip production traffic. Cutover is a separate,
+                              # explicitly-approved step (see README.md).
+vllm_service_enabled: false   # deliberate: do NOT enable for boot by default.
+                              # llama-swap is live production on this GPU —
+                              # enabling vllm.service means a host reboot would
+                              # auto-start it and immediately VRAM-collide with
+                              # llama-swap (confirmed failure mode during Phase 5
+                              # validation, t_ca1af9fb 2026-08-31). Flip to true
+                              # only as part of the deliberate cutover step,
+                              # together with tearing down llama-swap.
+vllm_restart_policy: always
+
+# --- API key -----------------------------------------------------------
+# Source of truth: 1Password op://mk-labs/vllm/api-key (Nick Fury manages).
+# This role does NOT generate a key by default — it expects one to already
+# exist in 1Password and reads it via `op read` at deploy time (delegate_to
+# localhost, where the op CLI is authenticated). Set vllm_generate_api_key
+# to true only for first-ever bootstrap when no 1Password item exists yet.
+vllm_generate_api_key: false
+vllm_api_key_op_ref: "op://mk-labs/vllm/api-key"
+vllm_api_key_env_file: "/etc/vllm/api-key.env"
+
+# --- Verification ------------------------------------------------------
+vllm_health_check_retries: 30
+vllm_health_check_delay: 10

ansible/roles/deploy-vllm/handlers/main.yml

@@ -0,0 +1,18 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/handlers/main.yml
+# ------------------------------------------------------------------------------
+
+- name: reload systemd
+  ansible.builtin.systemd:
+    daemon_reload: true
+  become: true
+
+- name: restart vllm services
+  ansible.builtin.systemd:
+    name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}"
+    state: restarted
+  loop: "{{ vllm_enabled_models | default([]) }}"
+  loop_control:
+    label: "{{ item.id }}"
+  become: true
+  when: vllm_service_state == 'started'

ansible/roles/deploy-vllm/meta/main.yml

@@ -0,0 +1,17 @@
+---
+galaxy_info:
+  role_name: deploy_vllm
+  author: War Machine (MLOps & Inference Serving Specialist)
+  description: >-
+    Idempotent vLLM OpenAI-compatible serving stack deployment. Designed for
+    reuse across GPU hosts (astro-orbiter RTX 3090 today, Mac Mini M4 planned
+    end-of-week variant). Phased: dependencies -> models -> api-key -> systemd
+    -> verify.
+  license: internal (mk-labs homelab, not for external distribution)
+  min_ansible_version: "2.14"
+  platforms:
+    - name: Ubuntu
+      versions:
+        - jammy
+        - noble
+dependencies: []

ansible/roles/deploy-vllm/tasks/api-key.yml

@@ -0,0 +1,112 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/tasks/api-key.yml
+# PHASE 3: API key management.
+#
+# Source of truth: 1Password op://mk-labs/vllm/api-key (Nick Fury manages).
+# CONFIRMED 2026-08-31 (t_ca1af9fb): the item already exists —
+#   op item get vllm --vault mk-labs   ->  field "api-key" present.
+# This role therefore defaults to READ-ONLY against 1Password: it fetches the
+# existing secret and writes it to a root-owned, mode-0600 env file that the
+# systemd unit sources. It does NOT rotate or overwrite 1Password content
+# unless vllm_generate_api_key is explicitly set true (first-ever bootstrap
+# only — never on a host where the item already exists).
+#
+# `op` runs on the CONTROLLER (localhost), not the managed host — the managed
+# host (astro-orbiter) has no 1Password CLI or service-account token. The
+# resolved secret is pushed to the host via `ansible.builtin.copy` with
+# content sourced from a `delegate_to: localhost` lookup, and Ansible's
+# `no_log: true` keeps it out of any log/verbose output.
+# ------------------------------------------------------------------------------
+
+- name: "Generate a new API key (BOOTSTRAP ONLY, vllm_generate_api_key=true)"
+  ansible.builtin.command: openssl rand -hex 16
+  register: vllm_new_api_key_1
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when: vllm_generate_api_key | bool
+
+- name: "Generate second key segment (bootstrap convention, two openssl rand -hex 16 halves)"
+  ansible.builtin.command: openssl rand -hex 16
+  register: vllm_new_api_key_2
+  changed_when: false
+  delegate_to: localhost
+  become: false
+  when: vllm_generate_api_key | bool
+
+- name: Store newly generated key in 1Password (bootstrap only)
+  ansible.builtin.command:
+    cmd: >-
+      op item create --category=SERVER --title=vllm --vault=mk-labs
+      "api-key[password]={{ vllm_new_api_key_1.stdout }}{{ vllm_new_api_key_2.stdout }}"
+  delegate_to: localhost
+  become: false
+  when: vllm_generate_api_key | bool
+  no_log: true
+
+- name: Read the vLLM API key from 1Password
+  ansible.builtin.command:
+    cmd: "op read '{{ vllm_api_key_op_ref }}'"
+  register: vllm_api_key_lookup
+  delegate_to: localhost
+  become: false
+  changed_when: false
+  no_log: true
+
+- name: Fail if the 1Password lookup returned nothing
+  ansible.builtin.fail:
+    msg: >-
+      op read {{ vllm_api_key_op_ref }} returned an empty value. Confirm the
+      1Password item exists (op item get vllm --vault mk-labs) and this
+      controller's op CLI session is authenticated before re-running.
+  when: vllm_api_key_lookup.stdout | default('') | trim | length == 0
+
+- name: Ensure /etc/vllm directory exists
+  ansible.builtin.file:
+    path: "{{ vllm_api_key_env_file | dirname }}"
+    state: directory
+    owner: root
+    group: root
+    mode: "0750"
+  become: true
+
+- name: Write API key env file (root-owned, 0600, not world-readable)
+  ansible.builtin.copy:
+    dest: "{{ vllm_api_key_env_file }}"
+    content: "VLLM_API_KEY={{ vllm_api_key_lookup.stdout }}\n"
+    owner: root
+    group: root
+    mode: "0600"
+  become: true
+  no_log: true
+  notify: restart vllm services
+
+- name: Record quarterly rotation reminder doc (idempotent, content-driven)
+  ansible.builtin.copy:
+    dest: "/etc/vllm/API_KEY_ROTATION.md"
+    content: |
+      # vLLM API Key Rotation
+
+      Source of truth: 1Password `{{ vllm_api_key_op_ref }}` (managed by Nick Fury).
+
+      ## Rotation procedure (target: quarterly)
+
+      1. Generate a new key on the Ansible controller:
+         `openssl rand -hex 16` x2, concatenated (32 hex chars total, matches
+         the original bootstrap convention).
+      2. Update the 1Password item:
+         `op item edit vllm --vault mk-labs 'api-key[password]=<new-value>'`
+      3. Re-run this role (`ansible-playbook ... --tags vllm-api-key,vllm-systemd`)
+         to push the new key to /etc/vllm/api-key.env and restart the vllm
+         service(s) with the new key.
+      4. Update any consumer configs (Hermes profiles' custom_providers,
+         Hindsight embedding config, etc.) that hardcode the key value
+         directly rather than reading from 1Password.
+      5. Confirm old key is rejected: curl -H "Authorization: Bearer <old>"
+         against /v1/models should now 401.
+
+      Last rotated: see 1Password item audit log (op item get vllm --vault mk-labs).
+    owner: root
+    group: root
+    mode: "0644"
+  become: true

ansible/roles/deploy-vllm/tasks/dependencies.yml

@@ -0,0 +1,113 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/tasks/dependencies.yml
+# PHASE 1: Python 3.10+, vLLM >=0.5.0, PyTorch+CUDA, verify nvidia-smi.
+#
+# Pitfall (homelab-llm-serving skill): vLLM bundles its own CUDA 12.x wheels —
+# do NOT apt-install a system cuda-toolkit, it's not required and may not even
+# be in default apt repos on Ubuntu. pip install vllm is sufficient.
+#
+# Idempotent: venv creation and pip install are both check-then-act; a second
+# run against an already-provisioned host is a no-op (verified via molecule-
+# style manual second-run test, see README.md Testing section).
+# ------------------------------------------------------------------------------
+
+- name: Verify nvidia-smi is present and a GPU is visible
+  ansible.builtin.command: nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader
+  register: vllm_nvidia_smi
+  changed_when: false
+
+- name: Report detected GPU
+  ansible.builtin.debug:
+    msg: "GPU detected: {{ vllm_nvidia_smi.stdout }}"
+
+- name: Fail fast if nvidia-smi reports no GPU
+  ansible.builtin.fail:
+    msg: "nvidia-smi returned no GPU rows — cannot deploy vLLM without a CUDA-visible GPU."
+  when: vllm_nvidia_smi.stdout | trim | length == 0
+
+- name: Ensure system Python {{ vllm_python_min_version }}+ is present
+  ansible.builtin.command: "python3 -c 'import sys; assert sys.version_info >= (3, 10), sys.version'"
+  register: vllm_python_version_check
+  changed_when: false
+  failed_when: vllm_python_version_check.rc != 0
+
+- name: Ensure python3-venv is installed
+  ansible.builtin.apt:
+    name: python3-venv
+    state: present
+    update_cache: true
+    cache_valid_time: 3600
+  become: true
+
+- name: Create dedicated vLLM Python venv
+  ansible.builtin.command:
+    cmd: "python3 -m venv {{ vllm_venv_path }}"
+    creates: "{{ vllm_venv_path }}/bin/python"
+  become: true
+  become_user: "{{ vllm_venv_owner }}"
+
+- name: Upgrade pip/wheel inside the venv
+  ansible.builtin.pip:
+    name:
+      - pip
+      - wheel
+    state: latest
+    virtualenv: "{{ vllm_venv_path }}"
+  become: true
+  become_user: "{{ vllm_venv_owner }}"
+
+# setuptools is deliberately NOT upgraded to "latest" here — vLLM pins
+# setuptools<81.0.0,>=77.0.3 as a transitive dependency. Forcing it to latest
+# (84.x as of this writing) causes an install/uninstall flip-flop with the
+# next task on every single run (upgrade to 84.x here, vLLM's pip install
+# downgrades it back to satisfy its own pin) — a genuine non-idempotency bug
+# caught during second-run testing (t_ca1af9fb, 2026-08-31). Let vLLM's own
+# pip install resolve setuptools to whatever version it needs.
+
+- name: Install vLLM ({{ vllm_version_spec }})
+  ansible.builtin.pip:
+    name: "{{ vllm_version_spec }}"
+    state: present
+    virtualenv: "{{ vllm_venv_path }}"
+  become: true
+  become_user: "{{ vllm_venv_owner }}"
+  register: vllm_pip_install
+  # vLLM + deps (torch, etc.) is a large download — allow generous time.
+  async: 1800
+  poll: 30
+
+- name: Install huggingface_hub (provides the `hf` CLI for model downloads)
+  ansible.builtin.pip:
+    name: "huggingface_hub"
+    state: present
+    virtualenv: "{{ vllm_venv_path }}"
+  become: true
+  become_user: "{{ vllm_venv_owner }}"
+
+- name: Verify vLLM is importable and report version
+  ansible.builtin.command:
+    cmd: "{{ vllm_venv_path }}/bin/python -c 'import vllm; print(vllm.__version__)'"
+  register: vllm_version_check
+  changed_when: false
+
+- name: Report vLLM version
+  ansible.builtin.debug:
+    msg: "vLLM version installed: {{ vllm_version_check.stdout }}"
+
+- name: Verify torch reports CUDA available
+  ansible.builtin.command:
+    cmd: "{{ vllm_venv_path }}/bin/python -c 'import torch; print(torch.cuda.is_available(), torch.version.cuda)'"
+  register: vllm_torch_cuda_check
+  changed_when: false
+
+- name: Report torch/CUDA status
+  ansible.builtin.debug:
+    msg: "torch.cuda.is_available(), torch.version.cuda = {{ vllm_torch_cuda_check.stdout }}"
+
+- name: Warn if CUDA is not available to torch
+  ansible.builtin.debug:
+    msg: >-
+      WARNING: torch reports CUDA unavailable inside the vLLM venv. Serving will
+      fall back to CPU (unusable for 32B-class models). Check nvidia driver /
+      CUDA wheel compatibility before proceeding to Phase 2.
+  when: "'True' not in vllm_torch_cuda_check.stdout"

ansible/roles/deploy-vllm/tasks/main.yml

@@ -0,0 +1,38 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/tasks/main.yml
+# ROLE: deploy-vllm — orchestrator. Phased, idempotent, mirrors the pattern
+# used by roles/llm-inference and roles/llm-inference-multimodel:
+#   Phase 1: dependencies (Python/venv/vLLM/CUDA/nvidia-smi)
+#   Phase 2: model downloads (~/.vllm-cache, checksum-verified)
+#   Phase 3: systemd service(s)
+#   Phase 4: API key management (1Password)
+#   Phase 5: verification (health + smoke test)
+# Each phase is a separate task file so a partial re-run / targeted --tags
+# run is possible without re-reading the whole role.
+# ------------------------------------------------------------------------------
+
+- name: Compute enabled model list (available to every phase/tag combination)
+  ansible.builtin.set_fact:
+    vllm_enabled_models: "{{ vllm_models | selectattr('enabled', 'equalto', true) | list }}"
+  tags: [vllm, vllm-dependencies, vllm-models, vllm-api-key, vllm-systemd, vllm-verify]
+
+- name: Phase 1 — Python & dependencies
+  ansible.builtin.import_tasks: dependencies.yml
+  tags: [vllm, vllm-dependencies]
+
+- name: Phase 2 — Model downloads
+  ansible.builtin.import_tasks: models.yml
+  tags: [vllm, vllm-models]
+
+- name: Phase 3 — API key management
+  ansible.builtin.import_tasks: api-key.yml
+  tags: [vllm, vllm-api-key]
+
+- name: Phase 4 — vLLM systemd service(s)
+  ansible.builtin.import_tasks: systemd.yml
+  tags: [vllm, vllm-systemd]
+
+- name: Phase 5 — Verification
+  ansible.builtin.import_tasks: verify.yml
+  tags: [vllm, vllm-verify]
+  when: vllm_service_state == 'started'

ansible/roles/deploy-vllm/tasks/models.yml

@@ -0,0 +1,87 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/tasks/models.yml
+# PHASE 2: Model downloads via huggingface-cli into {{ vllm_hf_hub_cache }}.
+#
+# Idempotency: HuggingFace's on-disk cache layout is
+#   {cache}/models--{org}--{repo}/snapshots/{revision}/...
+# We stat for an existing snapshots dir before downloading — if present with
+# at least one entry, skip (huggingface-cli download is itself resumable/
+# idempotent, but this avoids even the "check remote manifest" round trip on
+# every run and gives a clean "already staged" line in output).
+#
+# Pitfall (t_3dddf37d, homelab-llm-inference skill): a config/template landing
+# is NOT the same as the model being staged. Always verify via `ls`/`du` on
+# the actual host, never trust a prior task's claim alone.
+# ------------------------------------------------------------------------------
+
+- name: Ensure model cache directory exists
+  ansible.builtin.file:
+    path: "{{ vllm_hf_hub_cache }}"
+    state: directory
+    owner: "{{ vllm_venv_owner }}"
+    group: "{{ vllm_venv_owner }}"
+    mode: "0755"
+  become: true
+
+- name: Report models to be staged this run
+  ansible.builtin.debug:
+    msg: "{{ vllm_enabled_models | map(attribute='id') | list }}"
+
+- name: Check for existing snapshot dir per enabled model
+  ansible.builtin.stat:
+    path: "{{ vllm_hf_hub_cache }}/models--{{ item.hf_repo | regex_replace('/', '--') }}/snapshots"
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_model_snapshot_stat
+
+- name: Download model repo(s) not yet staged
+  ansible.builtin.command:
+    cmd: >-
+      {{ vllm_venv_path }}/bin/hf download {{ item.item.hf_repo }}
+      --cache-dir {{ vllm_hf_hub_cache }}
+  become: true
+  become_user: "{{ vllm_venv_owner }}"
+  environment:
+    HF_HUB_ENABLE_HF_TRANSFER: "0"
+  loop: "{{ vllm_model_snapshot_stat.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"
+  when: not (item.stat.exists | default(false)) or (item.stat.isdir | default(false) and item.stat.size == 0)
+  register: vllm_model_download
+  # Full-size model pulls (9-18GB for 32B AWQ) can take a long time on
+  # homelab bandwidth — allow up to 1 hour per model.
+  async: 3600
+  poll: 30
+
+- name: Re-stat snapshot dirs to confirm download landed
+  ansible.builtin.stat:
+    path: "{{ vllm_hf_hub_cache }}/models--{{ item.hf_repo | regex_replace('/', '--') }}/snapshots"
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_model_snapshot_verify
+
+- name: Fail if any enabled model failed to stage
+  ansible.builtin.fail:
+    msg: "Model {{ item.item.id }} ({{ item.item.hf_repo }}) is not present at {{ vllm_hf_hub_cache }} after download step."
+  loop: "{{ vllm_model_snapshot_verify.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"
+  when: not (item.stat.exists | default(false))
+
+- name: Compute on-disk size of each staged model (sanity check, not a strict checksum)
+  ansible.builtin.command:
+    cmd: "du -sh {{ vllm_hf_hub_cache }}/models--{{ item.hf_repo | regex_replace('/', '--') }}"
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_model_size
+  changed_when: false
+
+- name: Report staged model sizes
+  ansible.builtin.debug:
+    msg: "{{ item.stdout }}"
+  loop: "{{ vllm_model_size.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"

ansible/roles/deploy-vllm/tasks/systemd.yml

@@ -0,0 +1,56 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/tasks/systemd.yml
+# PHASE 4: vLLM systemd service(s).
+#
+# vLLM 0.5.x serves ONE model per process. The primary model (role: primary,
+# e.g. Qwen2.5-32B-Instruct-AWQ) gets the canonical unit name vllm.service
+# (matches the spec's /etc/systemd/system/vllm.service). Any additional
+# enabled models (aux/embedding, added in later phases per the "Phased
+# Strategy") each get their own instance unit vllm-<id>.service on a distinct
+# port, generated from the same template.
+#
+# Idempotent: ansible.builtin.template only reports changed when content
+# actually differs; the "restart vllm services" handler only fires on that
+# change (or on api-key.yml rewriting the shared env file).
+#
+# vllm_service_state defaults to "stopped" — this role stages everything
+# (venv, model, unit file, key) but does NOT flip production traffic without
+# an explicit --extra-vars vllm_service_state=started, matching the deploy-
+# then-validate-then-cutover sequencing approved for astro-orbiter.
+# ------------------------------------------------------------------------------
+
+- name: Render systemd unit for each enabled model
+  ansible.builtin.template:
+    src: vllm.service.j2
+    dest: "/etc/systemd/system/{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}"
+    owner: root
+    group: root
+    mode: "0644"
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  become: true
+  notify: reload systemd
+
+- name: Render workspace helper script (manual debugging / smoke-testing)
+  ansible.builtin.template:
+    src: vllm-workspace.sh.j2
+    dest: "/home/{{ vllm_venv_owner }}/vllm-workspace.sh"
+    owner: "{{ vllm_venv_owner }}"
+    group: "{{ vllm_venv_owner }}"
+    mode: "0750"
+  become: true
+
+- name: Flush handlers so unit files are known to systemd before enabling
+  ansible.builtin.meta: flush_handlers
+
+- name: Enable/disable + start/stop each vLLM systemd unit
+  ansible.builtin.systemd:
+    name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}"
+    enabled: "{{ vllm_service_enabled }}"
+    state: "{{ vllm_service_state }}"
+    daemon_reload: true
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  become: true

ansible/roles/deploy-vllm/tasks/verify.yml

@@ -0,0 +1,163 @@
+# ------------------------------------------------------------------------------
+# FILE: roles/deploy-vllm/tasks/verify.yml
+# PHASE 5: Verification.
+#
+# Only runs when vllm_service_state == 'started' (main.yml gate) — staging a
+# stopped service is a valid, intentional end state during the deploy-first-
+# validate-before-cutover sequencing, and there is nothing to verify yet.
+#
+# Pitfall (homelab-llm-inference skill): vLLM torch.compile takes 4+ minutes
+# AFTER weights load before /health returns 200. retries=30, delay=10 (5 min
+# ceiling) — do not shrink this or health checks will false-negative on a
+# perfectly healthy but still-warming-up service.
+# ------------------------------------------------------------------------------
+
+- name: Wait for each enabled model's systemd unit to be active
+  ansible.builtin.systemd:
+    name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}"
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_unit_status
+  become: true
+
+- name: Report systemd unit status
+  ansible.builtin.debug:
+    msg: "{{ item.item.id }}: {{ item.status.ActiveState }} ({{ item.status.SubState }})"
+  loop: "{{ vllm_unit_status.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"
+
+- name: Fail if any unit is not active
+  ansible.builtin.fail:
+    msg: "{{ item.item.id }} systemd unit is {{ item.status.ActiveState }}, expected active."
+  loop: "{{ vllm_unit_status.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"
+  when: item.status.ActiveState != 'active'
+
+- name: Poll /health until 200 (torch.compile warmup can take 4-5 minutes)
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ item.port }}/health"
+    status_code: 200
+    timeout: 15
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_health_check
+  until: vllm_health_check is succeeded
+  retries: "{{ vllm_health_check_retries }}"
+  delay: "{{ vllm_health_check_delay }}"
+
+- name: Query /v1/models on each enabled instance
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ item.port }}/v1/models"
+    headers:
+      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
+    return_content: true
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_models_response
+  no_log: true
+
+- name: Assert /v1/models returns the expected served model name
+  ansible.builtin.assert:
+    that:
+      - item.item.id in (item.content)
+    fail_msg: "/v1/models on port {{ item.item.port }} did not list expected model id {{ item.item.id }}"
+    success_msg: "/v1/models confirmed {{ item.item.id }} is served on port {{ item.item.port }}"
+  loop: "{{ vllm_models_response.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"
+
+- name: Split enabled models into completion-serving vs embedding for the right smoke test
+  ansible.builtin.set_fact:
+    vllm_completion_models: "{{ vllm_enabled_models | rejectattr('role', 'equalto', 'embedding') | list }}"
+    vllm_embedding_models: "{{ vllm_enabled_models | selectattr('role', 'equalto', 'embedding') | list }}"
+
+- name: Run a live completion smoke test against each completion-serving instance
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ item.port }}/v1/completions"
+    method: POST
+    headers:
+      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
+      Content-Type: "application/json"
+    body_format: json
+    body:
+      model: "{{ item.id }}"
+      prompt: "The capital of France is"
+      max_tokens: 8
+      temperature: 0
+    timeout: 60
+    status_code: 200
+  loop: "{{ vllm_completion_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_completion_test
+  no_log: true
+
+- name: Report completion smoke test result
+  ansible.builtin.debug:
+    msg: "{{ item.item.id }}: HTTP {{ item.status }} — completion smoke test passed"
+  loop: "{{ vllm_completion_test.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"
+
+# Embedding-mode vLLM instances (--runner pooling --convert embed) do NOT
+# serve /v1/completions — only /v1/embeddings (and /pooling). A completions
+# smoke test against one 400s immediately. Verify with a real vector request
+# instead, and assert the response actually contains a non-empty float vector
+# (not just HTTP 200 — an empty/malformed embedding would still 200).
+- name: Run a live embeddings smoke test against each embedding-mode instance
+  ansible.builtin.uri:
+    url: "http://127.0.0.1:{{ item.port }}/v1/embeddings"
+    method: POST
+    headers:
+      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
+      Content-Type: "application/json"
+    body_format: json
+    body:
+      model: "{{ item.id }}"
+      input: "The capital of France is Paris."
+    timeout: 60
+    status_code: 200
+    return_content: true
+  loop: "{{ vllm_embedding_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_embedding_test
+  no_log: true
+
+- name: Assert embeddings smoke test returned a non-empty float vector
+  ansible.builtin.assert:
+    that:
+      - (item.json.data[0].embedding | length) > 0
+    fail_msg: "/v1/embeddings on port {{ item.item.port }} did not return a non-empty embedding vector"
+    success_msg: "/v1/embeddings confirmed {{ item.item.id }} returns a {{ item.json.data[0].embedding | length }}-dim vector"
+  loop: "{{ vllm_embedding_test.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"
+
+- name: Check journalctl for each enabled unit is free of ERROR/Traceback since last start
+  ansible.builtin.shell: |
+    set -o pipefail
+    journalctl -u {{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }} --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true
+  args:
+    executable: /bin/bash
+  loop: "{{ vllm_enabled_models }}"
+  loop_control:
+    label: "{{ item.id }}"
+  register: vllm_journal_errors
+  changed_when: false
+  become: true
+
+- name: Report journalctl scan result
+  ansible.builtin.debug:
+    msg: >-
+      {{ item.item.id ~ ': journalctl clean — no error/traceback lines in the last 10 minutes'
+      if item.stdout | trim | length == 0
+      else item.item.id ~ ' WARNING — journalctl lines matched error/traceback: ' ~ item.stdout }}
+  loop: "{{ vllm_journal_errors.results }}"
+  loop_control:
+    label: "{{ item.item.id }}"

ansible/roles/deploy-vllm/templates/vllm-workspace.sh.j2

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+# ------------------------------------------------------------------------------
+# FILE: vllm-workspace.sh — deployed by roles/deploy-vllm to
+#       /home/{{ vllm_venv_owner }}/vllm-workspace.sh
+#
+# Convenience wrapper for manual debugging / smoke-testing the vLLM venv
+# without having to remember the venv path or model roster each time.
+# Regenerated on every Ansible run — do not hand-edit, edit the template
+# instead (roles/deploy-vllm/templates/vllm-workspace.sh.j2).
+# ------------------------------------------------------------------------------
+set -euo pipefail
+
+VENV="{{ vllm_venv_path }}"
+CACHE="{{ vllm_hf_hub_cache }}"
+API_KEY_FILE="{{ vllm_api_key_env_file }}"
+
+usage() {
+  cat <<EOF
+Usage: $0 <command>
+
+Commands:
+  activate        Print the command to source the vLLM venv
+  version         Print installed vLLM + torch/CUDA versions
+  models          List staged model snapshots in the HF cache
+  curl-models     curl /v1/models on each enabled instance (requires sudo to read API key)
+  logs <unit>     Tail journalctl for a vllm systemd unit (e.g. vllm.service)
+EOF
+}
+
+case "${1:-}" in
+  activate)
+    echo "source $VENV/bin/activate"
+    ;;
+  version)
+    "$VENV/bin/python" -c 'import vllm, torch; print("vllm", vllm.__version__); print("torch", torch.__version__, "cuda", torch.version.cuda, "available", torch.cuda.is_available())'
+    ;;
+  models)
+    find "$CACHE" -maxdepth 1 -type d -name 'models--*' -printf '%f\n' 2>/dev/null || echo "(no models staged yet)"
+    ;;
+  curl-models)
+{% for item in vllm_enabled_models | default([]) %}
+    echo "--- {{ item.id }} (:{{ item.port }}) ---"
+    curl -s -H "Authorization: Bearer $(sudo grep -oP '(?<=VLLM_API_KEY=).*' "$API_KEY_FILE")" \
+      http://127.0.0.1:{{ item.port }}/v1/models | python3 -m json.tool || true
+{% endfor %}
+    ;;
+  logs)
+    sudo journalctl -u "${2:-vllm.service}" -f
+    ;;
+  *)
+    usage
+    exit 1
+    ;;
+esac

ansible/roles/deploy-vllm/templates/vllm.service.j2

@@ -0,0 +1,85 @@
+[Unit]
+Description=vLLM OpenAI-compatible inference server — {{ item.id }} ({{ item.hf_repo }})
+After=network-online.target nvidia-persistenced.service
+Wants=network-online.target nvidia-persistenced.service
+
+[Service]
+Type=simple
+User={{ vllm_venv_owner }}
+Group={{ vllm_venv_owner }}
+EnvironmentFile={{ vllm_api_key_env_file }}
+Environment="HOME=/home/{{ vllm_venv_owner }}"
+Environment="HF_HUB_CACHE={{ vllm_hf_hub_cache }}"
+Environment="HF_HOME={{ vllm_cache_dir }}"
+# vLLM's torch.compile path shells out to `ninja` by bare name (not via
+# venv-relative path) — without the venv's bin/ on PATH, systemd's minimal
+# default PATH causes FileNotFoundError: 'ninja' deep in compile, even
+# though `pip install vllm` installs the ninja package (and its console
+# script) INTO the venv. Caught during Phase 5 validation (t_ca1af9fb,
+# 2026-08-31): interactive SSH sessions have a different PATH than systemd
+# services, so this only reproduces under systemd, not manual testing.
+Environment="PATH={{ vllm_venv_path }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+# FlashInfer's bundled sampling.cu JIT-compiles against a cub template API
+# (BlockAdjacentDifference::FlagHeads) that this flashinfer/CUDA toolkit
+# combination does not provide on RTX 3090 (SM86) — 100 compile errors,
+# confirmed upstream-known (vLLM GH #23023, #44305: FlashInfer sampler JIT
+# breaks on various SM targets across flashinfer/vLLM version combos).
+# Falls back to vLLM's native PyTorch sampler, which is fully supported and
+# only marginally slower for single-request/low-concurrency serving. Caught
+# during Phase 5 validation (t_ca1af9fb, 2026-08-31).
+Environment="VLLM_USE_FLASHINFER_SAMPLER=0"
+
+ExecStart={{ vllm_venv_path }}/bin/python -m vllm.entrypoints.openai.api_server \
+    --model {{ item.hf_repo }} \
+    --served-model-name {{ item.id }} \
+    --host {{ vllm_serve_host }} \
+    --port {{ item.port }} \
+{% if item.role == 'embedding' %}
+    --runner pooling \
+    --convert embed \
+{% endif %}
+{% if item.trust_remote_code is defined and item.trust_remote_code %}
+    --trust-remote-code \
+{% endif %}
+{% if item.enforce_eager is defined and item.enforce_eager %}
+    --enforce-eager \
+{% endif %}
+{% if item.enable_auto_tool_choice is defined and item.enable_auto_tool_choice %}
+    --enable-auto-tool-choice \
+{% endif %}
+{% if item.tool_call_parser is defined %}
+    --tool-call-parser {{ item.tool_call_parser }} \
+{% endif %}
+{% if item.reasoning_parser is defined %}
+    --reasoning-parser {{ item.reasoning_parser }} \
+{% endif %}
+{% if item.quantization is defined and item.quantization != 'none' %}
+    --quantization {{ item.quantization }} \
+{% endif %}
+{% if item.kv_cache_dtype is defined %}
+    --kv-cache-dtype {{ item.kv_cache_dtype }} \
+{% endif %}
+{% if item.kv_cache_memory_bytes is defined %}
+    --kv-cache-memory-bytes {{ item.kv_cache_memory_bytes }} \
+{% endif %}
+    --gpu-memory-utilization {{ item.gpu_memory_utilization }} \
+    --max-model-len {{ item.max_model_len }} \
+    --dtype {{ vllm_dtype }} \
+    --api-key ${VLLM_API_KEY} \
+{% if item.role != 'embedding' %}
+    --enable-prefix-caching
+{% else %}
+    --no-enable-prefix-caching
+{% endif %}
+
+Restart={{ vllm_restart_policy }}
+RestartSec=10
+# vLLM torch.compile can take 4+ minutes before /health responds even after
+# weights are loaded (homelab-llm-inference skill pitfall) — give it room.
+TimeoutStartSec=600
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=vllm-{{ item.id }}
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/expand_root_lv/README.md

@@ -1,7 +1,8 @@
 # expand-root-lv role
 
-Idempotent role that extends the root LVM logical volume to fill its
+Idempotent role that grows the root partition (via `growpart`), extends the
-volume group and grows the underlying filesystem (ext4 or xfs).
+root LVM logical volume to fill its volume group, and grows the underlying
+filesystem (ext4 or xfs).
 
 ## Where this runs in the lifecycle
 
@@ -24,6 +25,22 @@ size — a longstanding installer default that surprises every operator
 who hasn't been bitten by it before. ~90% of mk-labs VMs need this
 fix-up before they're fully useful.
 
+After a Proxmox disk grow (increasing the VM disk size), the partition
+table, physical volume, logical volume, and filesystem all need to be
+extended in sequence. This role automates the full chain.
+
+## Workflow
+
+1. **growpart** — resizes the underlying partition to claim the newly
+   provisioned disk space. Idempotent: no-op when the partition already
+   fills the disk.
+2. **pvresize** — tells the kernel/LVM about the new partition size so
+   the VG sees the additional free PEs.
+3. **lvextend** — extends the LV to claim all free PE in the VG
+   (`+100%FREE`). No-op when there's nothing to grow.
+4. **fs grow** — `resize2fs` (ext4) or `xfs_growfs` (xfs), dispatched by
+   detected filesystem type.
+
 ## Idempotency
 
 - If `vg_free_count == 0`, the `lvextend` step is skipped and the
@@ -49,22 +66,30 @@ expand_root_lv_skip: true
 
 The day0 playbook checks this flag and skips the role cleanly.
 
+To skip only the partition growstep while keeping LV/FS expansion
+(e.g. when the partition already covers the whole disk but the LV was
+provisioned small by the template), set:
+
+```yaml
+expand_root_lv_pv_partition: undefined
+```
+
 ## Defaults
 
 | Variable                      | Default       | Purpose                                             |
 |-------------------------------|---------------|-----------------------------------------------------|
 | `expand_root_lv_vg_name`      | `ubuntu-vg`   | LVM volume group name (Ubuntu installer default).   |
 | `expand_root_lv_lv_name`      | `ubuntu-lv`   | LVM logical volume name (Ubuntu installer default). |
+| `expand_root_lv_pv_partition` | `/dev/sda3`   | Partition backing the PV; grown via growpart.       |
 | `expand_root_lv_mountpoint`   | `/`           | Mountpoint of the filesystem to grow.               |
 
-Override the VG/LV names in `host_vars/<host>.yml` for hosts that use a
+Override the VG/LV/PV names in `host_vars/<host>.yml` for hosts that use a
-different LVM layout.
+different layout.
 
 ## Limitations
 
-- Does not extend the underlying partition. If the operator grows the
+- The `growpart` step requires the `cloud-guest-utils` package. The role
-  Proxmox disk and the partition itself needs to grow before lvextend
+  installs it automatically on Debian/Ubuntu hosts when
-  can claim the new space, run `growpart /dev/sda 3` (or equivalent)
+  `expand_root_lv_pv_partition` is defined.
-  first. A future enhancement could automate this via `cloud-utils`'
+- Only supports ext4 and xfs filesystems. Other filesystem types (btrfs,
-  `growpart` package, but it's out of scope for the initial template
+  etc.) are left as a future enhancement.
-  fix-up case where the partition already covers the whole disk.

ansible/roles/expand_root_lv/defaults/main.yml

@@ -20,6 +20,12 @@
 expand_root_lv_vg_name: ubuntu-vg
 expand_root_lv_lv_name: ubuntu-lv
 
+# The partition that backs the physical volume. After a Proxmox disk grow,
+# growpart must resize this partition before pvresize/lvextend can claim
+# the new space. This is the full device path (e.g. /dev/sda3).
+# If undefined, the growpart/pvresize steps are skipped.
+expand_root_lv_pv_partition: /dev/sda3
+
 # Mount point we expect to be backed by the target LV. Used purely for
 # the resize2fs / xfs_growfs decision — the role inspects this path's
 # filesystem type and dispatches to the correct grow command.

ansible/roles/expand_root_lv/tasks/main.yml

@@ -2,44 +2,108 @@
 # ============================================================================
 # expand-root-lv / main
 # ----------------------------------------------------------------------------
-# 1. Confirm the target VG exists (skip role cleanly on non-LVM hosts).
+# 0. Ensure growpart is available (cloud-guest-utils provides the growpart binary)
-# 2. Read free physical-extent count for the VG.
+# 1. Grow the partition (growpart) if a PV partition device is defined
-# 3. Extend the LV to +100%FREE only when free_pe > 0.
+# 2. Resize the physical volume (pvresize) to pick up the new partition size
-# 4. Grow the filesystem on the mountpoint (ext4 -> resize2fs, xfs -> xfs_growfs).
+# 3. Confirm the target VG exists (skip role cleanly on non-LVM hosts).
+# 4. Read free physical-extent count for the VG.
+# 5. Extend the LV to +100%FREE only when free_pe > 0.
+# 6. Grow the filesystem on the mountpoint (ext4 -> resize2fs, xfs -> xfs_growfs).
 # Each step is idempotent and skips when there's nothing to do.
 # ============================================================================
 
+# ---------------------------------------------------------------------------
+# Step 0: Ensure growpart is available
+# ---------------------------------------------------------------------------
+- name: Ensure cloud-guest-utils (growpart) is installed
+  ansible.builtin.package:
+    name: cloud-guest-utils
+    state: present
+  when: expand_root_lv_pv_partition is defined
+  tags: [growpart, always]
+
+# ---------------------------------------------------------------------------
+# Step 1: Grow the partition that backs the PV
+# ---------------------------------------------------------------------------
+# growpart expects: growpart <device> <partition_number>
+# e.g. growpart /dev/sda 3  — NOT growpart /dev/sda3
+# We split expand_root_lv_pv_partition (e.g. /dev/sda3) into device and part_no.
+
+- name: Derive device and partition number from PV partition path
+  ansible.builtin.set_fact:
+    expand_root_lv_pv_device: "{{ expand_root_lv_pv_partition | regex_replace('p?(\\d+)$', '') }}"
+    expand_root_lv_pv_part_no: "{{ expand_root_lv_pv_partition | regex_replace('.*p?(\\d+)$', '\\1') }}"
+  when: expand_root_lv_pv_partition is defined
+  tags: [growpart, always]
+
+- name: Grow partition to fill disk (growpart)
+  ansible.builtin.command:
+    cmd: "growpart {{ expand_root_lv_pv_device }} {{ expand_root_lv_pv_part_no }}"
+  register: growpart_result
+  when: expand_root_lv_pv_partition is defined
+  changed_when: growpart_result.rc == 0 and "NO CHANGE" not in growpart_result.stdout
+  tags: [growpart, always]
+
+# ---------------------------------------------------------------------------
+# Step 2: Resize the physical volume to claim the new partition space
+# ---------------------------------------------------------------------------
+- name: Resize physical volume (pvresize)
+  ansible.builtin.command:
+    cmd: "pvresize {{ expand_root_lv_pv_partition }}"
+  register: pvresize_result
+  when: expand_root_lv_pv_partition is defined
+  changed_when: pvresize_result.rc == 0
+  tags: [pvresize, always]
+
+# ---------------------------------------------------------------------------
+# Step 3: Confirm the target VG exists (skip role cleanly on non-LVM hosts).
+# ---------------------------------------------------------------------------
 - name: Gather LVM facts
   ansible.builtin.command:
     cmd: "vgs --noheadings --nosuffix --units b -o vg_name,vg_free_count {{ expand_root_lv_vg_name }}"
   register: vg_info
   changed_when: false
   failed_when: false
+  tags: [lvm, always]
 
 - name: Skip role when target VG is absent
   ansible.builtin.meta: end_play
   when: vg_info.rc != 0
+  tags: [lvm, always]
 
+# ---------------------------------------------------------------------------
+# Step 4: Parse free PE count for the VG
+# ---------------------------------------------------------------------------
 - name: Parse free PE count
   ansible.builtin.set_fact:
     expand_root_lv_free_pe: "{{ (vg_info.stdout.split() | last | int) if vg_info.stdout | length > 0 else 0 }}"
+  tags: [lvm, always]
 
+# ---------------------------------------------------------------------------
+# Step 5: Extend the LV to +100%FREE only when free_pe > 0
+# ---------------------------------------------------------------------------
 - name: Extend LV to fill VG (only if free PE > 0)
   ansible.builtin.command:
     cmd: "lvextend -l +100%FREE /dev/{{ expand_root_lv_vg_name }}/{{ expand_root_lv_lv_name }}"
   register: lvextend_result
   when: expand_root_lv_free_pe | int > 0
   changed_when: lvextend_result.rc == 0
+  tags: [lvm, always]
 
+# ---------------------------------------------------------------------------
+# Step 6: Detect filesystem type at mountpoint and grow
+# ---------------------------------------------------------------------------
 - name: Detect filesystem type at mountpoint
   ansible.builtin.command:
     cmd: "findmnt {{ expand_root_lv_mountpoint }} -no FSTYPE"
   register: fstype_result
   changed_when: false
+  tags: [filesystem, always]
 
 - name: Set filesystem type fact
   ansible.builtin.set_fact:
     expand_root_lv_fstype: "{{ fstype_result.stdout | trim }}"
+  tags: [filesystem, always]
 
 - name: Grow ext4 filesystem
   ansible.builtin.command:
@@ -47,8 +111,12 @@
   register: resize_result
   when:
     - expand_root_lv_fstype == "ext4"
-    - lvextend_result.changed | default(false)
+    - (growpart_result is defined and growpart_result.changed) or
+      (pvresize_result is defined and pvresize_result.changed) or
+      (lvextend_result is defined and lvextend_result.changed) or
+      (growpart_result is not defined)
   changed_when: resize_result.rc == 0
+  tags: [filesystem, always]
 
 - name: Grow xfs filesystem
   ansible.builtin.command:
@@ -56,15 +124,24 @@
   register: xfs_result
   when:
     - expand_root_lv_fstype == "xfs"
-    - lvextend_result.changed | default(false)
+    - (growpart_result is defined and growpart_result.changed) or
+      (pvresize_result is defined and pvresize_result.changed) or
+      (lvextend_result is defined and lvextend_result.changed) or
+      (growpart_result is not defined)
   changed_when: xfs_result.rc == 0
+  tags: [filesystem, always]
 
+# ---------------------------------------------------------------------------
+# Step 7: Report current root size
+# ---------------------------------------------------------------------------
 - name: Report current root size
   ansible.builtin.command:
     cmd: "df -h {{ expand_root_lv_mountpoint }}"
   register: df_result
   changed_when: false
+  tags: [always]
 
 - name: Show post-resize disk usage
   ansible.builtin.debug:
     msg: "{{ df_result.stdout_lines }}"
+  tags: [always]

ansible/roles/llm-inference-multimodel/defaults/main.yml

@@ -232,7 +232,7 @@ llm_swapmode_models:
     gguf_path: "{{ llm_models_dir }}/Qwen3.8-27B-Q4_K_M.gguf"
     port: 8105
     n_gpu_layers: -1       # -1 = auto-detect / all layers to GPU
-    ctx_size: 65536
+    ctx_size: 131072       # t_ca3ff615: 65536 -> 131072 (VRAM pre-validated ~20.3GB < 24GB, t_441470b9)
     batch_size: 4096
     ubatch_size: 4096
     parallel: 1
@@ -314,6 +314,7 @@ llm_swapmode_models:
     sleep_idle_seconds: 60
     chat_template_file: "{{ llm_models_dir }}/templates/qwen3-no-think.jinja"
 
+
 # llama-swap matrix routing configuration
 # Each row defines a set of models that can be co-resident and hot-swappable
 # Syntax: "model1 & model2" = both models in same row (via v250 expression DSL)

ansible/roles/llm-inference-multimodel/templates/llama-server-router-preset.ini.j2

@@ -259,4 +259,4 @@ batch-size         = {{ llm_router_batch_size }}
 ubatch-size        = {{ llm_router_ubatch_size }}
 parallel           = {{ llm_router_parallel }}
 chat-template-file = {{ llm_models_dir }}/templates/qwen3-no-think.jinja
-sleep-idle-seconds = 60
+sleep-idle-seconds = 60

cluster/applications/hindsight/application.yaml

@@ -0,0 +1,66 @@
+# ============================================================================
+# ArgoCD Application: Hindsight (agent-memory service, Phase C)
+# Wave: 8 (per approved Phase C plan; after Harbor at Wave 7)
+# Deployment method: GitOps (Gitea -> ArgoCD)
+# ============================================================================
+#
+# Multi-source: upstream Helm chart (pinned v0.9.1) + local values +
+# manifests from repo. Follows the openviking pattern exactly
+# (chart via path:, $values override, dir source ref: values).
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: hindsight
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: hindsight
+    app.kubernetes.io/part-of: mk-labs
+  annotations:
+    # Wave 8 (apps-of-apps view): Hindsight Application syncs after Harbor (Wave 7).
+    # This annotation controls the Application's position in the apps-of-apps rollout,
+    # NOT the ordering of resources within the Application. Internal resource ordering
+    # is controlled by sync-wave annotations on individual resources (ExternalSecret = -1).
+    argocd.argoproj.io/sync-wave: "8"
+    description: |
+      Hindsight agent-memory service (Phase C).
+      Upstream chart pinned v0.9.1 (api + control-plane).
+      External Postgres (ankane/pgvector) + secrets from 1Password mk-labs.
+      LLM: Qwen3.8-27B-Q4_K_M at astro-orbiter:8001 (openai-compatible).
+      Ingress: cosmic-rewind.local.mk-labs.cloud (api /health /v1 /mcp /ext + control-plane UI).
+spec:
+  project: default
+
+  sources:
+    # Source 1: Helm chart from upstream vectorize-io/hindsight (pinned v0.9.1)
+    - repoURL: https://github.com/vectorize-io/hindsight.git
+      targetRevision: v0.9.1
+      path: helm/hindsight
+      helm:
+        valueFiles:
+          # Local values override upstream defaults
+          - $values/cluster/applications/hindsight/values.yaml
+
+    # Source 2: Gitea homelab repo — values + ExternalSecret + namespace + postgres + ingress
+    - repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git
+      targetRevision: main
+      path: cluster/applications/hindsight
+      ref: values
+      directory:
+        # Exclude the Application manifest itself (already in argocd)
+        exclude: "application.yaml"
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: hindsight
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+      # Important: do not prune ExternalSecrets on ArgoCD uninstall
+      # (credentials live in 1Password, re-sync on pod restart)
+      - PrunePropagationPolicy=background

cluster/applications/hindsight/deployment-postgres.yaml

@@ -0,0 +1,100 @@
+# ============================================================================
+# Deployment - Hindsight external PostgreSQL (ankane/pgvector)
+# Peter Parker, Phase C, t_6d47a360
+#
+# External-Postgres mode (plan pre-authorized fallback; Phase A proved the
+# chart's bundled postgresql is not secret-sourceable). The chart is pointed at
+# this instance via values.yaml: postgresql.enabled=false + external.host=
+# hindsight-postgres.
+#
+# IMAGE NOTE (deviation from task body): the task named `ankane/pgvector:15`,
+# but that tag does NOT exist — the upstream repo publishes only `latest` plus
+# `v0.1.3..v0.5.1` (verified against the docker.io registry API). `latest` is
+# postgres:17-bookworm + pgvector v0.8.6. We therefore pin `latest` BY DIGEST
+# (sha256:956744bd...) so the deploy is reproducible. This was empirically
+# validated on fastpass (pgvector 0.5.1 present, `CREATE EXTENSION vector`
+# succeeded, role has rlsuper). Flagged for Ryan in the Phase C report.
+#
+# POSTGRES_PASSWORD comes from the ExternalSecret (hindsight-credentials,
+# key `postgres-password`) — the same value the chart interpolates into
+# HINDSIGHT_API_DATABASE_URL, so the server login and the API DSN agree.
+# ============================================================================
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hindsight-postgres
+  namespace: hindsight
+  labels:
+    app.kubernetes.io/name: hindsight
+    app.kubernetes.io/component: database
+    app.kubernetes.io/part-of: platform-buildout
+spec:
+  replicas: 1
+  strategy:
+    # Recreate: RWO PVC + single writer. No rolling overlap (no two pods on one volume).
+    type: Recreate
+  selector:
+    matchLabels:
+      app: hindsight-postgres
+  template:
+    metadata:
+      labels:
+        app: hindsight-postgres
+        app.kubernetes.io/name: hindsight
+        app.kubernetes.io/component: database
+    spec:
+      containers:
+        - name: postgres
+          image: docker.io/ankane/pgvector@sha256:956744bd14e9cbdf639c61c2a2a7c7c2c48a9c8cdd42f7de4ac034f4e96b90f8
+          # Digest-pinned (immutable) => IfNotPresent avoids redundant re-pulls.
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: postgres
+              containerPort: 5432
+              protocol: TCP
+          env:
+            - name: POSTGRES_USER
+              value: hindsight
+            - name: POSTGRES_DB
+              value: hindsight
+            - name: POSTGRES_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: hindsight-credentials
+                  key: postgres-password
+          volumeMounts:
+            - name: postgres-data
+              # Stock postgres data path (ankane/pgvector is postgres:17-based,
+              # no PGDATA override needed — unlike firecrawl's custom layout).
+              mountPath: /var/lib/postgresql/data
+          resources:
+            requests:
+              cpu: 500m
+              memory: 1Gi
+            limits:
+              cpu: "1"
+              memory: 2Gi
+          livenessProbe:
+            exec:
+              command:
+                - pg_isready
+                - -U
+                - hindsight
+            initialDelaySeconds: 30
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          readinessProbe:
+            exec:
+              command:
+                - pg_isready
+                - -U
+                - hindsight
+            initialDelaySeconds: 5
+            periodSeconds: 5
+            timeoutSeconds: 3
+            failureThreshold: 3
+      volumes:
+        - name: postgres-data
+          persistentVolumeClaim:
+            claimName: hindsight-postgres-data

cluster/applications/hindsight/externalsecret.yaml

@@ -0,0 +1,92 @@
+# ============================================================================
+# ExternalSecret - Hindsight Credentials
+# Peter Parker, Phase C, t_6d47a360
+#
+# Syncs Hindsight credentials from the 1Password `mk-labs` vault (item
+# "hindsight", UUID q6pwoohexogdyvitt7sm2xcusu).
+# Pattern: openviking proven pattern (cluster/platform/openviking/externalsecret.yaml).
+# Store: onepassword-connect ClusterSecretStore.
+# Namespace: hindsight.
+#
+# Wave: -1  — must sync BEFORE the chart's Deployments (which default to wave 0)
+# so the materialized Secret exists before the api / control-plane pods attempt
+# envFrom / secretKeyRef injection.
+#
+# The materialized Secret (hindsight-credentials) must carry keys in TWO shapes
+# (Phase B record: inbox/ryan/2026-08-24-hindsight-phase-b-secrets.md):
+#   1. `postgres-password`  (lowercase, hyphenated) — read by the chart's EXPLICIT
+#      secretKeyRef for POSTGRES_PASSWORD, NOT via envFrom. A naming error here
+#      silently drops the DB password (pod starts, then fails to connect).
+#   2. `HINDSIGHT_API_LLM_API_KEY` / `HINDSIGHT_API_MCP_AUTH_TOKEN` — env-var-named,
+#      injected via envFrom (always runs when existingSecret is set).
+#
+# 1Password item "hindsight" fields (all CONCEALED):
+#   - postgres-password                (32 chars, letters+digits only, URL-safe)
+#   - HINDSIGHT_API_MCP_AUTH_TOKEN     (48 chars urlsafe bearer token)
+# HINDSIGHT_API_LLM_API_KEY now sourced from 1Password item "vllm" (vault
+# mk-labs), field "api-key" -- vLLM DOES validate its bearer token (unlike
+# llama-swap, which never checked one), so this must be a real matching
+# secret post-cutover (t_5508360a, 2026-08-31). See values.yaml api.env
+# comment and roles/deploy-vllm/README.md for the full cutover writeup.
+# ============================================================================
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: hindsight-credentials
+  namespace: hindsight
+  annotations:
+    # Wave -1: ensure the ExternalSecret syncs BEFORE the chart's Deployments
+    # (wave 0). Guarantees the Secret exists before the api / control-plane
+    # pods attempt envFrom / secretKeyRef injection.
+    argocd.argoproj.io/sync-wave: "-1"
+    description: "Phase C secrets for Hindsight deployment (1Password mk-labs item: hindsight)"
+spec:
+  refreshInterval: "1h"
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: hindsight-credentials
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        # LOWERCASE, HYPHENATED — read by the chart's explicit secretKeyRef for
+        # POSTGRES_PASSWORD. Must be exactly "postgres-password".
+        postgres-password: "{{ .postgresPassword }}"
+        # env-var-named keys injected via envFrom
+        HINDSIGHT_API_MCP_AUTH_TOKEN: "{{ .HINDSIGHT_API_MCP_AUTH_TOKEN }}"
+        HINDSIGHT_API_LLM_API_KEY: "{{ .HINDSIGHT_API_LLM_API_KEY }}"
+        # NOTE: no HINDSIGHT_API_EMBEDDINGS_OPENAI_API_KEY here — Hindsight's
+        # embeddings stayed on its bundled local provider after the vLLM
+        # embeddings cutover attempt crash-looped the API on an embedding
+        # dimension mismatch (384 existing vs 768 nomic-embed). See
+        # values.yaml's api.env comment for the full incident writeup
+        # (t_e6facb19, 2026-08-31).
+  data:
+    # 1Password item "hindsight", field "postgres-password".
+    # letters+digits only / URL-safe: required because the chart interpolates the
+    # value RAW into HINDSIGHT_API_DATABASE_URL (no URL-encoding).
+    - secretKey: postgresPassword
+      remoteRef:
+        key: hindsight
+        property: postgres-password
+    # 1Password item "hindsight", field "HINDSIGHT_API_MCP_AUTH_TOKEN"
+    # (MCP bearer token, D6 — enables bearer-auth on the /mcp endpoint)
+    - secretKey: HINDSIGHT_API_MCP_AUTH_TOKEN
+      remoteRef:
+        key: hindsight
+        property: HINDSIGHT_API_MCP_AUTH_TOKEN
+    # CUT OVER to vLLM's real auth key (t_5508360a, 2026-08-31): llama-swap
+    # is now stopped+disabled on astro-orbiter (dashboard-approved breaking
+    # change) and Hindsight's LLM path points at vLLM :8000
+    # (Qwen2.5-32B-Instruct-AWQ), which DOES validate its API key (unlike
+    # llama-swap, which ignored it). Source is the same 1Password item
+    # deploy-vllm's Ansible role reads at deploy time
+    # (op://mk-labs/vllm/api-key) -- keeps exactly one source of truth for
+    # the vLLM auth key across the Ansible-deployed server and this
+    # Kubernetes-deployed consumer.
+    - secretKey: HINDSIGHT_API_LLM_API_KEY
+      remoteRef:
+        key: vllm
+        property: api-key

cluster/applications/hindsight/namespace.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: hindsight
+  labels:
+    name: hindsight
+    app.kubernetes.io/name: hindsight
+    app.kubernetes.io/component: application
+    app.kubernetes.io/part-of: platform-buildout
+    epcot-theme: cosmic-rewind
+  annotations:
+    description: "Hindsight agent-memory service (Phase C) - memory retain/recall for all agent profiles on fastpass"

cluster/applications/hindsight/postgres-pvc.yaml

@@ -0,0 +1,24 @@
+---
+# ============================================================================
+# PersistentVolumeClaim - Hindsight external PostgreSQL data
+# Peter Parker, Phase C, t_6d47a360
+#
+# 10Gi on nfs-emporium (homelab NFS-backed storage class, firecrawl precedent).
+# RWO single-writer: matches the single-replica Recreate postgres Deployment.
+# ============================================================================
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: hindsight-postgres-data
+  namespace: hindsight
+  labels:
+    app.kubernetes.io/name: hindsight
+    app.kubernetes.io/component: database
+    app.kubernetes.io/part-of: platform-buildout
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: nfs-emporium
+  resources:
+    requests:
+      storage: 10Gi

cluster/applications/hindsight/service-postgres.yaml

@@ -0,0 +1,27 @@
+---
+# ============================================================================
+# Service - Hindsight external PostgreSQL
+# Peter Parker, Phase C, t_6d47a360
+#
+# ClusterIP service named `hindsight-postgres` — this is the host the chart's
+# external Postgres config points at (values.yaml: postgresql.external.host).
+# The Hindsight API reaches it at hindsight-postgres:5432 within the namespace.
+# ============================================================================
+apiVersion: v1
+kind: Service
+metadata:
+  name: hindsight-postgres
+  namespace: hindsight
+  labels:
+    app.kubernetes.io/name: hindsight
+    app.kubernetes.io/component: database
+    app.kubernetes.io/part-of: platform-buildout
+spec:
+  type: ClusterIP
+  selector:
+    app: hindsight-postgres
+  ports:
+    - name: postgres
+      port: 5432
+      targetPort: 5432
+      protocol: TCP

cluster/applications/hindsight/values.yaml

@@ -0,0 +1,190 @@
+# ============================================================================
+# Hindsight — helm values (Phase C). Consumed by the ArgoCD Application source 1
+# via `helm.valueFiles: ["$values/values.yaml"]` (openviking multi-source pattern).
+#
+# Design decisions (all verified against chart v0.9.1 + rendered output):
+#   - Chart is the single source for the app (api, control-plane, services,
+#     probes, ingress). We do NOT hand-roll Deployments/Services.
+#   - Postgres is EXTERNAL (separate Deployment in this dir, firecrawl pattern)
+#     => postgresql.enabled: false, external.* points at hindsight-postgres:5432.
+#   - Secrets come from 1Password via ExternalSecret => existingSecret:
+#     hindsight-credentials. The chart does envFrom(secretRef) so
+#     HINDSIGHT_API_LLM_API_KEY / HINDSIGHT_API_MCP_AUTH_TOKEN are injected
+#     automatically; POSTGRES_PASSWORD is a secretKeyRef that K8s expands into
+#     HINDSIGHT_API_DATABASE_URL (verified with a live envFrom test pod).
+#   - LLM is the Nous free-tier inference endpoint
+#     (https://inference-api.nousresearch.com/v1), model
+#     `upstage/solar-pro4:free` (tool-calling, verified reflect). Fallback
+#     (documented, NOT deployed): `stepfun/step-3.7-flash:free`. API key via existingSecret
+#     envFrom (hindsight-credentials / HINDSIGHT_API_LLM_API_KEY), sourced from
+#     1Password `nous` item per decision 4.
+#   - Ingress is driven through the chart's NATIVE ingress template (approved
+#     plan: "Ingress driven through values.yaml"). api.service.port=8888,
+#     controlPlane.service.port=3000.
+#   - Image tag defaults to .Values.version (root) when api.image.tag is unset,
+#     so version: "0.9.1" pins the API image to 0.9.1.
+# ============================================================================
+
+version: "0.9.1"
+
+# ----------------------------------------------------------------------------
+# External PostgreSQL (chart's bundled postgresql is disabled).
+# password is the K8s env expansion `$(POSTGRES_PASSWORD)` — the chart defines
+# POSTGRES_PASSWORD as a secretKeyRef (hindsight-credentials / postgres-password)
+# earlier in the same container, so K8s substitutes it at container start.
+# ----------------------------------------------------------------------------
+postgresql:
+  enabled: false
+  external:
+    host: hindsight-postgres
+    port: 5432
+    username: hindsight
+    database: hindsight
+    password: $(POSTGRES_PASSWORD)
+
+# ExternalSecret (from 1Password) that the chart injects via envFrom(secretRef).
+# Keys it must expose: postgres-password, HINDSIGHT_API_LLM_API_KEY,
+# HINDSIGHT_API_MCP_AUTH_TOKEN. See externalsecret.yaml in this dir.
+existingSecret: hindsight-credentials
+
+# ----------------------------------------------------------------------------
+# API container environment (explicit env entries; the chart renders this map
+# to individual env vars). LLM points at the Nous free-tier inference endpoint
+# (https://inference-api.nousresearch.com/v1), model upstage/solar-pro4:free
+# (tool-calling, verified retain/recall). stepfun/step-3.7-flash was swapped out
+# because it rejects Hindsight's tagged fact-extraction (BadRequestError 400
+# 'missing tags'); solar-pro4 is the previously-verified-good Nous model for the
+# extract/retain path (tasks t_e3375410 / t_d0dffc3d).
+# HINDSIGHT_API_LLM_API_KEY is NOT set here — it comes from the existingSecret
+# via envFrom (1Password `nous` item).
+# ----------------------------------------------------------------------------
+api:
+  env:
+    # CUT OVER to vLLM (t_5508360a, 2026-08-31): dashboard explicitly
+    # approved "stop and disable llama-swap and start vLLM" as a breaking
+    # change. llama-swap is now stopped+disabled on astro-orbiter; vLLM
+    # was the permanent, boot-persistent replacement (originally
+    # Qwen2.5-32B-Instruct-AWQ). This exact config (base URL, model name,
+    # retry-safe low concurrency) was validated end-to-end in t_e6facb19's
+    # shadow window (health, /v1/chat/completions, live hindsight_retain+
+    # recall round-trip) before that task reverted it pending this
+    # decision -- now re-applied for real. See roles/deploy-vllm/README.md
+    # "Critical architectural finding" + "Consumer cutover status" for the
+    # full history.
+    #
+    # MODEL SWAP (t_r1d32b_swap, 2026-09-01): astro-orbiter's vLLM primary
+    # model changed from Qwen2.5-32B-Instruct-AWQ to
+    # DeepSeek-R1-Distill-Qwen-32B-AWQ (single-model deployment; nomic-embed
+    # and Qwen3-8B-AWQ both disabled on that host). Superseded below.
+    #
+    # MODEL SWAP #2 (t_gemma4_swap, 2026-09-01): DeepSeek-R1-Distill-Qwen-32B
+    # retired after confirming its tool_choice="auto" reliability is a
+    # known, documented DeepSeek-R1-distillation limitation (upstream
+    # GitHub-confirmed: trained on pure reasoning traces, no function-
+    # calling data) -- not relevant to Hindsight's pure-text extraction
+    # use case, but disqualifying for agent-facing Hermes profiles, which
+    # drove the swap. Replaced with Gemma 4 26B A4B (Google, Apache 2.0,
+    # US-origin). Same endpoint (http://astro-orbiter:8000/v1), same API
+    # key -- only the served model name changed. Gemma 4 does NOT emit a
+    # reasoning trace by default (confirmed live) -- simpler completion
+    # parsing than DeepSeek-R1's always-on <think> blocks.
+    HINDSIGHT_API_LLM_BASE_URL: "http://astro-orbiter:8000/v1"
+    HINDSIGHT_API_LLM_PROVIDER: "openai"
+    HINDSIGHT_API_LLM_MODEL: "Gemma-4-26B-A4B-it-AWQ"
+    # vLLM's max_model_len is now 65536 (up from DeepSeek's 32768, up from
+    # the original 8192 role default). Gemma 4's native context is 256K;
+    # 65536 is astro-orbiter's configured ceiling, comfortably above
+    # Hermes's 64K floor. Completion cap left at 4096 pending live
+    # verification -- Gemma 4 doesn't burn tokens on unwanted reasoning
+    # traces the way DeepSeek-R1 did, so 4096 should have MORE effective
+    # headroom for the actual extraction output than it did before.
+    HINDSIGHT_API_RETAIN_MAX_COMPLETION_TOKENS: "4096"
+    # DO NOT set HINDSIGHT_API_EMBEDDINGS_* here (t_e6facb19, 2026-08-31
+    # attempted this, reverted after a production incident — see below).
+    #
+    # DISCOVERY: Hindsight's embeddings provider was NEVER pointed at
+    # astro-orbiter. It defaults to "local" (bundled sentence-transformers,
+    # BAAI/bge-small-en-v1.5, 384 dimensions) whenever
+    # HINDSIGHT_API_EMBEDDINGS_PROVIDER is unset — verified via
+    # `kubectl exec ... env | grep -i embed` showing NO
+    # HINDSIGHT_API_EMBEDDINGS_* vars in the live pod, despite this file's
+    # LLM section referencing astro-orbiter for years. The nomic-embed-
+    # text-v1.5 model documented across mk-labs skills as "Hindsight's
+    # embedding model" was OpenViking's embedding model, not Hindsight's.
+    #
+    # INCIDENT: pointing HINDSIGHT_API_EMBEDDINGS_PROVIDER at vLLM's
+    # nomic-embed-text-v1.5 (768 dimensions) crash-looped hindsight-api on
+    # rollout: `RuntimeError: Cannot change embedding dimension from 384 to
+    # 768: memory_units table contains 1289 rows with embeddings.` The
+    # migration path (`ensure_embedding_dimension` in migrations.py) refuses
+    # a live dimension change without either re-embedding everything or
+    # deleting all existing memory_units rows across every bank (jarvis,
+    # hermes, war-machine, and ~18 other agent banks) — a destructive,
+    # irreversible operation requiring explicit human approval, not
+    # something to do as a side effect of an infra migration task. Reverted
+    # immediately; Hindsight keeps its bundled local embedder (384-dim,
+    # unchanged, zero data risk) until a deliberate, approved re-embedding
+    # migration is planned as its own task.
+    # --- t_d7f8cd65: fix 502s on the serial astro-orbiter node ---
+    # astro-orbiter is a single llama-swap process (serial: 1 generate at a
+    # time, ctx 64K). Hindsight's default LLM concurrency is 32, so a retain
+    # burst hits the node with N parallel calls -> the node rejects/times out
+    # the extras -> hindsight-api surfaces APITimeoutError as 502. astro-orbiter
+    # is the ONLY LLM endpoint (all ops route there), so cap the whole pool to
+    # 1 and pin retain to 1 as well. The upstream chart exposes these as native
+    # semaphore config (HINDSIGHT_API_*_MAX_CONCURRENT); no code change needed.
+    # Kept at 1 post-cutover: vLLM's single-process-per-model design is also
+    # effectively serial for a single generative model instance under this
+    # GPU's VRAM budget (KV cache sized tight against the 24GB card at
+    # max_model_len=65536 for Gemma-4-26B-A4B-it-AWQ, t_gemma4_swap
+    # 2026-09-01 — previously 32768 for DeepSeek-R1-Distill-Qwen-32B-AWQ,
+    # previously 8192 for Qwen2.5-32B-Instruct-AWQ).
+    HINDSIGHT_API_LLM_MAX_CONCURRENT: "1"
+    HINDSIGHT_API_RETAIN_LLM_MAX_CONCURRENT: "1"
+    # Client + per-request timeout. Default is 120s; a 29K-token retain runs
+    # ~29s and under load a single long retain can reach ~90s. Raise to 600s to
+    # cover the longest round-trip so the serial call never times out the client
+    # (keep >= ingress proxy-read-timeout below). Per-op retain timeout pins the
+    # retain path explicitly; the global timeout covers reflect/consolidation.
+    HINDSIGHT_API_LLM_TIMEOUT: "600"
+    HINDSIGHT_API_RETAIN_LLM_TIMEOUT: "600"
+
+# ----------------------------------------------------------------------------
+# Ingress via the chart's native template.
+#   /health,/v1,/mcp,/ext  -> api:8888        (longest-prefix wins in nginx)
+#   /                      -> controlPlane:3000
+# TLS secret hindsight-tls provisioned by the letsencrypt-prod issuer.
+# ----------------------------------------------------------------------------
+ingress:
+  enabled: true
+  className: "nginx"
+  annotations:
+    cert-manager.io/cluster-issuer: "letsencrypt-prod"
+    # Raised read/send timeout so a slow agentic reflect / long single retain
+    # (up to ~90s under load on the serial astro-orbiter node; client timeout
+    # is 600s per t_d7f8cd65) can complete before nginx cuts the connection.
+    # Raised 300 -> 600 (t_d7f8cd65) to cover the longest retain round-trip.
+    nginx.ingress.kubernetes.io/proxy-read-timeout: "600"
+    nginx.ingress.kubernetes.io/proxy-send-timeout: "600"
+  hosts:
+    - host: cosmic-rewind.local.mk-labs.cloud
+      paths:
+        - path: /health
+          pathType: Prefix
+          service: api
+        - path: /v1
+          pathType: Prefix
+          service: api
+        - path: /mcp
+          pathType: Prefix
+          service: api
+        - path: /ext
+          pathType: Prefix
+          service: api
+        - path: /
+          pathType: Prefix
+          service: controlPlane
+  tls:
+    - hosts:
+        - cosmic-rewind.local.mk-labs.cloud
+      secretName: hindsight-tls

cluster/applications/open-webui/application.yaml

@@ -0,0 +1,50 @@
+# ============================================================================
+# ArgoCD Application: Body Wars Observability WebUI (Open WebUI)
+# Wave: 9 (post-Hindsight observability layer)
+# Deployment method: GitOps (Gitea -> ArgoCD)
+# ============================================================================
+#
+# Connects Open WebUI to astro-orbiter vLLM (Gemma-4-26B-A4B-it-AWQ).
+# Manifests: namespace, service account, deployment, service, ingress, secrets.
+# TLS: letsencrypt-prod
+#
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: open-webui
+  namespace: argocd
+  labels:
+    app.kubernetes.io/name: open-webui
+    app.kubernetes.io/part-of: mk-labs
+  annotations:
+    # Wave 9: Observability layer, after Hindsight (Wave 8)
+    argocd.argoproj.io/sync-wave: "9"
+    description: |
+      Body Wars: Observability WebUI for vLLM inference.
+      Frontend: Open WebUI (latest)
+      Backend: http://astro-orbiter:8000/v1 (vLLM OpenAI-compatible)
+      Model: Gemma-4-26B-A4B-it-AWQ
+      Ingress: body-wars.local.mk-labs.cloud (TLS via letsencrypt-prod)
+      Auth: API key from 1Password (op://mk-labs/vllm/api-key)
+spec:
+  project: default
+
+  source:
+    repoURL: https://gitea.mk-labs.cloud/rblundon/homelab.git
+    targetRevision: main
+    path: cluster/applications/open-webui
+    directory:
+      exclude: "application.yaml"
+
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: open-webui
+
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true
+      - ServerSideApply=true
+      - PrunePropagationPolicy=background

cluster/applications/open-webui/deployment.yaml

@@ -0,0 +1,92 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: open-webui
+  namespace: open-webui
+  labels:
+    app.kubernetes.io/name: open-webui
+    app.kubernetes.io/part-of: open-webui
+spec:
+  replicas: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 0
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: open-webui
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: open-webui
+        app.kubernetes.io/part-of: open-webui
+    spec:
+      serviceAccountName: open-webui
+      containers:
+      - name: open-webui
+        image: ghcr.io/open-webui/open-webui:latest
+        imagePullPolicy: IfNotPresent
+        ports:
+        - name: http
+          containerPort: 8080
+          protocol: TCP
+        env:
+        # OpenWebUI configuration
+        - name: WEBUI_SECRET_KEY
+          valueFrom:
+            secretKeyRef:
+              name: body-wars-secret
+              key: vllm-api-key
+              optional: false
+        # Backend configuration for vLLM (OpenAI-compatible, NOT Ollama)
+        # OLLAMA_BASE_URLS removed: astro-orbiter runs vLLM, not Ollama
+        - name: OPENAI_BASE_URL
+          value: "http://astro-orbiter:8000/v1"
+        - name: OPENAI_API_KEY
+          valueFrom:
+            secretKeyRef:
+              name: body-wars-secret
+              key: vllm-api-key
+              optional: false
+        - name: OPENAI_MODEL_FALLBACK
+          value: "Gemma-4-26B-A4B-it-AWQ"
+        # Disable OpenAI API key validation (we're using vLLM, not OpenAI)
+        - name: OPENAI_VALIDATE_ENDPOINT
+          value: "false"
+        # Allow access from ingress
+        - name: WEBUI_URL
+          value: "https://body-wars.local.mk-labs.cloud"
+        # Observability settings
+        - name: ENABLE_API_KEY
+          value: "true"
+        - name: ENABLE_MODEL_FILTER
+          value: "false"
+        # Disable Ollama model detection (we use vLLM/OpenAI-compatible only)
+        - name: ENABLE_OLLAMA_API
+          value: "false"
+        # Storage
+        - name: DATA_DIR
+          value: "/app/backend/data"
+        resources:
+          requests:
+            cpu: 100m
+            memory: 256Mi
+          limits:
+            cpu: 1000m
+            memory: 1Gi
+        livenessProbe:
+          httpGet:
+            path: /health
+            port: http
+          initialDelaySeconds: 30
+          periodSeconds: 10
+          timeoutSeconds: 5
+          failureThreshold: 3
+        volumeMounts:
+        - name: data
+          mountPath: /app/backend/data
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: open-webui-data

cluster/applications/open-webui/externalsecret.yaml

@@ -0,0 +1,32 @@
+apiVersion: external-secrets.io/v1beta1
+kind: ExternalSecret
+metadata:
+  name: body-wars-credentials
+  namespace: open-webui
+  annotations:
+    # Sync priority: ExternalSecret fetches from 1Password before other resources deploy
+    argocd.argoproj.io/sync-wave: "-1"
+spec:
+  refreshInterval: 1h
+  secretStoreRef:
+    kind: ClusterSecretStore
+    name: onepassword-connect
+  target:
+    name: body-wars-secret
+    creationPolicy: Owner
+    template:
+      engineVersion: v2
+      data:
+        vllm-api-key: "{{ .vllm_api_key }}"
+  data:
+    # vLLM API key from 1Password (mk-labs vault, vllm item, api-key field)
+    - secretKey: vllm_api_key
+      remoteRef:
+        key: vllm
+        property: api-key
+    # WebUI JWT signing secret - TEMPORARILY DISABLED
+    # Uncomment and configure once open-webui item exists in 1Password:
+    # - secretKey: webui_secret_key
+    #   remoteRef:
+    #     key: open-webui
+    #     property: secret-key

cluster/applications/open-webui/httproute.yaml

@@ -0,0 +1,44 @@
+# ------------------------------------------------------------------------------
+# HTTPRoute — Skyway via Cilium Gateway
+# ------------------------------------------------------------------------------
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: open-webui-tls
+  namespace: open-webui
+spec:
+  secretName: skyway-tls
+  issuerRef:
+    name: letsencrypt-prod
+    kind: ClusterIssuer
+  dnsNames:
+    - body-wars.local.mk-labs.cloud
+---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: body-wars
+  namespace: open-webui
+  annotations:
+    external-dns.alpha.kubernetes.io/hostname: body-wars.local.mk-labs.cloud
+    external-dns.alpha.kubernetes.io/target: "10.1.71.90"
+spec:
+  parentRefs:
+    - group: gateway.networking.k8s.io
+      kind: Gateway
+      name: fastpass-gateway
+      namespace: gateway
+      sectionName: https
+  hostnames:
+    - body-wars.local.mk-labs.cloud
+  rules:
+    - matches:
+        - path:
+            type: PathPrefix
+            value: /
+      backendRefs:
+        - group: ""
+          kind: Service
+          name: open-webui
+          port: 80
+          weight: 1

cluster/applications/open-webui/namespace.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: open-webui
+  labels:
+    app.kubernetes.io/name: open-webui
+    app.kubernetes.io/part-of: mk-labs

cluster/applications/open-webui/pvc.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: open-webui-data
+  namespace: open-webui
+  labels:
+    app.kubernetes.io/name: open-webui
+    app.kubernetes.io/component: data
+spec:
+  accessModes:
+    - ReadWriteOnce
+  storageClassName: nfs-emporium
+  resources:
+    requests:
+      storage: 10Gi

cluster/applications/open-webui/service.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: open-webui
+  namespace: open-webui
+  labels:
+    app.kubernetes.io/name: open-webui
+    app.kubernetes.io/part-of: open-webui
+spec:
+  type: ClusterIP
+  ports:
+  - name: http
+    port: 80
+    targetPort: http
+    protocol: TCP
+  selector:
+    app.kubernetes.io/name: open-webui

cluster/applications/open-webui/serviceaccount.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: open-webui
+  namespace: open-webui
+  labels:
+    app.kubernetes.io/name: open-webui
+    app.kubernetes.io/part-of: open-webui

cluster/platform/astro-orbiter-gateway/externalname-service.yaml

@@ -0,0 +1,37 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: astro-orbiter
+  namespace: default
+  labels:
+    app.kubernetes.io/name: astro-orbiter-gateway
+    app.kubernetes.io/component: llm-inference
+spec:
+  type: ExternalName
+  externalName: 10.1.71.130
+  ports:
+  - name: vllm-8000
+    port: 8000
+    protocol: TCP
+  - name: llama-cpp-8001
+    port: 8001
+    protocol: TCP
+  - name: router-8002
+    port: 8002
+    protocol: TCP
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: astro-orbiter-vllm
+  namespace: default
+  labels:
+    app.kubernetes.io/name: astro-orbiter-gateway
+    app.kubernetes.io/component: llm-inference
+spec:
+  type: ExternalName
+  externalName: 10.1.71.130
+  ports:
+  - name: vllm
+    port: 8000
+    protocol: TCP

cluster/platform/openviking/values.yaml

@@ -140,7 +140,7 @@ config:
   embedding:
     dense:
       provider: "openai"
-      api_base: "http://astro-orbiter:8002/v1"
+      api_base: "http://astro-orbiter:8001/v1"
       api_key: "${OPENVIKING_EMBEDDING_API_KEY}"  # Placeholder: "local-nomic" or similar
       model: "nomic-embed-text-v1.5"
       dimension: 768
@@ -179,7 +179,7 @@ config:
   # max_concurrent: 4 (recommend capping background indexing load on shared VLM)
   # ============================================================================
   vlm:
-    api_base: "http://astro-orbiter:8002/v1"
+    api_base: "http://astro-orbiter:8001/v1"
     api_key: "${OPENVIKING_VLM_API_KEY}"  # Placeholder: "local-llama" or similar
     # Fixed 2026-08-15: "llama3.1-8b" does not exist on astro-orbiter's /v1/models
     # (caused every summarization call to fail with 400 model not found, endless