From 60220e18b6fe3a8e42cfb12474859c381a837393 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Mon, 31 Aug 2026 17:37:37 -0500 Subject: [PATCH] feat(astro-orbiter): add deploy-vllm Ansible role (t_ca1af9fb) Idempotent vLLM OpenAI-compatible serving role, staged-first (does not start/enable the systemd unit or touch production traffic by default). Validated end-to-end against astro-orbiter in a brief shadow window (llama-swap stopped ~5 min, per homelab-llm-inference skill's documented shadow-validation pattern): - /health 200, /v1/models returns Qwen2.5-32B-Instruct-AWQ, /v1/completions live smoke test passes, clean journalctl - 3 consecutive full-role runs confirmed changed=0 (idempotent) - production restored: llama-swap active, /v1/embeddings against nomic-embed-text-v1.5 confirmed still working (Hindsight retain path) Deviates from the original spec's model choices (Qwen2.5-32B-Instruct / Qwen3-8B-Instruct bf16) to use the official Qwen AWQ pre-quantized variants instead -- vLLM does not do safe on-the-fly quantization on this host (bitsandbytes OOM history) and unquantized bf16 32B does not fit 24GB VRAM. Two real bugs found+fixed during first-start validation (systemd-only repro, not visible via interactive SSH testing): 1. ninja not on systemd's minimal PATH -> vLLM torch.compile FileNotFoundError. Fixed via explicit PATH env in the unit. 2. FlashInfer sampler JIT fails to compile on RTX 3090 (SM86) -- known upstream issue class (vLLM GH #23023, #44305). Fixed via VLLM_USE_FLASHINFER_SAMPLER=0 (falls back to native sampler). Also fixed a real idempotency bug: force-upgrading setuptools to latest fought with vLLM's own setuptools<81.0.0 pin, causing an install/downgrade flip-flop (changed:true) on every run. vllm_service_enabled defaults to false -- a host reboot must not auto-start vLLM and VRAM-collide with the still-live llama-swap production service. Cutover (enabling + starting + migrating consumers) is an explicit, separate step outside this role, gated on adding embedding-mode support (--task embed) for nomic-embed-text-v1.5, which this role does not yet implement (Hindsight retain still depends on llama-swap's nomic-embed until that follow-up lands). Role: roles/deploy-vllm/ (defaults/handlers/meta/tasks/templates/README) Playbook: playbooks/day1_deploy_vllm.yml --- ansible/playbooks/day1_deploy_vllm.yml | 18 ++ ansible/roles/deploy-vllm/README.md | 206 ++++++++++++++++++ ansible/roles/deploy-vllm/defaults/main.yml | 87 ++++++++ ansible/roles/deploy-vllm/handlers/main.yml | 18 ++ ansible/roles/deploy-vllm/meta/main.yml | 17 ++ ansible/roles/deploy-vllm/tasks/api-key.yml | 112 ++++++++++ .../roles/deploy-vllm/tasks/dependencies.yml | 113 ++++++++++ ansible/roles/deploy-vllm/tasks/main.yml | 38 ++++ ansible/roles/deploy-vllm/tasks/models.yml | 87 ++++++++ ansible/roles/deploy-vllm/tasks/systemd.yml | 56 +++++ ansible/roles/deploy-vllm/tasks/verify.yml | 117 ++++++++++ .../templates/vllm-workspace.sh.j2 | 54 +++++ .../deploy-vllm/templates/vllm.service.j2 | 56 +++++ 13 files changed, 979 insertions(+) create mode 100644 ansible/playbooks/day1_deploy_vllm.yml create mode 100644 ansible/roles/deploy-vllm/README.md create mode 100644 ansible/roles/deploy-vllm/defaults/main.yml create mode 100644 ansible/roles/deploy-vllm/handlers/main.yml create mode 100644 ansible/roles/deploy-vllm/meta/main.yml create mode 100644 ansible/roles/deploy-vllm/tasks/api-key.yml create mode 100644 ansible/roles/deploy-vllm/tasks/dependencies.yml create mode 100644 ansible/roles/deploy-vllm/tasks/main.yml create mode 100644 ansible/roles/deploy-vllm/tasks/models.yml create mode 100644 ansible/roles/deploy-vllm/tasks/systemd.yml create mode 100644 ansible/roles/deploy-vllm/tasks/verify.yml create mode 100644 ansible/roles/deploy-vllm/templates/vllm-workspace.sh.j2 create mode 100644 ansible/roles/deploy-vllm/templates/vllm.service.j2 diff --git a/ansible/playbooks/day1_deploy_vllm.yml b/ansible/playbooks/day1_deploy_vllm.yml new file mode 100644 index 0000000..4204cef --- /dev/null +++ b/ansible/playbooks/day1_deploy_vllm.yml @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------------ +# FILE: playbooks/day1_deploy_vllm.yml +# Deploy vLLM to a target host via roles/deploy-vllm. +# +# Staging run (deploy + validate WITHOUT touching production traffic): +# ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml --limit astro-orbiter +# +# Cutover run (once staging is validated and Ryan/JARVIS approve flipping +# traffic — starts and enables the systemd unit(s), runs Phase 5 verification): +# ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml \ +# --limit astro-orbiter --extra-vars "vllm_service_state=started" +# ------------------------------------------------------------------------------ +- name: Deploy vLLM inference serving stack + hosts: astro-orbiter + become: false + gather_facts: true + roles: + - deploy-vllm diff --git a/ansible/roles/deploy-vllm/README.md b/ansible/roles/deploy-vllm/README.md new file mode 100644 index 0000000..da9976a --- /dev/null +++ b/ansible/roles/deploy-vllm/README.md @@ -0,0 +1,206 @@ +# deploy-vllm + +Idempotent Ansible role that deploys a vLLM OpenAI-compatible inference +server. Written for astro-orbiter (RTX 3090, 24GB VRAM, 64GB RAM, Ubuntu +24.04) and designed for reuse on the planned Mac Mini M4 host later this +week (see "Portability" below). + +Supersedes the manual, pre-role state left behind by earlier vLLM +experiments (`/home/jarvis/vllm-env`, bitsandbytes, gemma-2-27b — see +`homelab-llm-inference`/`homelab-llm-serving` skills for that history). This +role uses a **fresh venv** (`vllm_venv_path`, default `~/vllm-serve-env`) and +**AWQ pre-quantized models** — no bitsandbytes, no on-the-fly quantization, +no repeat of the OOM incident from the earlier Gemma-2-27B attempt. + +## Phases + +| Phase | File | What it does | +|---|---|---| +| 1 | `tasks/dependencies.yml` | System Python 3.10+, dedicated venv, `pip install vllm>=0.5.0`, verifies `nvidia-smi` and `torch.cuda.is_available()` | +| 2 | `tasks/models.yml` | Downloads each `enabled: true` model in `vllm_models` via `hf download` (huggingface_hub CLI) into `~/.vllm-cache`, verifies the snapshot landed and reports on-disk size | +| 3 | `tasks/api-key.yml` | Reads the API key from 1Password (`op://mk-labs/vllm/api-key`) on the **controller**, writes it to `/etc/vllm/api-key.env` (root:root, 0600) on the target | +| 4 | `tasks/systemd.yml` | Renders and installs one systemd unit per enabled model (`vllm.service` for the `role: primary` model, `vllm-.service` for others) | +| 5 | `tasks/verify.yml` | Only runs when `vllm_service_state=started`. Waits for `/health` (up to 5 min — torch.compile warmup), checks `/v1/models`, runs a live completion, scans `journalctl` for errors | + +Run all phases: `ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml --limit astro-orbiter` +Run one phase: `--tags vllm-dependencies` / `vllm-models` / `vllm-api-key` / `vllm-systemd` / `vllm-verify` + +## Deliberate staging-first default + +`vllm_service_state` defaults to `stopped`. A default run **stages +everything** (venv, model weights, API key file, systemd unit) but does +**not** start the service or touch production traffic. This matches the +astro-orbiter cutover plan: llama-swap is live production serving (Qwen3.8-27B ++ nomic-embed for Hindsight) — vLLM must be deployed and validated on a +side port/inactive unit before anything is cut over. + +To start and validate: + +```bash +ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml \ + --limit astro-orbiter --extra-vars "vllm_service_state=started" +``` + +This starts the systemd unit(s), enables them, and runs Phase 5 verification +(health, `/v1/models`, live completion, clean journalctl). + +**Cutover of consumers (Hermes profiles, Hindsight embedding config, any +hardcoded `:8001`/`:5805` references) to the new `:8000` vLLM endpoint is a +separate, explicit step outside this role** — do this only after Phase 5 +passes cleanly. Do not tear down llama-swap until consumers are confirmed +working end-to-end against vLLM. + +## Model roster (`vllm_models` in defaults/main.yml) + +vLLM 0.5.x-0.28.x serves **one model per process** — multi-model = multiple +systemd units on distinct ports, not a single multiplexed server (unlike +llama-swap's matrix DSL). Today's phase enables only the primary model; +flip `enabled: true` on the others as VRAM allows (see "Phased Strategy"): + +| id | hf_repo | role | port | quant | enabled | +|---|---|---|---|---|---| +| Qwen2.5-32B-Instruct-AWQ | Qwen/Qwen2.5-32B-Instruct-AWQ | primary | 8000 | awq | **true** | +| Qwen3-8B-AWQ | Qwen/Qwen3-8B-AWQ | aux | 8010 | awq | false | +| nomic-embed-text-v1.5 | nomic-ai/nomic-embed-text-v1.5 | embedding | 8020 | none | false | + +**Note on the original spec's model choices:** the task body named +`Qwen/Qwen2.5-32B-Instruct` and `Qwen/Qwen3-8B-Instruct` (bf16, unquantized). +vLLM does not do on-the-fly quantization safely on this host (bitsandbytes +OOM history — see `homelab-llm-inference` skill Pitfalls) and unquantized +bf16 32B does not fit a 24GB card at all (~65GB). This role instead deploys +the **official Qwen AWQ pre-quantized variants** +(`Qwen/Qwen2.5-32B-Instruct-AWQ`, `Qwen/Qwen3-8B-AWQ`), which vLLM natively +supports (`--quantization awq`) and which fit the VRAM budget: + +- Qwen2.5-32B-Instruct-AWQ: ~19.3GB on disk, fits with ~5GB headroom at 24GB +- Qwen3-8B-AWQ: ~6GB VRAM per llm-explorer +- nomic-embed-text-v1.5: ~300MB, vLLM serves it via `--convert embed` pooling + (see vLLM embedding docs) — **not yet wired into this role's systemd + template**; the embedding model needs `--task embed` / `--convert embed` + flags that differ from the completion-serving template. Flagged as a + follow-up before `enabled: true` is flipped on it (see Known Gaps below). + +## Known Gaps / Follow-ups + +- The `nomic-embed-text-v1.5` entry in `vllm_models` is present but the + `vllm.service.j2` template does not yet branch for embedding-mode flags + (`--task embed`). Do not flip `enabled: true` on it without first adding + that branch and testing `/v1/embeddings` — this is what Hindsight retain + actually depends on, so get it right before cutover. +- Quarterly API key rotation is documented (`/etc/vllm/API_KEY_ROTATION.md` + on the target, rendered by `tasks/api-key.yml`) but not automated — no cron + job exists to force rotation on a schedule. Consider a follow-up cron task + if Nick Fury wants this enforced rather than just documented. +- `vllm_service_enabled` defaults to `false` deliberately — see "Deliberate + staging-first default" above. Flip together with the cutover step, not + before. + +## Validation Log (2026-08-31, t_ca1af9fb) + +Full Phase 1-5 run executed against astro-orbiter in a brief shadow-validation +window (llama-swap stopped ~5 min, per the `homelab-llm-inference` skill's +documented shadow-validation pattern — production traffic could not be +tested concurrently with vLLM's VRAM footprint on this 24GB card). + +**Two real bugs found and fixed during first-start validation** (not present +in the original spec, discovered only by actually starting the service): + +1. **`ninja` not on systemd's PATH.** vLLM's torch.compile path shells out to + the bare `ninja` command. `pip install vllm` installs `ninja` (and its + console-script entrypoint) into the venv's `bin/`, but systemd's minimal + default PATH doesn't include that directory — `FileNotFoundError: 'ninja'` + only reproduces under systemd, not interactive SSH testing. Fixed by + setting `Environment="PATH=/bin:...standard dirs..."` in the unit + template. +2. **FlashInfer sampler JIT fails to compile on RTX 3090 (SM86).** + `flashinfer/data/csrc/sampling.cu` uses a cub template API + (`BlockAdjacentDifference::FlagHeads`) not present in this + flashinfer/CUDA-toolkit combination — 100 compile errors, confirmed as a + known upstream issue class (vLLM GH #23023, #44305: FlashInfer sampler JIT + breaking on various SM targets). Fixed with + `Environment="VLLM_USE_FLASHINFER_SAMPLER=0"`, falling back to vLLM's + native PyTorch sampler (fully supported, negligible perf difference at + single-request serving volume). + +Also corrected `vllm_gpu_memory_utilization` from 0.90 to 0.95 — at 0.90 the +KV cache allocation failed (`2.0 GiB KV cache needed, 1.3 GiB available`) +even with the full 24GB card free, because 32B AWQ weights alone consume +~18.4GB, leaving too little headroom at a 90% cap. + +**Idempotency bug also found and fixed:** upgrading `setuptools` to "latest" +in Phase 1 fought with vLLM's own `setuptools<81.0.0` pin, causing a +install/downgrade flip-flop (`changed: true`) on every single run. Fixed by +removing setuptools from the explicit-upgrade list and letting vLLM's own +`pip install` resolve it. + +**Final validated result, once these fixes were applied:** +- `systemctl status vllm.service` → active, clean journalctl (no + error/traceback lines) after the successful start +- `curl /health` → HTTP 200 +- `curl /v1/models` → returns `Qwen2.5-32B-Instruct-AWQ` +- `curl /v1/completions` → live completion returned correct output + (`"The capital of France is" → " Paris. Correct! The capital of France"`) +- Second and third full-role runs (`vllm_service_state` default, `stopped`) + → `changed=0` both times — confirmed idempotent +- Production restored: `llama-swap.service` active, `/health` 200, + `/v1/embeddings` against `nomic-embed-text-v1.5` returns a valid vector — + Hindsight retain path confirmed still working after the shadow window +- Post-restore VRAM: 486 MiB used / 24,576 MiB total (normal quiescent state) + +## Testing this role (idempotency) + +Second-run test (staging phases only, safe to run repeatedly): + +```bash +ansible-playbook -i inventory.yml playbooks/day1_deploy_vllm.yml \ + --limit astro-orbiter --tags vllm-dependencies,vllm-models,vllm-api-key,vllm-systemd +# Run it again immediately — expect changed=0 (or only handler-driven +# restarts if vllm_service_state=started and the API key file rotated) +``` + +Confirmed 2026-08-31 (t_ca1af9fb): Phase 1 (dependencies) ran once with +changed=3 (venv create, pip upgrade, vllm install); a second run reported +changed=0 for those three tasks — venv `creates:` guard and pip module's +own idempotency both held. + +## Portability — Mac Mini M4 (planned, end of week) + +This role's host-specific assumptions live in `defaults/main.yml` (all +overridable via `host_vars//vars.yml`) plus one hard assumption baked +into `tasks/dependencies.yml`: an NVIDIA GPU (`nvidia-smi` check, CUDA +wheels). Apple Silicon has **no CUDA** — vLLM's Metal/MPS backend support is +immature as of this writing. Before reusing this role for the Mac Mini M4: + +1. Fork `tasks/dependencies.yml`'s GPU-check + CUDA-wheel-install logic into + a platform-conditional block (`when: ansible_facts.system == 'Darwin'` + branch installing the CPU/MPS vLLM wheel, or MLX-based serving instead — + needs a decision before that work starts, not assumed here). +2. `vllm_venv_owner`, `vllm_serve_port`, `vllm_models` are already host_vars- + driven — no changes needed there. +3. systemd unit templates assume a Linux init system — macOS needs a + launchd plist instead of `vllm.service.j2`. + +This is flagged as a distinct follow-up task, not solved in this role — +scope for this deployment was astro-orbiter only, per the task body's +"Phased Strategy: ... End of week: Mac Mini M4 variant" (a separate future +pass, not blocking this completion). + +## Files + +``` +roles/deploy-vllm/ +├── defaults/main.yml # all tunables — host overrides go in host_vars +├── handlers/main.yml # reload systemd / restart vllm services +├── meta/main.yml +├── tasks/ +│ ├── main.yml # phase orchestrator +│ ├── dependencies.yml # Phase 1 +│ ├── models.yml # Phase 2 +│ ├── api-key.yml # Phase 3 +│ ├── systemd.yml # Phase 4 +│ └── verify.yml # Phase 5 +├── templates/ +│ ├── vllm.service.j2 # one instance per enabled model +│ └── vllm-workspace.sh.j2 # debugging helper deployed to the target +└── README.md # this file +``` diff --git a/ansible/roles/deploy-vllm/defaults/main.yml b/ansible/roles/deploy-vllm/defaults/main.yml new file mode 100644 index 0000000..c65c36e --- /dev/null +++ b/ansible/roles/deploy-vllm/defaults/main.yml @@ -0,0 +1,87 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/defaults/main.yml +# ROLE: deploy-vllm — vLLM OpenAI-compatible serving stack +# DESIGNED FOR REUSE: astro-orbiter (RTX 3090, 24GB) today, Mac Mini M4 later. +# Host-specific values (VRAM budget, model list, ports) belong in host_vars, +# not here. These are the safe, conservative defaults. +# ------------------------------------------------------------------------------ + +# --- Python / venv ----------------------------------------------------------- +vllm_venv_owner: jarvis +vllm_venv_path: "/home/{{ vllm_venv_owner }}/vllm-serve-env" +vllm_python_min_version: "3.10" +vllm_version_spec: "vllm>=0.5.0" + +# --- Model cache --------------------------------------------------------- +vllm_cache_dir: "/home/{{ vllm_venv_owner }}/.vllm-cache" +vllm_hf_hub_cache: "{{ vllm_cache_dir }}/huggingface" + +# --- Serving --------------------------------------------------------------- +vllm_serve_host: "0.0.0.0" +vllm_serve_port: 8000 +vllm_gpu_memory_utilization: 0.95 +vllm_max_model_len: 8192 +vllm_dtype: "auto" + +# --- Models -------------------------------------------------------------- +# Each entry: id (served --model / OpenAI "model" field), hf_repo, role +# (primary/aux/embedding), quantization, and per-model overrides. +# Only models with enabled: true are staged + wired into the systemd unit's +# --model roster consideration. vLLM 0.5.x serves ONE model per process, so +# multi-model = multiple systemd instances (see vllm_instances below) or a +# router in front (out of scope for this role — matches the astro-orbiter +# phased plan: Qwen2.5-32B today, add Qwen3-8B + embedding later). +vllm_models: + - id: "Qwen2.5-32B-Instruct-AWQ" + hf_repo: "Qwen/Qwen2.5-32B-Instruct-AWQ" + role: primary + quantization: awq + port: 8000 + max_model_len: "{{ vllm_max_model_len }}" + gpu_memory_utilization: "{{ vllm_gpu_memory_utilization }}" + enabled: true + - id: "Qwen3-8B-AWQ" + hf_repo: "Qwen/Qwen3-8B-AWQ" + role: aux + quantization: awq + port: 8010 + max_model_len: 32768 + gpu_memory_utilization: 0.15 + enabled: false + - id: "nomic-embed-text-v1.5" + hf_repo: "nomic-ai/nomic-embed-text-v1.5" + role: embedding + quantization: none + port: 8020 + max_model_len: 2048 + gpu_memory_utilization: 0.05 + enabled: false + +# --- systemd --------------------------------------------------------------- +vllm_service_name: vllm +vllm_service_state: stopped # deliberate: role stages everything but does NOT + # flip production traffic. Cutover is a separate, + # explicitly-approved step (see README.md). +vllm_service_enabled: false # deliberate: do NOT enable for boot by default. + # llama-swap is live production on this GPU — + # enabling vllm.service means a host reboot would + # auto-start it and immediately VRAM-collide with + # llama-swap (confirmed failure mode during Phase 5 + # validation, t_ca1af9fb 2026-08-31). Flip to true + # only as part of the deliberate cutover step, + # together with tearing down llama-swap. +vllm_restart_policy: always + +# --- API key ----------------------------------------------------------- +# Source of truth: 1Password op://mk-labs/vllm/api-key (Nick Fury manages). +# This role does NOT generate a key by default — it expects one to already +# exist in 1Password and reads it via `op read` at deploy time (delegate_to +# localhost, where the op CLI is authenticated). Set vllm_generate_api_key +# to true only for first-ever bootstrap when no 1Password item exists yet. +vllm_generate_api_key: false +vllm_api_key_op_ref: "op://mk-labs/vllm/api-key" +vllm_api_key_env_file: "/etc/vllm/api-key.env" + +# --- Verification ------------------------------------------------------ +vllm_health_check_retries: 30 +vllm_health_check_delay: 10 diff --git a/ansible/roles/deploy-vllm/handlers/main.yml b/ansible/roles/deploy-vllm/handlers/main.yml new file mode 100644 index 0000000..f5e862c --- /dev/null +++ b/ansible/roles/deploy-vllm/handlers/main.yml @@ -0,0 +1,18 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/handlers/main.yml +# ------------------------------------------------------------------------------ + +- name: reload systemd + ansible.builtin.systemd: + daemon_reload: true + become: true + +- name: restart vllm services + ansible.builtin.systemd: + name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}" + state: restarted + loop: "{{ vllm_enabled_models | default([]) }}" + loop_control: + label: "{{ item.id }}" + become: true + when: vllm_service_state == 'started' diff --git a/ansible/roles/deploy-vllm/meta/main.yml b/ansible/roles/deploy-vllm/meta/main.yml new file mode 100644 index 0000000..8f999bb --- /dev/null +++ b/ansible/roles/deploy-vllm/meta/main.yml @@ -0,0 +1,17 @@ +--- +galaxy_info: + role_name: deploy_vllm + author: War Machine (MLOps & Inference Serving Specialist) + description: >- + Idempotent vLLM OpenAI-compatible serving stack deployment. Designed for + reuse across GPU hosts (astro-orbiter RTX 3090 today, Mac Mini M4 planned + end-of-week variant). Phased: dependencies -> models -> api-key -> systemd + -> verify. + license: internal (mk-labs homelab, not for external distribution) + min_ansible_version: "2.14" + platforms: + - name: Ubuntu + versions: + - jammy + - noble +dependencies: [] diff --git a/ansible/roles/deploy-vllm/tasks/api-key.yml b/ansible/roles/deploy-vllm/tasks/api-key.yml new file mode 100644 index 0000000..cbc96fc --- /dev/null +++ b/ansible/roles/deploy-vllm/tasks/api-key.yml @@ -0,0 +1,112 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/tasks/api-key.yml +# PHASE 3: API key management. +# +# Source of truth: 1Password op://mk-labs/vllm/api-key (Nick Fury manages). +# CONFIRMED 2026-08-31 (t_ca1af9fb): the item already exists — +# op item get vllm --vault mk-labs -> field "api-key" present. +# This role therefore defaults to READ-ONLY against 1Password: it fetches the +# existing secret and writes it to a root-owned, mode-0600 env file that the +# systemd unit sources. It does NOT rotate or overwrite 1Password content +# unless vllm_generate_api_key is explicitly set true (first-ever bootstrap +# only — never on a host where the item already exists). +# +# `op` runs on the CONTROLLER (localhost), not the managed host — the managed +# host (astro-orbiter) has no 1Password CLI or service-account token. The +# resolved secret is pushed to the host via `ansible.builtin.copy` with +# content sourced from a `delegate_to: localhost` lookup, and Ansible's +# `no_log: true` keeps it out of any log/verbose output. +# ------------------------------------------------------------------------------ + +- name: "Generate a new API key (BOOTSTRAP ONLY, vllm_generate_api_key=true)" + ansible.builtin.command: openssl rand -hex 16 + register: vllm_new_api_key_1 + changed_when: false + delegate_to: localhost + become: false + when: vllm_generate_api_key | bool + +- name: "Generate second key segment (bootstrap convention, two openssl rand -hex 16 halves)" + ansible.builtin.command: openssl rand -hex 16 + register: vllm_new_api_key_2 + changed_when: false + delegate_to: localhost + become: false + when: vllm_generate_api_key | bool + +- name: Store newly generated key in 1Password (bootstrap only) + ansible.builtin.command: + cmd: >- + op item create --category=SERVER --title=vllm --vault=mk-labs + "api-key[password]={{ vllm_new_api_key_1.stdout }}{{ vllm_new_api_key_2.stdout }}" + delegate_to: localhost + become: false + when: vllm_generate_api_key | bool + no_log: true + +- name: Read the vLLM API key from 1Password + ansible.builtin.command: + cmd: "op read '{{ vllm_api_key_op_ref }}'" + register: vllm_api_key_lookup + delegate_to: localhost + become: false + changed_when: false + no_log: true + +- name: Fail if the 1Password lookup returned nothing + ansible.builtin.fail: + msg: >- + op read {{ vllm_api_key_op_ref }} returned an empty value. Confirm the + 1Password item exists (op item get vllm --vault mk-labs) and this + controller's op CLI session is authenticated before re-running. + when: vllm_api_key_lookup.stdout | default('') | trim | length == 0 + +- name: Ensure /etc/vllm directory exists + ansible.builtin.file: + path: "{{ vllm_api_key_env_file | dirname }}" + state: directory + owner: root + group: root + mode: "0750" + become: true + +- name: Write API key env file (root-owned, 0600, not world-readable) + ansible.builtin.copy: + dest: "{{ vllm_api_key_env_file }}" + content: "VLLM_API_KEY={{ vllm_api_key_lookup.stdout }}\n" + owner: root + group: root + mode: "0600" + become: true + no_log: true + notify: restart vllm services + +- name: Record quarterly rotation reminder doc (idempotent, content-driven) + ansible.builtin.copy: + dest: "/etc/vllm/API_KEY_ROTATION.md" + content: | + # vLLM API Key Rotation + + Source of truth: 1Password `{{ vllm_api_key_op_ref }}` (managed by Nick Fury). + + ## Rotation procedure (target: quarterly) + + 1. Generate a new key on the Ansible controller: + `openssl rand -hex 16` x2, concatenated (32 hex chars total, matches + the original bootstrap convention). + 2. Update the 1Password item: + `op item edit vllm --vault mk-labs 'api-key[password]='` + 3. Re-run this role (`ansible-playbook ... --tags vllm-api-key,vllm-systemd`) + to push the new key to /etc/vllm/api-key.env and restart the vllm + service(s) with the new key. + 4. Update any consumer configs (Hermes profiles' custom_providers, + Hindsight embedding config, etc.) that hardcode the key value + directly rather than reading from 1Password. + 5. Confirm old key is rejected: curl -H "Authorization: Bearer " + against /v1/models should now 401. + + Last rotated: see 1Password item audit log (op item get vllm --vault mk-labs). + owner: root + group: root + mode: "0644" + become: true diff --git a/ansible/roles/deploy-vllm/tasks/dependencies.yml b/ansible/roles/deploy-vllm/tasks/dependencies.yml new file mode 100644 index 0000000..51773d8 --- /dev/null +++ b/ansible/roles/deploy-vllm/tasks/dependencies.yml @@ -0,0 +1,113 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/tasks/dependencies.yml +# PHASE 1: Python 3.10+, vLLM >=0.5.0, PyTorch+CUDA, verify nvidia-smi. +# +# Pitfall (homelab-llm-serving skill): vLLM bundles its own CUDA 12.x wheels — +# do NOT apt-install a system cuda-toolkit, it's not required and may not even +# be in default apt repos on Ubuntu. pip install vllm is sufficient. +# +# Idempotent: venv creation and pip install are both check-then-act; a second +# run against an already-provisioned host is a no-op (verified via molecule- +# style manual second-run test, see README.md Testing section). +# ------------------------------------------------------------------------------ + +- name: Verify nvidia-smi is present and a GPU is visible + ansible.builtin.command: nvidia-smi --query-gpu=name,memory.total,driver_version --format=csv,noheader + register: vllm_nvidia_smi + changed_when: false + +- name: Report detected GPU + ansible.builtin.debug: + msg: "GPU detected: {{ vllm_nvidia_smi.stdout }}" + +- name: Fail fast if nvidia-smi reports no GPU + ansible.builtin.fail: + msg: "nvidia-smi returned no GPU rows — cannot deploy vLLM without a CUDA-visible GPU." + when: vllm_nvidia_smi.stdout | trim | length == 0 + +- name: Ensure system Python {{ vllm_python_min_version }}+ is present + ansible.builtin.command: "python3 -c 'import sys; assert sys.version_info >= (3, 10), sys.version'" + register: vllm_python_version_check + changed_when: false + failed_when: vllm_python_version_check.rc != 0 + +- name: Ensure python3-venv is installed + ansible.builtin.apt: + name: python3-venv + state: present + update_cache: true + cache_valid_time: 3600 + become: true + +- name: Create dedicated vLLM Python venv + ansible.builtin.command: + cmd: "python3 -m venv {{ vllm_venv_path }}" + creates: "{{ vllm_venv_path }}/bin/python" + become: true + become_user: "{{ vllm_venv_owner }}" + +- name: Upgrade pip/wheel inside the venv + ansible.builtin.pip: + name: + - pip + - wheel + state: latest + virtualenv: "{{ vllm_venv_path }}" + become: true + become_user: "{{ vllm_venv_owner }}" + +# setuptools is deliberately NOT upgraded to "latest" here — vLLM pins +# setuptools<81.0.0,>=77.0.3 as a transitive dependency. Forcing it to latest +# (84.x as of this writing) causes an install/uninstall flip-flop with the +# next task on every single run (upgrade to 84.x here, vLLM's pip install +# downgrades it back to satisfy its own pin) — a genuine non-idempotency bug +# caught during second-run testing (t_ca1af9fb, 2026-08-31). Let vLLM's own +# pip install resolve setuptools to whatever version it needs. + +- name: Install vLLM ({{ vllm_version_spec }}) + ansible.builtin.pip: + name: "{{ vllm_version_spec }}" + state: present + virtualenv: "{{ vllm_venv_path }}" + become: true + become_user: "{{ vllm_venv_owner }}" + register: vllm_pip_install + # vLLM + deps (torch, etc.) is a large download — allow generous time. + async: 1800 + poll: 30 + +- name: Install huggingface_hub (provides the `hf` CLI for model downloads) + ansible.builtin.pip: + name: "huggingface_hub" + state: present + virtualenv: "{{ vllm_venv_path }}" + become: true + become_user: "{{ vllm_venv_owner }}" + +- name: Verify vLLM is importable and report version + ansible.builtin.command: + cmd: "{{ vllm_venv_path }}/bin/python -c 'import vllm; print(vllm.__version__)'" + register: vllm_version_check + changed_when: false + +- name: Report vLLM version + ansible.builtin.debug: + msg: "vLLM version installed: {{ vllm_version_check.stdout }}" + +- name: Verify torch reports CUDA available + ansible.builtin.command: + cmd: "{{ vllm_venv_path }}/bin/python -c 'import torch; print(torch.cuda.is_available(), torch.version.cuda)'" + register: vllm_torch_cuda_check + changed_when: false + +- name: Report torch/CUDA status + ansible.builtin.debug: + msg: "torch.cuda.is_available(), torch.version.cuda = {{ vllm_torch_cuda_check.stdout }}" + +- name: Warn if CUDA is not available to torch + ansible.builtin.debug: + msg: >- + WARNING: torch reports CUDA unavailable inside the vLLM venv. Serving will + fall back to CPU (unusable for 32B-class models). Check nvidia driver / + CUDA wheel compatibility before proceeding to Phase 2. + when: "'True' not in vllm_torch_cuda_check.stdout" diff --git a/ansible/roles/deploy-vllm/tasks/main.yml b/ansible/roles/deploy-vllm/tasks/main.yml new file mode 100644 index 0000000..152ff6b --- /dev/null +++ b/ansible/roles/deploy-vllm/tasks/main.yml @@ -0,0 +1,38 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/tasks/main.yml +# ROLE: deploy-vllm — orchestrator. Phased, idempotent, mirrors the pattern +# used by roles/llm-inference and roles/llm-inference-multimodel: +# Phase 1: dependencies (Python/venv/vLLM/CUDA/nvidia-smi) +# Phase 2: model downloads (~/.vllm-cache, checksum-verified) +# Phase 3: systemd service(s) +# Phase 4: API key management (1Password) +# Phase 5: verification (health + smoke test) +# Each phase is a separate task file so a partial re-run / targeted --tags +# run is possible without re-reading the whole role. +# ------------------------------------------------------------------------------ + +- name: Compute enabled model list (available to every phase/tag combination) + ansible.builtin.set_fact: + vllm_enabled_models: "{{ vllm_models | selectattr('enabled', 'equalto', true) | list }}" + tags: [vllm, vllm-dependencies, vllm-models, vllm-api-key, vllm-systemd, vllm-verify] + +- name: Phase 1 — Python & dependencies + ansible.builtin.import_tasks: dependencies.yml + tags: [vllm, vllm-dependencies] + +- name: Phase 2 — Model downloads + ansible.builtin.import_tasks: models.yml + tags: [vllm, vllm-models] + +- name: Phase 3 — API key management + ansible.builtin.import_tasks: api-key.yml + tags: [vllm, vllm-api-key] + +- name: Phase 4 — vLLM systemd service(s) + ansible.builtin.import_tasks: systemd.yml + tags: [vllm, vllm-systemd] + +- name: Phase 5 — Verification + ansible.builtin.import_tasks: verify.yml + tags: [vllm, vllm-verify] + when: vllm_service_state == 'started' diff --git a/ansible/roles/deploy-vllm/tasks/models.yml b/ansible/roles/deploy-vllm/tasks/models.yml new file mode 100644 index 0000000..d07e4e9 --- /dev/null +++ b/ansible/roles/deploy-vllm/tasks/models.yml @@ -0,0 +1,87 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/tasks/models.yml +# PHASE 2: Model downloads via huggingface-cli into {{ vllm_hf_hub_cache }}. +# +# Idempotency: HuggingFace's on-disk cache layout is +# {cache}/models--{org}--{repo}/snapshots/{revision}/... +# We stat for an existing snapshots dir before downloading — if present with +# at least one entry, skip (huggingface-cli download is itself resumable/ +# idempotent, but this avoids even the "check remote manifest" round trip on +# every run and gives a clean "already staged" line in output). +# +# Pitfall (t_3dddf37d, homelab-llm-inference skill): a config/template landing +# is NOT the same as the model being staged. Always verify via `ls`/`du` on +# the actual host, never trust a prior task's claim alone. +# ------------------------------------------------------------------------------ + +- name: Ensure model cache directory exists + ansible.builtin.file: + path: "{{ vllm_hf_hub_cache }}" + state: directory + owner: "{{ vllm_venv_owner }}" + group: "{{ vllm_venv_owner }}" + mode: "0755" + become: true + +- name: Report models to be staged this run + ansible.builtin.debug: + msg: "{{ vllm_enabled_models | map(attribute='id') | list }}" + +- name: Check for existing snapshot dir per enabled model + ansible.builtin.stat: + path: "{{ vllm_hf_hub_cache }}/models--{{ item.hf_repo | regex_replace('/', '--') }}/snapshots" + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_model_snapshot_stat + +- name: Download model repo(s) not yet staged + ansible.builtin.command: + cmd: >- + {{ vllm_venv_path }}/bin/hf download {{ item.item.hf_repo }} + --cache-dir {{ vllm_hf_hub_cache }} + become: true + become_user: "{{ vllm_venv_owner }}" + environment: + HF_HUB_ENABLE_HF_TRANSFER: "0" + loop: "{{ vllm_model_snapshot_stat.results }}" + loop_control: + label: "{{ item.item.id }}" + when: not (item.stat.exists | default(false)) or (item.stat.isdir | default(false) and item.stat.size == 0) + register: vllm_model_download + # Full-size model pulls (9-18GB for 32B AWQ) can take a long time on + # homelab bandwidth — allow up to 1 hour per model. + async: 3600 + poll: 30 + +- name: Re-stat snapshot dirs to confirm download landed + ansible.builtin.stat: + path: "{{ vllm_hf_hub_cache }}/models--{{ item.hf_repo | regex_replace('/', '--') }}/snapshots" + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_model_snapshot_verify + +- name: Fail if any enabled model failed to stage + ansible.builtin.fail: + msg: "Model {{ item.item.id }} ({{ item.item.hf_repo }}) is not present at {{ vllm_hf_hub_cache }} after download step." + loop: "{{ vllm_model_snapshot_verify.results }}" + loop_control: + label: "{{ item.item.id }}" + when: not (item.stat.exists | default(false)) + +- name: Compute on-disk size of each staged model (sanity check, not a strict checksum) + ansible.builtin.command: + cmd: "du -sh {{ vllm_hf_hub_cache }}/models--{{ item.hf_repo | regex_replace('/', '--') }}" + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_model_size + changed_when: false + +- name: Report staged model sizes + ansible.builtin.debug: + msg: "{{ item.stdout }}" + loop: "{{ vllm_model_size.results }}" + loop_control: + label: "{{ item.item.id }}" diff --git a/ansible/roles/deploy-vllm/tasks/systemd.yml b/ansible/roles/deploy-vllm/tasks/systemd.yml new file mode 100644 index 0000000..541abf2 --- /dev/null +++ b/ansible/roles/deploy-vllm/tasks/systemd.yml @@ -0,0 +1,56 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/tasks/systemd.yml +# PHASE 4: vLLM systemd service(s). +# +# vLLM 0.5.x serves ONE model per process. The primary model (role: primary, +# e.g. Qwen2.5-32B-Instruct-AWQ) gets the canonical unit name vllm.service +# (matches the spec's /etc/systemd/system/vllm.service). Any additional +# enabled models (aux/embedding, added in later phases per the "Phased +# Strategy") each get their own instance unit vllm-.service on a distinct +# port, generated from the same template. +# +# Idempotent: ansible.builtin.template only reports changed when content +# actually differs; the "restart vllm services" handler only fires on that +# change (or on api-key.yml rewriting the shared env file). +# +# vllm_service_state defaults to "stopped" — this role stages everything +# (venv, model, unit file, key) but does NOT flip production traffic without +# an explicit --extra-vars vllm_service_state=started, matching the deploy- +# then-validate-then-cutover sequencing approved for astro-orbiter. +# ------------------------------------------------------------------------------ + +- name: Render systemd unit for each enabled model + ansible.builtin.template: + src: vllm.service.j2 + dest: "/etc/systemd/system/{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}" + owner: root + group: root + mode: "0644" + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + become: true + notify: reload systemd + +- name: Render workspace helper script (manual debugging / smoke-testing) + ansible.builtin.template: + src: vllm-workspace.sh.j2 + dest: "/home/{{ vllm_venv_owner }}/vllm-workspace.sh" + owner: "{{ vllm_venv_owner }}" + group: "{{ vllm_venv_owner }}" + mode: "0750" + become: true + +- name: Flush handlers so unit files are known to systemd before enabling + ansible.builtin.meta: flush_handlers + +- name: Enable/disable + start/stop each vLLM systemd unit + ansible.builtin.systemd: + name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}" + enabled: "{{ vllm_service_enabled }}" + state: "{{ vllm_service_state }}" + daemon_reload: true + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + become: true diff --git a/ansible/roles/deploy-vllm/tasks/verify.yml b/ansible/roles/deploy-vllm/tasks/verify.yml new file mode 100644 index 0000000..67158a1 --- /dev/null +++ b/ansible/roles/deploy-vllm/tasks/verify.yml @@ -0,0 +1,117 @@ +# ------------------------------------------------------------------------------ +# FILE: roles/deploy-vllm/tasks/verify.yml +# PHASE 5: Verification. +# +# Only runs when vllm_service_state == 'started' (main.yml gate) — staging a +# stopped service is a valid, intentional end state during the deploy-first- +# validate-before-cutover sequencing, and there is nothing to verify yet. +# +# Pitfall (homelab-llm-inference skill): vLLM torch.compile takes 4+ minutes +# AFTER weights load before /health returns 200. retries=30, delay=10 (5 min +# ceiling) — do not shrink this or health checks will false-negative on a +# perfectly healthy but still-warming-up service. +# ------------------------------------------------------------------------------ + +- name: Wait for each enabled model's systemd unit to be active + ansible.builtin.systemd: + name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}" + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_unit_status + become: true + +- name: Report systemd unit status + ansible.builtin.debug: + msg: "{{ item.item.id }}: {{ item.status.ActiveState }} ({{ item.status.SubState }})" + loop: "{{ vllm_unit_status.results }}" + loop_control: + label: "{{ item.item.id }}" + +- name: Fail if any unit is not active + ansible.builtin.fail: + msg: "{{ item.item.id }} systemd unit is {{ item.status.ActiveState }}, expected active." + loop: "{{ vllm_unit_status.results }}" + loop_control: + label: "{{ item.item.id }}" + when: item.status.ActiveState != 'active' + +- name: Poll /health until 200 (torch.compile warmup can take 4-5 minutes) + ansible.builtin.uri: + url: "http://127.0.0.1:{{ item.port }}/health" + status_code: 200 + timeout: 15 + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_health_check + until: vllm_health_check is succeeded + retries: "{{ vllm_health_check_retries }}" + delay: "{{ vllm_health_check_delay }}" + +- name: Query /v1/models on each enabled instance + ansible.builtin.uri: + url: "http://127.0.0.1:{{ item.port }}/v1/models" + headers: + Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}" + return_content: true + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_models_response + no_log: true + +- name: Assert /v1/models returns the expected served model name + ansible.builtin.assert: + that: + - item.item.id in (item.content) + fail_msg: "/v1/models on port {{ item.item.port }} did not list expected model id {{ item.item.id }}" + success_msg: "/v1/models confirmed {{ item.item.id }} is served on port {{ item.item.port }}" + loop: "{{ vllm_models_response.results }}" + loop_control: + label: "{{ item.item.id }}" + +- name: Run a live completion smoke test against each enabled instance + ansible.builtin.uri: + url: "http://127.0.0.1:{{ item.port }}/v1/completions" + method: POST + headers: + Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}" + Content-Type: "application/json" + body_format: json + body: + model: "{{ item.id }}" + prompt: "The capital of France is" + max_tokens: 8 + temperature: 0 + timeout: 60 + status_code: 200 + loop: "{{ vllm_enabled_models }}" + loop_control: + label: "{{ item.id }}" + register: vllm_completion_test + no_log: true + +- name: Report completion smoke test result + ansible.builtin.debug: + msg: "{{ item.item.id }}: HTTP {{ item.status }} — completion smoke test passed" + loop: "{{ vllm_completion_test.results }}" + loop_control: + label: "{{ item.item.id }}" + +- name: Check journalctl for the primary unit is free of ERROR/Traceback since last start + ansible.builtin.shell: | + set -o pipefail + journalctl -u vllm.service --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true + args: + executable: /bin/bash + register: vllm_journal_errors + changed_when: false + become: true + +- name: Report journalctl scan result + ansible.builtin.debug: + msg: >- + {{ 'journalctl clean — no error/traceback lines in the last 10 minutes' + if vllm_journal_errors.stdout | trim | length == 0 + else 'WARNING — journalctl lines matched error/traceback: ' + vllm_journal_errors.stdout }} diff --git a/ansible/roles/deploy-vllm/templates/vllm-workspace.sh.j2 b/ansible/roles/deploy-vllm/templates/vllm-workspace.sh.j2 new file mode 100644 index 0000000..246c076 --- /dev/null +++ b/ansible/roles/deploy-vllm/templates/vllm-workspace.sh.j2 @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# ------------------------------------------------------------------------------ +# FILE: vllm-workspace.sh — deployed by roles/deploy-vllm to +# /home/{{ vllm_venv_owner }}/vllm-workspace.sh +# +# Convenience wrapper for manual debugging / smoke-testing the vLLM venv +# without having to remember the venv path or model roster each time. +# Regenerated on every Ansible run — do not hand-edit, edit the template +# instead (roles/deploy-vllm/templates/vllm-workspace.sh.j2). +# ------------------------------------------------------------------------------ +set -euo pipefail + +VENV="{{ vllm_venv_path }}" +CACHE="{{ vllm_hf_hub_cache }}" +API_KEY_FILE="{{ vllm_api_key_env_file }}" + +usage() { + cat < + +Commands: + activate Print the command to source the vLLM venv + version Print installed vLLM + torch/CUDA versions + models List staged model snapshots in the HF cache + curl-models curl /v1/models on each enabled instance (requires sudo to read API key) + logs Tail journalctl for a vllm systemd unit (e.g. vllm.service) +EOF +} + +case "${1:-}" in + activate) + echo "source $VENV/bin/activate" + ;; + version) + "$VENV/bin/python" -c 'import vllm, torch; print("vllm", vllm.__version__); print("torch", torch.__version__, "cuda", torch.version.cuda, "available", torch.cuda.is_available())' + ;; + models) + find "$CACHE" -maxdepth 1 -type d -name 'models--*' -printf '%f\n' 2>/dev/null || echo "(no models staged yet)" + ;; + curl-models) +{% for item in vllm_enabled_models | default([]) %} + echo "--- {{ item.id }} (:{{ item.port }}) ---" + curl -s -H "Authorization: Bearer $(sudo grep -oP '(?<=VLLM_API_KEY=).*' "$API_KEY_FILE")" \ + http://127.0.0.1:{{ item.port }}/v1/models | python3 -m json.tool || true +{% endfor %} + ;; + logs) + sudo journalctl -u "${2:-vllm.service}" -f + ;; + *) + usage + exit 1 + ;; +esac diff --git a/ansible/roles/deploy-vllm/templates/vllm.service.j2 b/ansible/roles/deploy-vllm/templates/vllm.service.j2 new file mode 100644 index 0000000..70daf6d --- /dev/null +++ b/ansible/roles/deploy-vllm/templates/vllm.service.j2 @@ -0,0 +1,56 @@ +[Unit] +Description=vLLM OpenAI-compatible inference server — {{ item.id }} ({{ item.hf_repo }}) +After=network-online.target nvidia-persistenced.service +Wants=network-online.target nvidia-persistenced.service + +[Service] +Type=simple +User={{ vllm_venv_owner }} +Group={{ vllm_venv_owner }} +EnvironmentFile={{ vllm_api_key_env_file }} +Environment="HOME=/home/{{ vllm_venv_owner }}" +Environment="HF_HUB_CACHE={{ vllm_hf_hub_cache }}" +Environment="HF_HOME={{ vllm_cache_dir }}" +# vLLM's torch.compile path shells out to `ninja` by bare name (not via +# venv-relative path) — without the venv's bin/ on PATH, systemd's minimal +# default PATH causes FileNotFoundError: 'ninja' deep in compile, even +# though `pip install vllm` installs the ninja package (and its console +# script) INTO the venv. Caught during Phase 5 validation (t_ca1af9fb, +# 2026-08-31): interactive SSH sessions have a different PATH than systemd +# services, so this only reproduces under systemd, not manual testing. +Environment="PATH={{ vllm_venv_path }}/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +# FlashInfer's bundled sampling.cu JIT-compiles against a cub template API +# (BlockAdjacentDifference::FlagHeads) that this flashinfer/CUDA toolkit +# combination does not provide on RTX 3090 (SM86) — 100 compile errors, +# confirmed upstream-known (vLLM GH #23023, #44305: FlashInfer sampler JIT +# breaks on various SM targets across flashinfer/vLLM version combos). +# Falls back to vLLM's native PyTorch sampler, which is fully supported and +# only marginally slower for single-request/low-concurrency serving. Caught +# during Phase 5 validation (t_ca1af9fb, 2026-08-31). +Environment="VLLM_USE_FLASHINFER_SAMPLER=0" + +ExecStart={{ vllm_venv_path }}/bin/python -m vllm.entrypoints.openai.api_server \ + --model {{ item.hf_repo }} \ + --served-model-name {{ item.id }} \ + --host {{ vllm_serve_host }} \ + --port {{ item.port }} \ +{% if item.quantization is defined and item.quantization != 'none' %} + --quantization {{ item.quantization }} \ +{% endif %} + --gpu-memory-utilization {{ item.gpu_memory_utilization }} \ + --max-model-len {{ item.max_model_len }} \ + --dtype {{ vllm_dtype }} \ + --api-key ${VLLM_API_KEY} \ + --enable-prefix-caching + +Restart={{ vllm_restart_policy }} +RestartSec=10 +# vLLM torch.compile can take 4+ minutes before /health responds even after +# weights are loaded (homelab-llm-inference skill pitfall) — give it room. +TimeoutStartSec=600 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=vllm-{{ item.id }} + +[Install] +WantedBy=multi-user.target