deploy-vllm: add embedding-mode support, cut over Hindsight to vLLM (t_e6facb19)
- vllm.service.j2: branch on role==embedding for --runner pooling --convert embed, --no-enable-prefix-caching, per-model trust_remote_code toggle (needed for nomic-embed-text-v1.5's custom NomicBertModel code), and enforce_eager toggle (needed to avoid CUDA graph capture OOM when co-resident with another vLLM process on this 24GB card). - tasks/verify.yml: split completions vs embedding smoke tests -- embedding-mode instances don't serve /v1/completions. Assert a non-empty embedding vector, not just HTTP 200. - host_vars/astro-orbiter: enable nomic-embed-text-v1.5 (port 8020), lower primary model's gpu_memory_utilization 0.95->0.90 + add enforce_eager after finding 0.95 crash-looped 6-7x before stabilizing with co-resident nomic-embed (real fix, confirmed via NRestarts=0 after clean stop/start, not luck). - Hindsight (values.yaml + externalsecret.yaml): cut LLM + embeddings over to vLLM (:8000, :8020), wire the previously-unset HINDSIGHT_API_EMBEDDINGS_* env vars for the first time, and swap the API key secret source from the Nous fallback item to vllm/api-key (vLLM enforces real auth, llama-swap did not). - README: document the embedding-mode branch, VRAM findings, and a genuine architecture gap -- vLLM's one-model-per-process design cannot replace llama-swap's 5-model LRU roster on this 24GB card, so 21 Hermes profiles' aux-model consumers (Qwen3-8B-no_think, Phi-3.5-mini, Meta-Llama-3.1-8B, Qwen2.5-Coder-14B) and OpenViking's VLM stay on llama-swap. Full teardown (t_6dff1ecc) needs a human decision on the aux-model strategy before it can proceed.
This commit is contained in:
@@ -71,7 +71,12 @@
|
||||
loop_control:
|
||||
label: "{{ item.item.id }}"
|
||||
|
||||
- name: Run a live completion smoke test against each enabled instance
|
||||
- name: Split enabled models into completion-serving vs embedding for the right smoke test
|
||||
ansible.builtin.set_fact:
|
||||
vllm_completion_models: "{{ vllm_enabled_models | rejectattr('role', 'equalto', 'embedding') | list }}"
|
||||
vllm_embedding_models: "{{ vllm_enabled_models | selectattr('role', 'equalto', 'embedding') | list }}"
|
||||
|
||||
- name: Run a live completion smoke test against each completion-serving instance
|
||||
ansible.builtin.uri:
|
||||
url: "http://127.0.0.1:{{ item.port }}/v1/completions"
|
||||
method: POST
|
||||
@@ -86,7 +91,7 @@
|
||||
temperature: 0
|
||||
timeout: 60
|
||||
status_code: 200
|
||||
loop: "{{ vllm_enabled_models }}"
|
||||
loop: "{{ vllm_completion_models }}"
|
||||
loop_control:
|
||||
label: "{{ item.id }}"
|
||||
register: vllm_completion_test
|
||||
@@ -99,12 +104,50 @@
|
||||
loop_control:
|
||||
label: "{{ item.item.id }}"
|
||||
|
||||
- name: Check journalctl for the primary unit is free of ERROR/Traceback since last start
|
||||
# Embedding-mode vLLM instances (--runner pooling --convert embed) do NOT
|
||||
# serve /v1/completions — only /v1/embeddings (and /pooling). A completions
|
||||
# smoke test against one 400s immediately. Verify with a real vector request
|
||||
# instead, and assert the response actually contains a non-empty float vector
|
||||
# (not just HTTP 200 — an empty/malformed embedding would still 200).
|
||||
- name: Run a live embeddings smoke test against each embedding-mode instance
|
||||
ansible.builtin.uri:
|
||||
url: "http://127.0.0.1:{{ item.port }}/v1/embeddings"
|
||||
method: POST
|
||||
headers:
|
||||
Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
|
||||
Content-Type: "application/json"
|
||||
body_format: json
|
||||
body:
|
||||
model: "{{ item.id }}"
|
||||
input: "The capital of France is Paris."
|
||||
timeout: 60
|
||||
status_code: 200
|
||||
return_content: true
|
||||
loop: "{{ vllm_embedding_models }}"
|
||||
loop_control:
|
||||
label: "{{ item.id }}"
|
||||
register: vllm_embedding_test
|
||||
no_log: true
|
||||
|
||||
- name: Assert embeddings smoke test returned a non-empty float vector
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- (item.json.data[0].embedding | length) > 0
|
||||
fail_msg: "/v1/embeddings on port {{ item.item.port }} did not return a non-empty embedding vector"
|
||||
success_msg: "/v1/embeddings confirmed {{ item.item.id }} returns a {{ item.json.data[0].embedding | length }}-dim vector"
|
||||
loop: "{{ vllm_embedding_test.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.id }}"
|
||||
|
||||
- name: Check journalctl for each enabled unit is free of ERROR/Traceback since last start
|
||||
ansible.builtin.shell: |
|
||||
set -o pipefail
|
||||
journalctl -u vllm.service --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true
|
||||
journalctl -u {{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }} --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true
|
||||
args:
|
||||
executable: /bin/bash
|
||||
loop: "{{ vllm_enabled_models }}"
|
||||
loop_control:
|
||||
label: "{{ item.id }}"
|
||||
register: vllm_journal_errors
|
||||
changed_when: false
|
||||
become: true
|
||||
@@ -112,6 +155,9 @@
|
||||
- name: Report journalctl scan result
|
||||
ansible.builtin.debug:
|
||||
msg: >-
|
||||
{{ 'journalctl clean — no error/traceback lines in the last 10 minutes'
|
||||
if vllm_journal_errors.stdout | trim | length == 0
|
||||
else 'WARNING — journalctl lines matched error/traceback: ' + vllm_journal_errors.stdout }}
|
||||
{{ item.item.id ~ ': journalctl clean — no error/traceback lines in the last 10 minutes'
|
||||
if item.stdout | trim | length == 0
|
||||
else item.item.id ~ ' WARNING — journalctl lines matched error/traceback: ' ~ item.stdout }}
|
||||
loop: "{{ vllm_journal_errors.results }}"
|
||||
loop_control:
|
||||
label: "{{ item.item.id }}"
|
||||
|
||||
Reference in New Issue
Block a user