homelab/ansible/roles/deploy-vllm/tasks/verify.yml

# ------------------------------------------------------------------------------
# FILE: roles/deploy-vllm/tasks/verify.yml
# PHASE 5: Verification.
#
# Only runs when vllm_service_state == 'started' (main.yml gate) — staging a
# stopped service is a valid, intentional end state during the deploy-first-
# validate-before-cutover sequencing, and there is nothing to verify yet.
#
# Pitfall (homelab-llm-inference skill): vLLM torch.compile takes 4+ minutes
# AFTER weights load before /health returns 200. retries=30, delay=10 (5 min
# ceiling) — do not shrink this or health checks will false-negative on a
# perfectly healthy but still-warming-up service.
# ------------------------------------------------------------------------------

- name: Wait for each enabled model's systemd unit to be active
  ansible.builtin.systemd:
    name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}"
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_unit_status
  become: true

- name: Report systemd unit status
  ansible.builtin.debug:
    msg: "{{ item.item.id }}: {{ item.status.ActiveState }} ({{ item.status.SubState }})"
  loop: "{{ vllm_unit_status.results }}"
  loop_control:
    label: "{{ item.item.id }}"

- name: Fail if any unit is not active
  ansible.builtin.fail:
    msg: "{{ item.item.id }} systemd unit is {{ item.status.ActiveState }}, expected active."
  loop: "{{ vllm_unit_status.results }}"
  loop_control:
    label: "{{ item.item.id }}"
  when: item.status.ActiveState != 'active'

- name: Poll /health until 200 (torch.compile warmup can take 4-5 minutes)
  ansible.builtin.uri:
    url: "http://127.0.0.1:{{ item.port }}/health"
    status_code: 200
    timeout: 15
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_health_check
  until: vllm_health_check is succeeded
  retries: "{{ vllm_health_check_retries }}"
  delay: "{{ vllm_health_check_delay }}"

- name: Query /v1/models on each enabled instance
  ansible.builtin.uri:
    url: "http://127.0.0.1:{{ item.port }}/v1/models"
    headers:
      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
    return_content: true
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_models_response
  no_log: true

- name: Assert /v1/models returns the expected served model name
  ansible.builtin.assert:
    that:
      - item.item.id in (item.content)
    fail_msg: "/v1/models on port {{ item.item.port }} did not list expected model id {{ item.item.id }}"
    success_msg: "/v1/models confirmed {{ item.item.id }} is served on port {{ item.item.port }}"
  loop: "{{ vllm_models_response.results }}"
  loop_control:
    label: "{{ item.item.id }}"

- name: Split enabled models into completion-serving vs embedding for the right smoke test
  ansible.builtin.set_fact:
    vllm_completion_models: "{{ vllm_enabled_models | rejectattr('role', 'equalto', 'embedding') | list }}"
    vllm_embedding_models: "{{ vllm_enabled_models | selectattr('role', 'equalto', 'embedding') | list }}"

- name: Run a live completion smoke test against each completion-serving instance
  ansible.builtin.uri:
    url: "http://127.0.0.1:{{ item.port }}/v1/completions"
    method: POST
    headers:
      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
      Content-Type: "application/json"
    body_format: json
    body:
      model: "{{ item.id }}"
      prompt: "The capital of France is"
      max_tokens: 8
      temperature: 0
    timeout: 60
    status_code: 200
  loop: "{{ vllm_completion_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_completion_test
  no_log: true

- name: Report completion smoke test result
  ansible.builtin.debug:
    msg: "{{ item.item.id }}: HTTP {{ item.status }} — completion smoke test passed"
  loop: "{{ vllm_completion_test.results }}"
  loop_control:
    label: "{{ item.item.id }}"

# Embedding-mode vLLM instances (--runner pooling --convert embed) do NOT
# serve /v1/completions — only /v1/embeddings (and /pooling). A completions
# smoke test against one 400s immediately. Verify with a real vector request
# instead, and assert the response actually contains a non-empty float vector
# (not just HTTP 200 — an empty/malformed embedding would still 200).
- name: Run a live embeddings smoke test against each embedding-mode instance
  ansible.builtin.uri:
    url: "http://127.0.0.1:{{ item.port }}/v1/embeddings"
    method: POST
    headers:
      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
      Content-Type: "application/json"
    body_format: json
    body:
      model: "{{ item.id }}"
      input: "The capital of France is Paris."
    timeout: 60
    status_code: 200
    return_content: true
  loop: "{{ vllm_embedding_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_embedding_test
  no_log: true

- name: Assert embeddings smoke test returned a non-empty float vector
  ansible.builtin.assert:
    that:
      - (item.json.data[0].embedding | length) > 0
    fail_msg: "/v1/embeddings on port {{ item.item.port }} did not return a non-empty embedding vector"
    success_msg: "/v1/embeddings confirmed {{ item.item.id }} returns a {{ item.json.data[0].embedding | length }}-dim vector"
  loop: "{{ vllm_embedding_test.results }}"
  loop_control:
    label: "{{ item.item.id }}"

- name: Check journalctl for each enabled unit is free of ERROR/Traceback since last start
  ansible.builtin.shell: |
    set -o pipefail
    journalctl -u {{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }} --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true
  args:
    executable: /bin/bash
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_journal_errors
  changed_when: false
  become: true

- name: Report journalctl scan result
  ansible.builtin.debug:
    msg: >-
      {{ item.item.id ~ ': journalctl clean — no error/traceback lines in the last 10 minutes'
      if item.stdout | trim | length == 0
      else item.item.id ~ ' WARNING — journalctl lines matched error/traceback: ' ~ item.stdout }}
  loop: "{{ vllm_journal_errors.results }}"
  loop_control:
    label: "{{ item.item.id }}"