homelab/ansible/roles/deploy-vllm/tasks/verify.yml

# ------------------------------------------------------------------------------
# FILE: roles/deploy-vllm/tasks/verify.yml
# PHASE 5: Verification.
#
# Only runs when vllm_service_state == 'started' (main.yml gate) — staging a
# stopped service is a valid, intentional end state during the deploy-first-
# validate-before-cutover sequencing, and there is nothing to verify yet.
#
# Pitfall (homelab-llm-inference skill): vLLM torch.compile takes 4+ minutes
# AFTER weights load before /health returns 200. retries=30, delay=10 (5 min
# ceiling) — do not shrink this or health checks will false-negative on a
# perfectly healthy but still-warming-up service.
# ------------------------------------------------------------------------------

- name: Wait for each enabled model's systemd unit to be active
  ansible.builtin.systemd:
    name: "{{ 'vllm.service' if item.role == 'primary' else 'vllm-' + item.id + '.service' }}"
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_unit_status
  become: true

- name: Report systemd unit status
  ansible.builtin.debug:
    msg: "{{ item.item.id }}: {{ item.status.ActiveState }} ({{ item.status.SubState }})"
  loop: "{{ vllm_unit_status.results }}"
  loop_control:
    label: "{{ item.item.id }}"

- name: Fail if any unit is not active
  ansible.builtin.fail:
    msg: "{{ item.item.id }} systemd unit is {{ item.status.ActiveState }}, expected active."
  loop: "{{ vllm_unit_status.results }}"
  loop_control:
    label: "{{ item.item.id }}"
  when: item.status.ActiveState != 'active'

- name: Poll /health until 200 (torch.compile warmup can take 4-5 minutes)
  ansible.builtin.uri:
    url: "http://127.0.0.1:{{ item.port }}/health"
    status_code: 200
    timeout: 15
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_health_check
  until: vllm_health_check is succeeded
  retries: "{{ vllm_health_check_retries }}"
  delay: "{{ vllm_health_check_delay }}"

- name: Query /v1/models on each enabled instance
  ansible.builtin.uri:
    url: "http://127.0.0.1:{{ item.port }}/v1/models"
    headers:
      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
    return_content: true
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_models_response
  no_log: true

- name: Assert /v1/models returns the expected served model name
  ansible.builtin.assert:
    that:
      - item.item.id in (item.content)
    fail_msg: "/v1/models on port {{ item.item.port }} did not list expected model id {{ item.item.id }}"
    success_msg: "/v1/models confirmed {{ item.item.id }} is served on port {{ item.item.port }}"
  loop: "{{ vllm_models_response.results }}"
  loop_control:
    label: "{{ item.item.id }}"

- name: Run a live completion smoke test against each enabled instance
  ansible.builtin.uri:
    url: "http://127.0.0.1:{{ item.port }}/v1/completions"
    method: POST
    headers:
      Authorization: "Bearer {{ vllm_api_key_lookup.stdout }}"
      Content-Type: "application/json"
    body_format: json
    body:
      model: "{{ item.id }}"
      prompt: "The capital of France is"
      max_tokens: 8
      temperature: 0
    timeout: 60
    status_code: 200
  loop: "{{ vllm_enabled_models }}"
  loop_control:
    label: "{{ item.id }}"
  register: vllm_completion_test
  no_log: true

- name: Report completion smoke test result
  ansible.builtin.debug:
    msg: "{{ item.item.id }}: HTTP {{ item.status }} — completion smoke test passed"
  loop: "{{ vllm_completion_test.results }}"
  loop_control:
    label: "{{ item.item.id }}"

- name: Check journalctl for the primary unit is free of ERROR/Traceback since last start
  ansible.builtin.shell: |
    set -o pipefail
    journalctl -u vllm.service --since "10 min ago" | grep -iE "error|traceback" | grep -v "no entries" || true
  args:
    executable: /bin/bash
  register: vllm_journal_errors
  changed_when: false
  become: true

- name: Report journalctl scan result
  ansible.builtin.debug:
    msg: >-
      {{ 'journalctl clean — no error/traceback lines in the last 10 minutes'
      if vllm_journal_errors.stdout | trim | length == 0
      else 'WARNING — journalctl lines matched error/traceback: ' + vllm_journal_errors.stdout }}