homelab/ansible/roles/llm-inference-multimodel/tasks/verify.yml

---
# ------------------------------------------------------------------------------
# FILE: roles/llm-inference-multimodel/tasks/verify.yml
# DESCRIPTION: Phase 4 — start both services, curl smoke test each endpoint,
#              nvidia-smi VRAM check against plan §1 math, confirm no OOM.
#
#              This is the ONLY phase that actually starts the services
#              (systemd.yml deliberately does not). Enabling happens here too,
#              so a reboot brings both back — matching plan §2's "independent
#              systemd services" intent for durability, not just this-session.
# ------------------------------------------------------------------------------

# BUGFIX (found in production): this file used to go straight from Phase 2's
# "units deployed, nothing running" state into starting/enabling both new
# services, without first stopping the pre-existing single-model Gemma
# llama-server that Phase 0 (discover.yml) confirmed is running on the same
# GPU. Plan §1's VRAM math (aux ~Xgb + toolcall ~Ygb ≈ ~23.5GB / 24GB) only
# holds if that card isn't ALSO holding the old Gemma model's VRAM at the
# same time. Running --tags verify (or the full playbook) against a host
# where Gemma was still up meant starting both new instances on top of it —
# a near-guaranteed OOM-kill, not just a "risk" the old §6 debug message
# warned about after the fact.
#
# Fix: if discover.yml found a systemd-managed Gemma unit
# (llm_existing_gemma_unit_found), stop it here, before either new service
# starts. This only stops the unit — it does not disable it, so plan §6's
# rollback ("systemctl start llama-server-gemma to fully revert") still
# works unchanged. If no such unit was found, this is a no-op and the
# original WARNING from discover.yml (any Gemma process running outside
# systemd) still stands as a human decision point.
#
# BUGFIX (found in production): main.yml imports each phase file with
# import_tasks + a per-phase tag (tags: [discover], tags: [verify], ...).
# Tags on import_tasks apply to every task inside that file, so running
# `--tags verify` alone — a normal, supported way to re-run just this
# phase per the header comment in main.yml — skips discover.yml entirely.
# llm_existing_gemma_unit_found was then simply undefined, and the
# `| default(false)` on this task's `when:` silently swallowed that,
# defeating the whole point of this fix: --tags verify against a host
# with Gemma still running would go straight to starting both new
# services on top of it, the exact OOM scenario this task exists to
# prevent. Gather the fact locally here too so this task is correct
# regardless of which tags were requested.
- name: Gather service facts (systemd unit inventory) — ensure available even if discover.yml's tag wasn't selected
  ansible.builtin.service_facts:
  when: llm_existing_gemma_unit_found is not defined

- name: Determine whether a systemd unit matching the existing Gemma service exists (if not already known from discover.yml)
  ansible.builtin.set_fact:
    llm_existing_gemma_unit_found: "{{ (llm_existing_gemma_service_name_guess + '.service') in ansible_facts.services }}"
  when: llm_existing_gemma_unit_found is not defined

- name: Stop pre-existing Gemma llama-server before starting new instances (avoid double VRAM usage / OOM)
  ansible.builtin.systemd:
    name: "{{ llm_existing_gemma_service_name_guess }}"
    state: stopped
  become: true
  when: llm_existing_gemma_unit_found | default(false)

- name: Enable llama-server-aux and start/restart based on Phase 2 unit-content change
  ansible.builtin.systemd:
    name: "{{ llm_aux_service_name }}"
    state: "{{ 'restarted' if (llm_aux_unit_deployed.changed | default(false)) else 'started' }}"
    enabled: true
    daemon_reload: true
  become: true

- name: Enable llama-server-toolcall and start/restart based on Phase 2 unit-content change
  ansible.builtin.systemd:
    name: "{{ llm_toolcall_service_name }}"
    state: "{{ 'restarted' if (llm_toolcall_unit_deployed.changed | default(false)) else 'started' }}"
    enabled: true
    daemon_reload: true
  become: true

- name: Wait for aux instance API to become available (model load may take a couple minutes)
  ansible.builtin.uri:
    url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/health"
    status_code: 200
  register: llm_aux_health
  retries: 24
  delay: 10
  until: llm_aux_health.status == 200

- name: Wait for tool-calling instance API to become available
  ansible.builtin.uri:
    url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/health"
    status_code: 200
  register: llm_toolcall_health
  retries: 24
  delay: 10
  until: llm_toolcall_health.status == 200

- name: Smoke-test — aux instance model listing
  ansible.builtin.uri:
    url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/models"
    status_code: 200
    return_content: true
  register: llm_aux_models

- name: Smoke-test — tool-calling instance model listing
  ansible.builtin.uri:
    url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/models"
    status_code: 200
    return_content: true
  register: llm_toolcall_models

- name: Report served models per instance
  ansible.builtin.debug:
    msg:
      - "Aux (:{{ llm_aux_port }}) serving:        {{ llm_aux_models.json.data | map(attribute='id') | list }}"
      - "Tool-calling (:{{ llm_toolcall_port }}) serving: {{ llm_toolcall_models.json.data | map(attribute='id') | list }}"

- name: Basic completion smoke test — aux instance (non-tool-calling sanity check only)
  ansible.builtin.uri:
    url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/chat/completions"
    method: POST
    body_format: json
    body:
      model: "{{ llm_aux_model_id }}"
      messages:
        - role: user
          content: "Reply with exactly one word: OK"
      max_tokens: 10
    status_code: 200
    return_content: true
  register: llm_aux_completion

- name: Basic completion smoke test — tool-calling instance (plain-text sanity check only)
  ansible.builtin.uri:
    url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/chat/completions"
    method: POST
    body_format: json
    body:
      model: "{{ llm_toolcall_model_id }}"
      messages:
        - role: user
          content: "Reply with exactly one word: OK"
      max_tokens: 10
    status_code: 200
    return_content: true
  register: llm_toolcall_completion

- name: NOTE — this smoke test is NOT the tool-calling validation harness
  ansible.builtin.debug:
    msg: >-
      Both endpoints respond to basic completions. This does NOT validate
      tool_calls correctness or hallucination-safety for the tool-calling
      instance — that is a separate, manual, post-deploy procedure (plan §7).
      See references/tool-calling-validation.sh (copied from the
      llm-inference-homelab skill) and docs/validation-log.md in this role.
      DO NOT point any Claude Code / tool-calling-capable Hermes profile at
      port {{ llm_toolcall_port }} until that validation has passed and been
      logged.

- name: Check GPU VRAM usage after both instances are running
  ansible.builtin.command:
    cmd: nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader
  register: llm_post_start_vram
  changed_when: false

- name: Report VRAM usage vs plan §1 expectations
  ansible.builtin.debug:
    msg:
      - "Measured (nvidia-smi):      {{ llm_post_start_vram.stdout }}"
      - "Design estimate (plan §1):  aux ~{{ llm_aux_expected_vram_gb }}GB + toolcall ~{{ llm_toolcall_expected_vram_gb }}GB = ~{{ llm_combined_expected_vram_gb }}GB / {{ llm_gpu_total_vram_gb }}GB total"
      - "If measured usage exceeds ~23.5GB or is within ~0.5GB of the 24GB card limit, treat as the OOM-risk trigger condition from plan §6 — do not leave both services running unattended without confirming headroom."

- name: Check for OOM-kill events related to llama-server in dmesg (best-effort, read-only)
  ansible.builtin.shell:
    cmd: "dmesg | grep -i 'llama-server' | grep -i -E 'oom|killed' || true"
  register: llm_oom_check
  changed_when: false
  become: true

- name: Report any OOM-kill findings
  ansible.builtin.debug:
    msg: >-
      {{ llm_oom_check.stdout if llm_oom_check.stdout | length > 0
         else 'No OOM-kill events found for llama-server in dmesg.' }}