llm-inference-multimodel: role + day1 playbook (phase 0 discover approved)

ansible/roles/llm-inference-multimodel/tasks/verify.yml

@@ -0,0 +1,133 @@
+---
+# ------------------------------------------------------------------------------
+# FILE: roles/llm-inference-multimodel/tasks/verify.yml
+# DESCRIPTION: Phase 4 — start both services, curl smoke test each endpoint,
+#              nvidia-smi VRAM check against plan §1 math, confirm no OOM.
+#
+#              This is the ONLY phase that actually starts the services
+#              (systemd.yml deliberately does not). Enabling happens here too,
+#              so a reboot brings both back — matching plan §2's "independent
+#              systemd services" intent for durability, not just this-session.
+# ------------------------------------------------------------------------------
+
+- name: Enable and start llama-server-aux
+  ansible.builtin.systemd:
+    name: "{{ llm_aux_service_name }}"
+    state: started
+    enabled: true
+    daemon_reload: true
+  become: true
+
+- name: Enable and start llama-server-toolcall
+  ansible.builtin.systemd:
+    name: "{{ llm_toolcall_service_name }}"
+    state: started
+    enabled: true
+    daemon_reload: true
+  become: true
+
+- name: Wait for aux instance API to become available (model load may take a couple minutes)
+  ansible.builtin.uri:
+    url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/health"
+    status_code: 200
+  register: llm_aux_health
+  retries: 24
+  delay: 10
+  until: llm_aux_health.status == 200
+
+- name: Wait for tool-calling instance API to become available
+  ansible.builtin.uri:
+    url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/health"
+    status_code: 200
+  register: llm_toolcall_health
+  retries: 24
+  delay: 10
+  until: llm_toolcall_health.status == 200
+
+- name: Smoke-test — aux instance model listing
+  ansible.builtin.uri:
+    url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/models"
+    status_code: 200
+    return_content: true
+  register: llm_aux_models
+
+- name: Smoke-test — tool-calling instance model listing
+  ansible.builtin.uri:
+    url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/models"
+    status_code: 200
+    return_content: true
+  register: llm_toolcall_models
+
+- name: Report served models per instance
+  ansible.builtin.debug:
+    msg:
+      - "Aux (:{{ llm_aux_port }}) serving:        {{ llm_aux_models.json.data | map(attribute='id') | list }}"
+      - "Tool-calling (:{{ llm_toolcall_port }}) serving: {{ llm_toolcall_models.json.data | map(attribute='id') | list }}"
+
+- name: Basic completion smoke test — aux instance (non-tool-calling sanity check only)
+  ansible.builtin.uri:
+    url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/chat/completions"
+    method: POST
+    body_format: json
+    body:
+      model: "{{ llm_aux_model_id }}"
+      messages:
+        - role: user
+          content: "Reply with exactly one word: OK"
+      max_tokens: 10
+    status_code: 200
+    return_content: true
+  register: llm_aux_completion
+
+- name: Basic completion smoke test — tool-calling instance (plain-text sanity check only)
+  ansible.builtin.uri:
+    url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/chat/completions"
+    method: POST
+    body_format: json
+    body:
+      model: "{{ llm_toolcall_model_id }}"
+      messages:
+        - role: user
+          content: "Reply with exactly one word: OK"
+      max_tokens: 10
+    status_code: 200
+    return_content: true
+  register: llm_toolcall_completion
+
+- name: NOTE — this smoke test is NOT the tool-calling validation harness
+  ansible.builtin.debug:
+    msg: >-
+      Both endpoints respond to basic completions. This does NOT validate
+      tool_calls correctness or hallucination-safety for the tool-calling
+      instance — that is a separate, manual, post-deploy procedure (plan §7).
+      See references/tool-calling-validation.sh (copied from the
+      llm-inference-homelab skill) and docs/validation-log.md in this role.
+      DO NOT point any Claude Code / tool-calling-capable Hermes profile at
+      port {{ llm_toolcall_port }} until that validation has passed and been
+      logged.
+
+- name: Check GPU VRAM usage after both instances are running
+  ansible.builtin.command:
+    cmd: nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader
+  register: llm_post_start_vram
+  changed_when: false
+
+- name: Report VRAM usage vs plan §1 expectations
+  ansible.builtin.debug:
+    msg:
+      - "Measured (nvidia-smi):      {{ llm_post_start_vram.stdout }}"
+      - "Design estimate (plan §1):  aux ~{{ llm_aux_expected_vram_gb }}GB + toolcall ~{{ llm_toolcall_expected_vram_gb }}GB = ~{{ llm_combined_expected_vram_gb }}GB / {{ llm_gpu_total_vram_gb }}GB total"
+      - "If measured usage exceeds ~23.5GB or is within ~0.5GB of the 24GB card limit, treat as the OOM-risk trigger condition from plan §6 — do not leave both services running unattended without confirming headroom."
+
+- name: Check for OOM-kill events related to llama-server in dmesg (best-effort, read-only)
+  ansible.builtin.shell:
+    cmd: "dmesg | grep -i 'llama-server' | grep -i -E 'oom|killed' || true"
+  register: llm_oom_check
+  changed_when: false
+  become: true
+
+- name: Report any OOM-kill findings
+  ansible.builtin.debug:
+    msg: >-
+      {{ llm_oom_check.stdout if llm_oom_check.stdout | length > 0
+         else 'No OOM-kill events found for llama-server in dmesg.' }}