homelab/ansible/roles/llm-inference-multimodel/tasks/router.yml

---
# ------------------------------------------------------------------------------
# FILE: roles/llm-inference-multimodel/tasks/router.yml
# DESCRIPTION: Phase R — llama.cpp router mode shadow deployment (port 8003).
#
#              This phase is ADDITIVE and IDEMPOTENT. The existing production
#              unit (llama-server-qwen, port 8002) is never touched here.
#
#              All tasks are gated on  llm_router_enabled | default(false).
#              With the default (false) this entire file is a no-op, making
#              it safe to import unconditionally from main.yml.
#
#              When llm_router_enabled: true (set by the shadow playbook or
#              extra-vars), this phase:
#                router_systemd  — deploy the router unit file (no auto-start)
#                router_firewall — open port 8003 scoped to the Hermes subnet
#                router_verify   — start router, run all 4 validation gates
#                router_ui_check — (nice-to-have) check bundled SvelteKit UI
#
#              Tags map 1:1 to the sub-phases so you can run each independently:
#                --tags router_systemd,router_firewall,router_verify,router_ui_check
#
#              IMPORTANT: router_verify starts the router service. Do not run
#              router_verify unless router_systemd and router_firewall have
#              already run (or use the full shadow playbook which runs all four).
#
#              Added 2026-08-12 (t_0cca74a2): Qwen router migration — War Machine.
#              Approved by Ryan 2026-08-12 (see task comment thread).
# ------------------------------------------------------------------------------


# =============================================================================
# TAG: router_systemd
# Deploy the router unit file and reload systemd.
# Does NOT start the service — that is Phase router_verify only.
# =============================================================================

- name: "[router_systemd] Deploy llama-server-router systemd unit (shadow, port {{ llm_router_port }})"
  ansible.builtin.template:
    src: llama-server-router.service.j2
    dest: "/etc/systemd/system/{{ llm_router_service_name }}.service"
    owner: root
    group: root
    mode: "0644"
  become: true
  register: llm_router_unit_deployed
  notify:
    - reload systemd
  when: llm_router_enabled | default(false)
  tags: [router_systemd]

- name: "[router_systemd] Flush handlers so daemon-reload lands before router_verify starts the unit"
  ansible.builtin.meta: flush_handlers
  when: llm_router_enabled | default(false)
  tags: [router_systemd]

# =============================================================================
# TAG: router_firewall
# Open port 8003 in ufw scoped to the Hermes source subnet.
# Idempotent: named comment + state: present prevents duplicate rules.
# =============================================================================

- name: "[router_firewall] Check whether ufw is installed/active"
  ansible.builtin.command:
    cmd: ufw status
  register: llm_router_ufw_status
  changed_when: false
  failed_when: false
  become: true
  when: llm_router_enabled | default(false)
  tags: [router_firewall]

- name: "[router_firewall] WARNING — ufw not active, port {{ llm_router_port }} scoping cannot be applied"
  ansible.builtin.debug:
    msg: >-
      ufw does not appear to be active on this host. Firewall scoping for
      port {{ llm_router_port }} was skipped. Bind address alone
      ({{ llm_router_bind_address }}) limits exposure — flag to Ryan.
  when:
    - llm_router_enabled | default(false)
    - "'Status: active' not in (llm_router_ufw_status.stdout | default(''))"
  tags: [router_firewall]

- name: "[router_firewall] Allow router port ({{ llm_router_port }}) from Hermes source subnet"
  community.general.ufw:
    rule: allow
    port: "{{ llm_router_port | string }}"
    proto: tcp
    src: "{{ llm_router_allowed_source_cidr }}"
    comment: "llm-inference-multimodel: router shadow ({{ llm_router_port }}) — scoped to Hermes subnet"
  become: true
  when:
    - llm_router_enabled | default(false)
    - "'Status: active' in (llm_router_ufw_status.stdout | default(''))"
  tags: [router_firewall]

# =============================================================================
# TAG: router_verify
# Start the router, then run the 4 validation gates defined in the proposal.
# This is the ONLY phase that starts llama-server-router.
# Gates 1-3 are hard failures (play fails on any gate miss).
# Gate 4 (UI) is informational only.
# =============================================================================

- name: "[router_verify] Pre-check: confirm port {{ llm_router_port }} is not already bound"
  ansible.builtin.command:
    cmd: "ss -ltnp"
  register: llm_router_port_check
  changed_when: false
  failed_when: false
  become: true
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] Check if the process on port {{ llm_router_port }} is already our router service"
  ansible.builtin.set_fact:
    llm_router_port_bound: "{{ ':' + (llm_router_port | string) + ' ' in (llm_router_port_check.stdout | default('')) or ':' + (llm_router_port | string) + ':' in (llm_router_port_check.stdout | default('')) }}"
    # ss output format: "LISTEN 0 512 10.1.71.130:8003 ... users:((\"llama-server\",pid=N,...))"
    # The router runs as "llama-server" process name; use the systemd service to verify it's ours
    llm_router_port_already_ours: >-
      {{ 'llama-server' in (llm_router_port_check.stdout | default('')) and
         ((':' + (llm_router_port | string) + ' ') in (llm_router_port_check.stdout | default('')) or
          (':' + (llm_router_port | string) + ':') in (llm_router_port_check.stdout | default(''))) }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_port_check is defined
  tags: [router_verify]

- name: "[router_verify] Fail if port {{ llm_router_port }} is in use by an UNKNOWN service (not our router)"
  ansible.builtin.fail:
    msg: >-
      Port {{ llm_router_port }} is already bound on astro-orbiter by an UNKNOWN
      process (not llama-server-router). Cannot safely start our router on this port.
      Check with 'ss -ltnp | grep :{{ llm_router_port }}' and resolve before retrying.
  when:
    - llm_router_enabled | default(false)
    - llm_router_port_bound | default(false)
    - not (llm_router_port_already_ours | default(false))
  tags: [router_verify]

- name: "[router_verify] Note: port {{ llm_router_port }} already bound by our router — skip start, proceed to gates"
  ansible.builtin.debug:
    msg: >-
      Port {{ llm_router_port }} is already bound by {{ llm_router_service_name }}.
      Skipping start task — router is already running. Proceeding to validation gates.
  when:
    - llm_router_enabled | default(false)
    - llm_router_port_bound | default(false)
    - llm_router_port_already_ours | default(false)
  tags: [router_verify]

- name: "[router_verify] Enable and start llama-server-router (shadow, port {{ llm_router_port }})"
  ansible.builtin.systemd:
    name: "{{ llm_router_service_name }}"
    state: "{{ 'restarted' if (llm_router_unit_deployed.changed | default(false)) else 'started' }}"
    enabled: true
    daemon_reload: true
  become: true
  when:
    - llm_router_enabled | default(false)
    - not (llm_router_port_already_ours | default(false))
  tags: [router_verify]

# --- Gate 1: /health (waits for cold model load ~30-60s for 35B) ---
# IMPORTANT: The router's model load requires free VRAM. If llama-server-qwen
# (production, port 8002) is running, it holds ~20GB of VRAM and the router
# cannot load Qwen concurrently on a 24GB card.
# This task temporarily stops the production service to free VRAM for the
# shadow validation. The production service is restarted after all gates pass.
# This is the expected shadow-test flow for a 24GB single-GPU host.

- name: "[router_verify] VRAM GATE PRE: Check if production service is holding VRAM"
  ansible.builtin.command:
    cmd: nvidia-smi --query-compute-apps=pid,name,used_memory --format=csv,noheader
  register: llm_router_vram_pre_stop
  changed_when: false
  become: true
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] VRAM GATE PRE: Stop production llama-server-qwen to free VRAM for router validation"
  ansible.builtin.systemd:
    name: "{{ llm_qwen_service_name }}"
    state: stopped
  become: true
  register: llm_router_qwen_stopped
  when:
    - llm_router_enabled | default(false)
    - "'llama-server' in (llm_router_vram_pre_stop.stdout | default(''))"
  tags: [router_verify]

- name: "[router_verify] VRAM GATE PRE: Wait 5s for VRAM to be released after production stop"
  ansible.builtin.pause:
    seconds: 5
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_stopped is defined
    - llm_router_qwen_stopped.changed | default(false)
  tags: [router_verify]

- name: "[router_verify] VRAM GATE PRE: Report VRAM state after stopping production service"
  ansible.builtin.command:
    cmd: nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader
  register: llm_router_vram_after_stop
  changed_when: false
  become: true
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] VRAM GATE PRE: Report VRAM available for router validation"
  ansible.builtin.debug:
    msg: >-
      VRAM after stopping production service:
      {{ llm_router_vram_after_stop.stdout | default('unknown') }}.
      {{ 'Production service was stopped to free VRAM for router validation.' if (llm_router_qwen_stopped.changed | default(false)) else 'Production service was not running (VRAM already free).' }}
      Router model load requires ~20GB. NOTE: production service will be restarted after validation.
  when:
    - llm_router_enabled | default(false)
    - llm_router_vram_after_stop is defined
  tags: [router_verify]

- name: "[router_verify] GATE 1a: Wait for router /health (up to 5min — cold model load)"
  ansible.builtin.uri:
    url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health"
    status_code: 200
  register: llm_router_health
  retries: 30
  delay: 10
  until: llm_router_health.status == 200
  when: llm_router_enabled | default(false)
  tags: [router_verify]

# Trigger the model load — router lazy-loads models on first request.
# This POST will block until the model is loaded (~30-60s for 35B).
# We use a short max_tokens=5 probe so the response is nearly instant
# after load completes. The timeout is set high (300s) to cover cold load.
- name: "[router_verify] GATE 1a: Trigger model load via first request (router lazy-loads on demand)"
  ansible.builtin.uri:
    url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions"
    method: POST
    body_format: json
    body:
      model: "{{ llm_router_expected_model_id }}"
      messages:
        - role: user
          content: "Reply with one word: hello"
      max_tokens: 5
      temperature: 0.0
    status_code: 200
    return_content: true
    timeout: 300
  register: llm_router_warmup
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] GATE 1a: Report warmup response (confirms model loaded successfully)"
  ansible.builtin.debug:
    msg:
      - "Model loaded via warmup request. finish_reason={{ llm_router_warmup.json.choices[0].finish_reason | default('unknown') }}"
      - "Response preview: {{ llm_router_warmup.json.choices[0].message.content | default('(empty)') | truncate(100) }}"
  when:
    - llm_router_enabled | default(false)
    - llm_router_warmup is defined
    - llm_router_warmup.json is defined
  tags: [router_verify]

# --- Gate 1: /v1/models — Qwen present, n_ctx_train >= 64K ---

- name: "[router_verify] GATE 1b: Query /v1/models on router endpoint"
  ansible.builtin.uri:
    url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models"
    status_code: 200
    return_content: true
  register: llm_router_models
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] GATE 1b: Report models returned by router"
  ansible.builtin.debug:
    msg:
      - "Router /v1/models response: {{ llm_router_models.json.data | map(attribute='id') | list }}"
      - "Model status: {{ llm_router_models.json.data | map(attribute='status') | map(attribute='value') | list }}"
      - "ctx-size in args: {{ llm_router_models.json.data[0].status.args | select('match', '^[0-9]+$') | list }}"
  when:
    - llm_router_enabled | default(false)
    - llm_router_models is defined
  tags: [router_verify]

- name: "[router_verify] GATE 1b: Fail if expected model ID not found in /v1/models"
  ansible.builtin.fail:
    msg: >-
      GATE 1 FAIL: Model '{{ llm_router_expected_model_id }}' not found in router
      /v1/models response. Returned IDs:
      {{ llm_router_models.json.data | map(attribute='id') | list }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_models is defined
    - llm_router_models.json.data | selectattr('id', 'equalto', llm_router_expected_model_id) | list | length == 0
  tags: [router_verify]

- name: "[router_verify] GATE 1b: Extract ctx-size from model args (router uses status.args, not meta.n_ctx)"
  ansible.builtin.set_fact:
    llm_router_qwen_n_ctx: >-
      {%- set model = llm_router_models.json.data | selectattr('id', 'equalto', llm_router_expected_model_id) | first -%}
      {%- set args = model.status.args -%}
      {%- set ctx_idx = args.index('--ctx-size') if '--ctx-size' in args else -1 -%}
      {{ args[ctx_idx + 1] | int if ctx_idx >= 0 else 0 }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_models is defined
    - llm_router_models.json.data | selectattr('id', 'equalto', llm_router_expected_model_id) | list | length > 0
  tags: [router_verify]

- name: "[router_verify] GATE 1b: Fail if n_ctx < 64000 (Hermes 64K context floor)"
  ansible.builtin.fail:
    msg: >-
      GATE 1 FAIL: Router args show --ctx-size={{ llm_router_qwen_n_ctx }} for
      {{ llm_router_expected_model_id }}. Hermes requires >= 64000 (64K floor).
      Check --ctx-size in the unit template and verify the model args.
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_n_ctx is defined
    - llm_router_qwen_n_ctx | int < 64000
  tags: [router_verify]

- name: "[router_verify] GATE 1b: PASS — n_ctx >= 64K confirmed (from router model args)"
  ansible.builtin.debug:
    msg: "GATE 1 PASS: Router passes --ctx-size={{ llm_router_qwen_n_ctx }} for {{ llm_router_expected_model_id }} (>= 64000 required)."
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_n_ctx is defined
    - llm_router_qwen_n_ctx | int >= 64000
  tags: [router_verify]

# --- Gate 2: Tool-calling through router path (hard gate) ---
# Tests the tool-calling trigger path THROUGH the router proxy, not via the
# bare llama-server. Router mode is a different process/proxy path — parity
# with port 8002 is not assumed.

- name: "[router_verify] GATE 2: Tool-calling trigger probe (weather probe — should return tool_calls)"
  ansible.builtin.uri:
    url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions"
    method: POST
    body_format: json
    body:
      model: "{{ llm_router_expected_model_id }}"
      messages:
        - role: user
          content: "What is the current weather in Chicago? Use the provided tool."
      tools:
        - type: function
          function:
            name: get_weather
            description: "Get current weather conditions for a city"
            parameters:
              type: object
              properties:
                city:
                  type: string
                  description: "The city name"
              required:
                - city
      temperature: 0.0
    status_code: 200
    return_content: true
    timeout: 120
  register: llm_router_toolcall_probe
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] GATE 2: Fail if tool-calling probe did not return finish_reason=tool_calls"
  ansible.builtin.fail:
    msg: >-
      GATE 2 FAIL: Tool-calling probe returned finish_reason=
      {{ llm_router_toolcall_probe.json.choices[0].finish_reason | default('(missing)') }}
      instead of 'tool_calls'. Router is not correctly proxying tool-call requests.
      Full response: {{ llm_router_toolcall_probe.json | to_json }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_toolcall_probe is defined
    - llm_router_toolcall_probe.json.choices[0].finish_reason | default('') != 'tool_calls'
  tags: [router_verify]

- name: "[router_verify] GATE 2: Validate tool_calls arguments are valid JSON"
  ansible.builtin.set_fact:
    llm_router_toolcall_args: >-
      {{ llm_router_toolcall_probe.json.choices[0].message.tool_calls[0].function.arguments | default('') }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_toolcall_probe is defined
    - llm_router_toolcall_probe.json.choices[0].finish_reason | default('') == 'tool_calls'
  tags: [router_verify]

- name: "[router_verify] GATE 2: PASS — tool_calls returned with arguments"
  ansible.builtin.debug:
    msg:
      - "GATE 2 PASS: Router proxied tool-calling correctly."
      - "finish_reason: {{ llm_router_toolcall_probe.json.choices[0].finish_reason }}"
      - "function: {{ llm_router_toolcall_probe.json.choices[0].message.tool_calls[0].function.name | default('(unknown)') }}"
      - "arguments: {{ llm_router_toolcall_args | default('(none)') }}"
  when:
    - llm_router_enabled | default(false)
    - llm_router_toolcall_probe is defined
    - llm_router_toolcall_probe.json.choices[0].finish_reason | default('') == 'tool_calls'
  tags: [router_verify]

# --- Gate 2b: Hallucination stress test (should NOT trigger tool_calls) ---

- name: "[router_verify] GATE 2b: Hallucination stress test (no-tool prompt — should return stop)"
  ansible.builtin.uri:
    url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/chat/completions"
    method: POST
    body_format: json
    body:
      model: "{{ llm_router_expected_model_id }}"
      messages:
        - role: user
          content: "Tell me a brief fact about the planet Mars. Do not call any functions."
      tools:
        - type: function
          function:
            name: get_weather
            description: "Get current weather conditions for a city"
            parameters:
              type: object
              properties:
                city:
                  type: string
              required:
                - city
      temperature: 0.1
    status_code: 200
    return_content: true
    timeout: 120
  register: llm_router_halluc_probe
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] GATE 2b: Fail if hallucination stress test triggered spurious tool_calls"
  ansible.builtin.fail:
    msg: >-
      GATE 2b FAIL: Hallucination stress test returned finish_reason=tool_calls
      (spurious tool call on an unrelated prompt). The model is over-triggering
      tool-calling through the router proxy. Investigate router mode tool-call
      framing before proceeding.
      Full response: {{ llm_router_halluc_probe.json | to_json }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_halluc_probe is defined
    - llm_router_halluc_probe.json.choices[0].finish_reason | default('') == 'tool_calls'
  tags: [router_verify]

- name: "[router_verify] GATE 2b: PASS — hallucination stress test returned stop (no spurious tool_calls)"
  ansible.builtin.debug:
    msg: "GATE 2b PASS: finish_reason={{ llm_router_halluc_probe.json.choices[0].finish_reason }} — no spurious tool call."
  when:
    - llm_router_enabled | default(false)
    - llm_router_halluc_probe is defined
    - llm_router_halluc_probe.json.choices[0].finish_reason | default('') != 'tool_calls'
  tags: [router_verify]

# --- Gate 3: VRAM guard — --models-max 1 confirmed effective ---

- name: "[router_verify] GATE 3: Check VRAM usage after router load (--models-max 1 guard)"
  ansible.builtin.command:
    cmd: nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader
  register: llm_router_vram_post
  changed_when: false
  become: true
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] GATE 3: Parse VRAM used (MiB)"
  ansible.builtin.set_fact:
    llm_router_vram_used_mib: "{{ llm_router_vram_post.stdout.split(',')[0].strip().split(' ')[0] | int }}"
  when:
    - llm_router_enabled | default(false)
    - llm_router_vram_post is defined
  tags: [router_verify]

- name: "[router_verify] GATE 3: Fail if VRAM usage exceeds safety ceiling ({{ llm_router_vram_max_mib }} MiB)"
  ansible.builtin.fail:
    msg: >-
      GATE 3 FAIL: nvidia-smi reports {{ llm_router_vram_used_mib }} MiB VRAM used,
      which exceeds the safety ceiling of {{ llm_router_vram_max_mib }} MiB.
      --models-max 1 may not be effective, or a second model may be resident.
      Full nvidia-smi output: {{ llm_router_vram_post.stdout }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_vram_used_mib is defined
    - llm_router_vram_used_mib | int > llm_router_vram_max_mib | int
  tags: [router_verify]

- name: "[router_verify] GATE 3: Count GPU processes (should be exactly 1 — the router's Qwen child)"
  ansible.builtin.command:
    cmd: nvidia-smi --query-compute-apps=pid,name --format=csv,noheader
  register: llm_router_gpu_procs
  changed_when: false
  failed_when: false
  become: true
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] GATE 3: PASS — VRAM usage and GPU process count"
  ansible.builtin.debug:
    msg:
      - "GATE 3 PASS: VRAM {{ llm_router_vram_used_mib }} MiB / {{ llm_router_vram_max_mib }} MiB ceiling."
      - "nvidia-smi compute apps: {{ llm_router_gpu_procs.stdout_lines | default(['(none — model not yet loaded under GPU?']) }}"
      - "Full nvidia-smi: {{ llm_router_vram_post.stdout }}"
  when:
    - llm_router_enabled | default(false)
    - llm_router_vram_used_mib is defined
    - llm_router_vram_used_mib | int <= llm_router_vram_max_mib | int
  tags: [router_verify]

- name: "[router_verify] OOM CHECK: Check for OOM events in dmesg (last 120s, router-related)"
  ansible.builtin.shell:
    cmd: "dmesg --ctime | tail -200 | grep -i -E 'oom|killed|llama' || true"
  register: llm_router_oom_check
  changed_when: false
  become: true
  when: llm_router_enabled | default(false)
  tags: [router_verify]

- name: "[router_verify] OOM CHECK: Report OOM check findings"
  ansible.builtin.debug:
    msg: >-
      OOM/kill events near router start:
      {{ llm_router_oom_check.stdout if (llm_router_oom_check.stdout | length > 0) else 'None found.' }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_oom_check is defined
  tags: [router_verify]

# --- POST-VALIDATION: Restart production service -----------------------
# After validation gates: stop the router (to free VRAM), then restart production.
# The router stays installed and enabled on port 8003 for Ryan's review period —
# it will restart automatically on next reboot or systemctl start.
# When Ryan approves cutover, day2_cutover_qwen_to_router.yml will handle the
# permanent transition (router on :8002, production unit retired).

- name: "[router_verify] POST-VALIDATION: Stop router to free VRAM for production restart"
  ansible.builtin.systemd:
    name: "{{ llm_router_service_name }}"
    state: stopped
    # Do NOT disable — keep it installed and enabled for Ryan's review.
    # Router will need to be manually started again for further testing.
  become: true
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_stopped is defined
    - llm_router_qwen_stopped.changed | default(false)
  tags: [router_verify]

- name: "[router_verify] POST-VALIDATION: Wait 5s for router VRAM to be released"
  ansible.builtin.pause:
    seconds: 5
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_stopped is defined
    - llm_router_qwen_stopped.changed | default(false)
  tags: [router_verify]

- name: "[router_verify] POST-VALIDATION: Restart production llama-server-qwen (port 8002)"
  ansible.builtin.systemd:
    name: "{{ llm_qwen_service_name }}"
    state: started
    enabled: true
  become: true
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_stopped is defined
    - llm_router_qwen_stopped.changed | default(false)
  tags: [router_verify]

- name: "[router_verify] POST-VALIDATION: Wait for production /health to confirm restart"
  ansible.builtin.uri:
    url: "http://{{ llm_bind_address }}:{{ llm_qwen_port }}/health"
    status_code: 200
  register: llm_router_qwen_post_health
  retries: 30
  delay: 10
  until: llm_router_qwen_post_health.status == 200
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_stopped is defined
    - llm_router_qwen_stopped.changed | default(false)
  tags: [router_verify]

- name: "[router_verify] POST-VALIDATION: Confirm production is back on port 8002"
  ansible.builtin.debug:
    msg: >-
      Production llama-server-qwen restarted on port {{ llm_qwen_port }} and confirmed healthy.
      7 Hermes profiles (bruce-banner, groot, happy, heimdall, rocket-raccoon, war-machine, wong)
      are back to full service. Router (port 8003) is stopped but installed; restart with:
      systemctl start llama-server-router  (or via a follow-up playbook run).
  when:
    - llm_router_enabled | default(false)
    - llm_router_qwen_stopped is defined
    - llm_router_qwen_stopped.changed | default(false)
  tags: [router_verify]

# =============================================================================
# TAG: router_ui_check
# Nice-to-have: verify the bundled SvelteKit web UI is served.
# This does NOT fail the playbook on UI error — it is informational only.
# =============================================================================

- name: "[router_ui_check] GATE 4 (nice-to-have): Check bundled SvelteKit UI returns HTTP 200"
  ansible.builtin.uri:
    url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/"
    status_code: [200, 301, 302]
    return_content: false
  register: llm_router_ui_check
  failed_when: false
  when: llm_router_enabled | default(false)
  tags: [router_ui_check]

- name: "[router_ui_check] Report UI check result (informational — does not gate cutover)"
  ansible.builtin.debug:
    msg: >-
      GATE 4 (nice-to-have): Bundled UI at http://{{ llm_router_bind_address }}:{{ llm_router_port }}/
      returned HTTP {{ llm_router_ui_check.status | default('UNREACHABLE') }}.
      {{ 'PASS — UI accessible.' if (llm_router_ui_check.status | default(0) | int in [200, 301, 302])
         else 'WARN — UI not accessible. This does NOT block cutover (API traffic only, UI is cosmetic).' }}
  when:
    - llm_router_enabled | default(false)
    - llm_router_ui_check is defined
  tags: [router_ui_check]

# =============================================================================
# Summary block — print when all gates pass
# =============================================================================

- name: "[router_verify] VALIDATION SUMMARY — all hard gates passed"
  ansible.builtin.debug:
    msg:
      - "======================================================================"
      - "llama-server-router (port {{ llm_router_port }}) shadow deployment PASSED all validation gates."
      - "Gate 1 (context): n_ctx={{ llm_router_qwen_n_ctx | default('N/A') }} >= 64000 required — PASS"
      - "Gate 2 (tool-calling through router): finish_reason=tool_calls — PASS"
      - "Gate 2b (hallucination stress): no spurious tool_calls — PASS"
      - "Gate 3 (VRAM guard): {{ llm_router_vram_used_mib | default('N/A') }} MiB <= {{ llm_router_vram_max_mib }} MiB — PASS"
      - "Gate 4 (UI): {{ llm_router_ui_check.status | default('N/A') }} (informational)"
      - "----------------------------------------------------------------------"
      - "Production port 8002 (llama-server-qwen) is UNCHANGED."
      - "Post Ryan's review and sign-off on these results, War Machine will"
      - "execute day2_cutover_qwen_to_router.yml to promote the router to port 8002."
      - "======================================================================"
  when: llm_router_enabled | default(false)
  tags: [router_verify]