---
# ------------------------------------------------------------------------------
# FILE: playbooks/day1_deploy_llm_router_shadow.yml
# DESCRIPTION: Deploy llama-server in router mode on a shadow port (8003).
#
# This playbook deploys and validates the llama.cpp router mode supervisor on
# astro-orbiter (10.1.71.130) WITHOUT touching the production endpoint
# (llama-server-qwen, port 8002). All 7 dependent Hermes profiles
# (bruce-banner, groot, happy, heimdall, rocket-raccoon, war-machine, wong)
# remain pointing at port 8002 throughout this run.
#
# Usage (from ~/git/homelab/ansible):
#   ansible-playbook -i inventory.yml playbooks/day1_deploy_llm_router_shadow.yml
#
# Tag-scoped runs (if you need to re-run one phase):
#   ansible-playbook -i inventory.yml playbooks/day1_deploy_llm_router_shadow.yml \
#     --tags router_systemd,router_firewall,router_verify
#
# Execution path (Ryan-approved 2026-08-12, task t_0cca74a2):
#   Direct ansible-playbook as documented exception — Semaphore template for
#   this role does not exist yet. Create template after cutover is confirmed.
#   This is the same exception pattern used in prior sessions on this box.
#
# Pre-requisites:
#   1. llama-server binary at /opt/llama.cpp/build/bin/llama-server supports
#      router mode (confirmed 2026-08-12: --models-dir flag present in --help).
#   2. /opt/models/ contains ONLY Qwen3.6-35B-A3B-UD-Q4_K_S.gguf
#      (confirmed 2026-08-12: directory is clean, Phi-4/Mistral already deleted).
#   3. Port 8002 is in use by the production llama-server-qwen service —
#      this playbook does NOT touch it.
#
# Validation gates this playbook runs (all hard gates EXCEPT Gate 4):
#   Gate 1: /v1/models reports Qwen with n_ctx >= 64000 (64K Hermes floor)
#   Gate 2: Tool-calling probe through router returns finish_reason=tool_calls
#   Gate 2b: Hallucination stress test does NOT trigger spurious tool_calls
#   Gate 3: nvidia-smi VRAM <= 23,000 MiB (--models-max 1 confirmed effective)
#   Gate 4: Bundled SvelteKit UI check (nice-to-have, non-blocking)
#
# What happens after this playbook:
#   War Machine posts validation gate results to Ryan.
#   Ryan reviews and signs off on cutover (or requests changes).
#   War Machine then runs day2_cutover_qwen_to_router.yml (not yet created)
#   to promote the router to port 8002 and retire the bare llama-server-qwen.
#
# Reference: proposal at
#   ~/friday/system/inbox/agents/war-machine/2026-08-12-qwen-router-mode-proposal.md
# Task: t_0cca74a2
# Author: War Machine (2026-08-12)
# ------------------------------------------------------------------------------

- name: Deploy llama-server router (shadow, port 8003) on astro-orbiter
  hosts: astro_orbiter
  gather_facts: true
  become: true

  vars:
    # Enable the router phase — this is the ONLY var that makes router.yml run.
    # Default in defaults/main.yml is false (no-op). Flip here for the shadow run.
    llm_router_enabled: true

    # Qwen model ID as it appears in /v1/models from the router.
    # llama-server router uses the GGUF filename (without .gguf) as the model id.
    llm_router_expected_model_id: "Qwen3.6-35B-A3B-UD-Q4_K_S"

  roles:
    - role: llm-inference-multimodel

  # No --tags needed here: router.yml is included dynamically from main.yml
  # whenever llm_router_enabled: true. The full role runs but the
  # discover/models/systemd/verify phases are gated on their own vars
  # (llm_qwen_service_enabled etc.) and are idempotent. The stale
  # models.yml (Phi-4/Mistral download tasks) uses variables no longer
  # defined — a follow-up cleanup task should update that file.

- name: "POST-VALIDATION SAFETY NET — ensure production service is running"
  hosts: astro_orbiter
  gather_facts: false
  become: true

  tasks:
    # Always run this, regardless of whether the validation play succeeded.
    # If the router.yml play stopped llama-server-qwen for VRAM validation
    # and then a gate failed (play aborted), this play ensures it comes back up.
    - name: "Ensure llama-server-qwen (port 8002) is running after validation (always)"
      ansible.builtin.systemd:
        name: llama-server-qwen
        state: started
        enabled: true
      ignore_errors: true   # don't fail if the unit doesn't exist

    - name: "Verify production /health after safety-net restart"
      ansible.builtin.uri:
        url: "http://10.1.71.130:8002/health"
        status_code: 200
        timeout: 30
      register: llm_safety_net_health
      failed_when: false
      ignore_errors: true

    - name: "Report production status (safety-net check)"
      ansible.builtin.debug:
        msg: >-
          Safety-net: llama-server-qwen :8002 health check returned
          {{ llm_safety_net_health.status | default('UNREACHABLE') }}.
          {{ 'OK — production is up.' if (llm_safety_net_health.status | default(0) | int == 200)
             else 'WARNING — production may not be healthy. Check manually.' }}