feat(llm-inference-multimodel): codify Phi-3.5-mini + Llama-3.1-8B GGUF staging (t_730f9584)

Adds idempotent, data-driven GGUF staging for the two new router models on
astro-orbiter alongside the production Qwen3.6-35B-A3B-UD-Q4_K_S. Both files
were already staged live (byte-exact); this commit codifies them in Ansible so
future re-runs and any new model adds are version-controlled and audit-friendly.

Changes:
- roles/llm-inference-multimodel/tasks/stage_model.yml (NEW)
  Idempotent per-model task: stat -> exact byte-size guard -> conditional
  get_url -> ownership/mode ensure -> notify router restart handler only on
  actual download. Loops from models.yml; nothing hardcoded.

- roles/llm-inference-multimodel/tasks/models.yml
  Appends the stage_model.yml loop (tagged: models) after the existing Qwen3.6
  download tasks. Data driven from host_vars/astro-orbiter/vars.yml.

- roles/llm-inference-multimodel/defaults/main.yml
  Adds llm_staged_models: [] default (empty = safe no-op for hosts with no
  staged model list defined).

- roles/llm-inference-multimodel/handlers/main.yml
  Adds 'restart llama-server-router on new GGUF' handler. Only fires when
  stage_model.yml performs an actual download or corrects ownership/mode.
  Normal idempotent re-runs (files already correct) do NOT fire this handler.

- host_vars/astro-orbiter/vars.yml
  Adds llm_staged_models list with the two new models:
    * Phi-3.5-mini-instruct-Q8_0.gguf (4,061,222,688 bytes,
      bartowski/Phi-3.5-mini-instruct-GGUF)
    * Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf (4,920,739,232 bytes,
      bartowski/Meta-Llama-3.1-8B-Instruct-GGUF)

- playbooks/day1_deploy_llm_inference_multimodel.yml
  Updates header comment: removes stale 'Semaphore broken' note, documents
  the correct execution channel (Semaphore template
  llm_inference_multimodel_stage_models, --tags models).

- group_vars/all/semaphore.yml
  Adds llm_inference_multimodel_stage_models template entry (config-as-code).
  Template is scoped to --tags models explicitly. Phase 4 (verify) is
  EXCLUDED: verify.yml starts llama-server-qwen on :8002, which would collide
  with the production llama-server-router.service already running on :8002.

Semaphore template created via API: project 1 / template id 19.
Execution: triggered immediately after this commit via Semaphore REST API.

ansible/group_vars/all/semaphore.yml

@@ -152,3 +152,28 @@ semaphore_config:
           required: true
           type: TextVar
           default_value: "all"
+
+    - name: "llm_inference_multimodel_stage_models"
+      description: >-
+        Stage additional GGUF models into /opt/models on astro-orbiter via the
+        llm-inference-multimodel role (--tags models only). Idempotent: skips
+        files already present at the correct byte size. Notifies the
+        llama-server-router restart handler ONLY when a new GGUF is actually
+        downloaded. Does NOT touch Phase 4 (verify) or the legacy
+        llama-server-qwen service. Safe to run repeatedly.
+      app: ansible
+      playbook: ansible/playbooks/day1_deploy_llm_inference_multimodel.yml
+      inventory: production
+      repository: homelab
+      environment: default
+      vault_password: ansible-vault-pass
+      arguments: '["--tags","models","--diff"]'
+      # Scoped to --tags models:
+      #   Phase 0 (discover) -- skipped (no tag)
+      #   Phase 1 (models)   -- RUN (idempotent GGUF staging via stage_model.yml)
+      #   Phase 2 (systemd)  -- skipped
+      #   Phase 3 (firewall) -- skipped
+      #   Phase 4 (verify)   -- SKIPPED (collision risk: verify.yml would start
+      #                         llama-server-qwen on :8002, conflicting with the
+      #                         production llama-server-router.service. Excluded
+      #                         here deliberately. See t_730f9584.)

ansible/host_vars/astro-orbiter/vars.yml

@@ -18,3 +18,24 @@ common_root_pv: /dev/sda3
 common_root_vg: ubuntu-vg
 common_root_lv: ubuntu-lv
 
+# --- Staged GGUF models for the llama.cpp router (:8002) ---------------------
+# Data-driven list consumed by roles/llm-inference-multimodel tasks/models.yml
+# (loop -> tasks/stage_model.yml). Each entry is idempotently staged into
+# /opt/models: stat + EXACT-size check vs HF manifest; skip (no download, no
+# restart) when present + size matches. Source repos are public bartowski GGUFs
+# on HuggingFace (no auth). A router restart is notified ONLY when a new GGUF
+# is actually downloaded.
+# Added 2026-08-12 (War Machine): codify Phi-3.5-mini-instruct-Q8_0 and
+# Meta-Llama-3.1-8B-Instruct-Q4_K_M as router models alongside the production
+# Qwen3.6-35B-A3B-UD-Q4_K_S. The live files were already present/correct on
+# astro-orbiter; this pass codifies them. Future adds = append to this list.
+llm_staged_models:
+  - filename: "Phi-3.5-mini-instruct-Q8_0.gguf"
+    url: "https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q8_0.gguf"
+    size_bytes: 4061222688
+    source_repo: "bartowski/Phi-3.5-mini-instruct-GGUF"
+  - filename: "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
+    url: "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf"
+    size_bytes: 4920739232
+    source_repo: "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF"
+

ansible/playbooks/day1_deploy_llm_inference_multimodel.yml

@@ -11,9 +11,13 @@
 #   # or scope to specific phases:
 #   ansible-playbook -i inventory.yml playbooks/day1_deploy_llm_inference_multimodel.yml --tags discover
 #
-# KNOWN GAP (2026-08-05): Semaphore is currently broken; this is being run
-# via direct ansible-playbook as an accepted interim stopgap. Retarget
-# through Semaphore once it's repaired.
+# EXECUTION CHANNEL (2026-08-12, War Machine): run via the Semaphore template
+# "llm_inference_multimodel_stage_models" (scoped to --tags models). Do NOT
+# run this via direct ansible-playbook or ad-hoc ssh/curl/systemctl — all
+# homelab inference changes go through Ansible roles executed by Semaphore for
+# audit/visibility. Phase 1 (models) is idempotent: it only downloads/stages a
+# GGUF when missing or size-mismatched, and only restarts the router when a new
+# GGUF is detected (normal re-runs that find the files correct touch nothing).
 #
 # Phases (see roles/llm-inference-multimodel/README.md for detail):
 #   0. discover  — read-only; confirm existing Gemma service management

ansible/roles/llm-inference-multimodel/defaults/main.yml

@@ -79,6 +79,21 @@ llm_qwen_expected_vram_gb: 20  # verified 2026-08-07: ~20,390 MiB / 24,576 MiB
 # llm_qwen_model_url is intentionally not set — see models.yml WARN task for
 # the HuggingFace URL if a re-download is ever needed.
 
+# --- Staged GGUF models (data-driven, idempotent staging) --------------------
+# Additional GGUFs to ensure are present in llm_models_dir, alongside the
+# production Qwen3.6-35B. Consumed by tasks/models.yml (loop over
+# tasks/stage_model.yml). Each entry:
+#   filename:    target filename in llm_models_dir
+#   url:         HuggingFace resolve URL (public repos; no auth needed)
+#   size_bytes:  EXACT expected byte size (HF manifest) — guard: download only
+#                if the file is missing OR its size != this value (idempotent;
+#                never re-pulls a correct file, never needlessly restarts).
+#   source_repo: upstream HF repo (audit/lineage)
+# The REAL list is defined per-host in host_vars/astro-orbiter/vars.yml (NOT
+# hardcoded here) so the role stays generic and reusable for future model adds.
+# Empty default = nothing staged (safe no-op).
+llm_staged_models: []
+
 # --- Existing Gemma baseline (rollback target — never modified by this role) -
 # Populated by Phase 0 discovery (tasks/discover.yml) if not already known.
 # Set here only as a fallback name to search for; discovery is authoritative.

ansible/roles/llm-inference-multimodel/handlers/main.yml

@@ -23,3 +23,15 @@
     daemon_reload: true
   become: true
   listen: "reload systemd"
+
+# Restart the llama.cpp router so it re-discovers /opt/models after a NEW GGUF
+# is staged. NOTIFIED ONLY from tasks/stage_model.yml when an actual download
+# (or permission correction) occurs — a normal idempotent re-run that finds the
+# files already correct will NOT fire this, so the live router is left
+# untouched. Safe on the idle GPU (router holds no resident model when all
+# entries are "unloaded"; restart is sub-second).
+- name: restart llama-server-router on new GGUF
+  ansible.builtin.systemd:
+    name: "{{ llm_router_service_name | default('llama-server-router') }}"
+    state: restarted
+  become: true

ansible/roles/llm-inference-multimodel/tasks/models.yml

@@ -59,3 +59,18 @@
       Expected URL (bartowski UD-Q4_K_S):
       https://huggingface.co/bartowski/Qwen3.6-35B-A3B-UD-Q4_K_S-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf
   when: not (llm_qwen_model_stat.stat.exists | default(false))
+
+# --- Staged GGUF models (data-driven, idempotent) ----------------------------
+# Ensure every entry in llm_staged_models is present in llm_models_dir with the
+# EXACT expected byte size. When present AND size matches, this is a pure
+# no-op: no download, no service touch. When a genuine new/mismatched GGUF is
+# detected, it is downloaded + ownership/mode corrected and the router restart
+# handler is notified so the llama.cpp router re-discovers the models_dir.
+# Driven entirely by inventory vars (host_vars) — nothing hardcoded here, so
+# adding a future model = append to llm_staged_models in host_vars.
+- name: Stage data-driven GGUF models into {{ llm_models_dir }}
+  ansible.builtin.include_tasks: stage_model.yml
+  loop: "{{ llm_staged_models | default([]) }}"
+  loop_control:
+    loop_var: staged_model
+  tags: [models]

ansible/roles/llm-inference-multimodel/tasks/stage_model.yml

@@ -0,0 +1,67 @@
+---
+# ------------------------------------------------------------------------------
+# FILE: roles/llm-inference-multimodel/tasks/stage_model.yml
+# DESCRIPTION: Idempotent staging of a SINGLE GGUF listed in llm_staged_models.
+#              Looped from tasks/models.yml (one include per entry).
+#
+#              GUARD (idempotency):
+#                - stat the target path in llm_models_dir
+#                - get_url ONLY when the file is MISSING or its on-disk size
+#                  != the exact HF manifest size (staged_model.size_bytes)
+#                - when present AND size matches -> pure no-op (no download,
+#                  no service touch)
+#
+#              SIDE EFFECT ON CHANGE:
+#                When a genuine new/mismatched GGUF is downloaded (or its
+#                ownership/mode corrected), this task notifies the
+#                "restart llama-server-router on new GGUF" handler so the
+#                llama.cpp router re-discovers the models_dir. A normal
+#                re-run that finds the files already correct will NOT fire
+#                the handler — the live router is left untouched.
+#
+#              Driven entirely by inventory vars (host_vars), nothing
+#              hardcoded here, so adding a future model = append to the list.
+# ------------------------------------------------------------------------------
+
+- name: "Check if {{ staged_model.filename }} is present on disk"
+  ansible.builtin.stat:
+    path: "{{ llm_models_dir }}/{{ staged_model.filename }}"
+  register: _staged_stat
+
+- name: "Report {{ staged_model.filename }} presence (source: {{ staged_model.source_repo | default('n/a') }})"
+  ansible.builtin.debug:
+    msg: >-
+      {{ staged_model.filename }}:
+      exists={{ _staged_stat.stat.exists | default(false) }},
+      size={{ (_staged_stat.stat.size | default(0) | int) }},
+      expected={{ staged_model.size_bytes | int }},
+      match={{ (_staged_stat.stat.exists | default(false)) and
+               ((_staged_stat.stat.size | default(0) | int) == (staged_model.size_bytes | int)) }}
+  when: _staged_stat.stat.exists | default(false)
+
+- name: "Download {{ staged_model.filename }} (missing or size mismatch)"
+  ansible.builtin.get_url:
+    url: "{{ staged_model.url }}"
+    dest: "{{ llm_models_dir }}/{{ staged_model.filename }}"
+    owner: "{{ llm_service_user }}"
+    group: "{{ llm_service_user }}"
+    mode: "0664"
+    timeout: 600
+  become: true
+  when: >-
+    not (_staged_stat.stat.exists | default(false))
+    or (_staged_stat.stat.size | default(0) | int != (staged_model.size_bytes | int))
+  register: _staged_download
+  notify: restart llama-server-router on new GGUF
+
+- name: "Ensure ownership/mode on {{ staged_model.filename }}"
+  ansible.builtin.file:
+    path: "{{ llm_models_dir }}/{{ staged_model.filename }}"
+    owner: "{{ llm_service_user }}"
+    group: "{{ llm_service_user }}"
+    mode: "0664"
+  become: true
+  when: >-
+    (_staged_stat.stat.exists | default(false))
+    or (_staged_download is changed | default(false))
+  notify: restart llama-server-router on new GGUF