diff --git a/ansible/group_vars/all/semaphore.yml b/ansible/group_vars/all/semaphore.yml index 20aeef0..21bcd71 100644 --- a/ansible/group_vars/all/semaphore.yml +++ b/ansible/group_vars/all/semaphore.yml @@ -152,3 +152,28 @@ semaphore_config: required: true type: TextVar default_value: "all" + + - name: "llm_inference_multimodel_stage_models" + description: >- + Stage additional GGUF models into /opt/models on astro-orbiter via the + llm-inference-multimodel role (--tags models only). Idempotent: skips + files already present at the correct byte size. Notifies the + llama-server-router restart handler ONLY when a new GGUF is actually + downloaded. Does NOT touch Phase 4 (verify) or the legacy + llama-server-qwen service. Safe to run repeatedly. + app: ansible + playbook: ansible/playbooks/day1_deploy_llm_inference_multimodel.yml + inventory: production + repository: homelab + environment: default + vault_password: ansible-vault-pass + arguments: '["--tags","models","--diff"]' + # Scoped to --tags models: + # Phase 0 (discover) -- skipped (no tag) + # Phase 1 (models) -- RUN (idempotent GGUF staging via stage_model.yml) + # Phase 2 (systemd) -- skipped + # Phase 3 (firewall) -- skipped + # Phase 4 (verify) -- SKIPPED (collision risk: verify.yml would start + # llama-server-qwen on :8002, conflicting with the + # production llama-server-router.service. Excluded + # here deliberately. See t_730f9584.) diff --git a/ansible/host_vars/astro-orbiter/vars.yml b/ansible/host_vars/astro-orbiter/vars.yml index 1727646..f5873f5 100644 --- a/ansible/host_vars/astro-orbiter/vars.yml +++ b/ansible/host_vars/astro-orbiter/vars.yml @@ -18,3 +18,24 @@ common_root_pv: /dev/sda3 common_root_vg: ubuntu-vg common_root_lv: ubuntu-lv +# --- Staged GGUF models for the llama.cpp router (:8002) --------------------- +# Data-driven list consumed by roles/llm-inference-multimodel tasks/models.yml +# (loop -> tasks/stage_model.yml). Each entry is idempotently staged into +# /opt/models: stat + EXACT-size check vs HF manifest; skip (no download, no +# restart) when present + size matches. Source repos are public bartowski GGUFs +# on HuggingFace (no auth). A router restart is notified ONLY when a new GGUF +# is actually downloaded. +# Added 2026-08-12 (War Machine): codify Phi-3.5-mini-instruct-Q8_0 and +# Meta-Llama-3.1-8B-Instruct-Q4_K_M as router models alongside the production +# Qwen3.6-35B-A3B-UD-Q4_K_S. The live files were already present/correct on +# astro-orbiter; this pass codifies them. Future adds = append to this list. +llm_staged_models: + - filename: "Phi-3.5-mini-instruct-Q8_0.gguf" + url: "https://huggingface.co/bartowski/Phi-3.5-mini-instruct-GGUF/resolve/main/Phi-3.5-mini-instruct-Q8_0.gguf" + size_bytes: 4061222688 + source_repo: "bartowski/Phi-3.5-mini-instruct-GGUF" + - filename: "Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf" + url: "https://huggingface.co/bartowski/Meta-Llama-3.1-8B-Instruct-GGUF/resolve/main/Meta-Llama-3.1-8B-Instruct-Q4_K_M.gguf" + size_bytes: 4920739232 + source_repo: "bartowski/Meta-Llama-3.1-8B-Instruct-GGUF" + diff --git a/ansible/playbooks/day1_deploy_llm_inference_multimodel.yml b/ansible/playbooks/day1_deploy_llm_inference_multimodel.yml index 756e00b..464de18 100644 --- a/ansible/playbooks/day1_deploy_llm_inference_multimodel.yml +++ b/ansible/playbooks/day1_deploy_llm_inference_multimodel.yml @@ -11,9 +11,13 @@ # # or scope to specific phases: # ansible-playbook -i inventory.yml playbooks/day1_deploy_llm_inference_multimodel.yml --tags discover # -# KNOWN GAP (2026-08-05): Semaphore is currently broken; this is being run -# via direct ansible-playbook as an accepted interim stopgap. Retarget -# through Semaphore once it's repaired. +# EXECUTION CHANNEL (2026-08-12, War Machine): run via the Semaphore template +# "llm_inference_multimodel_stage_models" (scoped to --tags models). Do NOT +# run this via direct ansible-playbook or ad-hoc ssh/curl/systemctl — all +# homelab inference changes go through Ansible roles executed by Semaphore for +# audit/visibility. Phase 1 (models) is idempotent: it only downloads/stages a +# GGUF when missing or size-mismatched, and only restarts the router when a new +# GGUF is detected (normal re-runs that find the files correct touch nothing). # # Phases (see roles/llm-inference-multimodel/README.md for detail): # 0. discover — read-only; confirm existing Gemma service management diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index 6e5fbed..6265923 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -79,6 +79,21 @@ llm_qwen_expected_vram_gb: 20 # verified 2026-08-07: ~20,390 MiB / 24,576 MiB # llm_qwen_model_url is intentionally not set — see models.yml WARN task for # the HuggingFace URL if a re-download is ever needed. +# --- Staged GGUF models (data-driven, idempotent staging) -------------------- +# Additional GGUFs to ensure are present in llm_models_dir, alongside the +# production Qwen3.6-35B. Consumed by tasks/models.yml (loop over +# tasks/stage_model.yml). Each entry: +# filename: target filename in llm_models_dir +# url: HuggingFace resolve URL (public repos; no auth needed) +# size_bytes: EXACT expected byte size (HF manifest) — guard: download only +# if the file is missing OR its size != this value (idempotent; +# never re-pulls a correct file, never needlessly restarts). +# source_repo: upstream HF repo (audit/lineage) +# The REAL list is defined per-host in host_vars/astro-orbiter/vars.yml (NOT +# hardcoded here) so the role stays generic and reusable for future model adds. +# Empty default = nothing staged (safe no-op). +llm_staged_models: [] + # --- Existing Gemma baseline (rollback target — never modified by this role) - # Populated by Phase 0 discovery (tasks/discover.yml) if not already known. # Set here only as a fallback name to search for; discovery is authoritative. diff --git a/ansible/roles/llm-inference-multimodel/handlers/main.yml b/ansible/roles/llm-inference-multimodel/handlers/main.yml index b23b3cb..51cbcdc 100644 --- a/ansible/roles/llm-inference-multimodel/handlers/main.yml +++ b/ansible/roles/llm-inference-multimodel/handlers/main.yml @@ -23,3 +23,15 @@ daemon_reload: true become: true listen: "reload systemd" + +# Restart the llama.cpp router so it re-discovers /opt/models after a NEW GGUF +# is staged. NOTIFIED ONLY from tasks/stage_model.yml when an actual download +# (or permission correction) occurs — a normal idempotent re-run that finds the +# files already correct will NOT fire this, so the live router is left +# untouched. Safe on the idle GPU (router holds no resident model when all +# entries are "unloaded"; restart is sub-second). +- name: restart llama-server-router on new GGUF + ansible.builtin.systemd: + name: "{{ llm_router_service_name | default('llama-server-router') }}" + state: restarted + become: true diff --git a/ansible/roles/llm-inference-multimodel/tasks/models.yml b/ansible/roles/llm-inference-multimodel/tasks/models.yml index 397d162..300b805 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/models.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/models.yml @@ -59,3 +59,18 @@ Expected URL (bartowski UD-Q4_K_S): https://huggingface.co/bartowski/Qwen3.6-35B-A3B-UD-Q4_K_S-GGUF/resolve/main/Qwen3.6-35B-A3B-UD-Q4_K_S.gguf when: not (llm_qwen_model_stat.stat.exists | default(false)) + +# --- Staged GGUF models (data-driven, idempotent) ---------------------------- +# Ensure every entry in llm_staged_models is present in llm_models_dir with the +# EXACT expected byte size. When present AND size matches, this is a pure +# no-op: no download, no service touch. When a genuine new/mismatched GGUF is +# detected, it is downloaded + ownership/mode corrected and the router restart +# handler is notified so the llama.cpp router re-discovers the models_dir. +# Driven entirely by inventory vars (host_vars) — nothing hardcoded here, so +# adding a future model = append to llm_staged_models in host_vars. +- name: Stage data-driven GGUF models into {{ llm_models_dir }} + ansible.builtin.include_tasks: stage_model.yml + loop: "{{ llm_staged_models | default([]) }}" + loop_control: + loop_var: staged_model + tags: [models] diff --git a/ansible/roles/llm-inference-multimodel/tasks/stage_model.yml b/ansible/roles/llm-inference-multimodel/tasks/stage_model.yml new file mode 100644 index 0000000..3068997 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/stage_model.yml @@ -0,0 +1,67 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/stage_model.yml +# DESCRIPTION: Idempotent staging of a SINGLE GGUF listed in llm_staged_models. +# Looped from tasks/models.yml (one include per entry). +# +# GUARD (idempotency): +# - stat the target path in llm_models_dir +# - get_url ONLY when the file is MISSING or its on-disk size +# != the exact HF manifest size (staged_model.size_bytes) +# - when present AND size matches -> pure no-op (no download, +# no service touch) +# +# SIDE EFFECT ON CHANGE: +# When a genuine new/mismatched GGUF is downloaded (or its +# ownership/mode corrected), this task notifies the +# "restart llama-server-router on new GGUF" handler so the +# llama.cpp router re-discovers the models_dir. A normal +# re-run that finds the files already correct will NOT fire +# the handler — the live router is left untouched. +# +# Driven entirely by inventory vars (host_vars), nothing +# hardcoded here, so adding a future model = append to the list. +# ------------------------------------------------------------------------------ + +- name: "Check if {{ staged_model.filename }} is present on disk" + ansible.builtin.stat: + path: "{{ llm_models_dir }}/{{ staged_model.filename }}" + register: _staged_stat + +- name: "Report {{ staged_model.filename }} presence (source: {{ staged_model.source_repo | default('n/a') }})" + ansible.builtin.debug: + msg: >- + {{ staged_model.filename }}: + exists={{ _staged_stat.stat.exists | default(false) }}, + size={{ (_staged_stat.stat.size | default(0) | int) }}, + expected={{ staged_model.size_bytes | int }}, + match={{ (_staged_stat.stat.exists | default(false)) and + ((_staged_stat.stat.size | default(0) | int) == (staged_model.size_bytes | int)) }} + when: _staged_stat.stat.exists | default(false) + +- name: "Download {{ staged_model.filename }} (missing or size mismatch)" + ansible.builtin.get_url: + url: "{{ staged_model.url }}" + dest: "{{ llm_models_dir }}/{{ staged_model.filename }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0664" + timeout: 600 + become: true + when: >- + not (_staged_stat.stat.exists | default(false)) + or (_staged_stat.stat.size | default(0) | int != (staged_model.size_bytes | int)) + register: _staged_download + notify: restart llama-server-router on new GGUF + +- name: "Ensure ownership/mode on {{ staged_model.filename }}" + ansible.builtin.file: + path: "{{ llm_models_dir }}/{{ staged_model.filename }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0664" + become: true + when: >- + (_staged_stat.stat.exists | default(false)) + or (_staged_download is changed | default(false)) + notify: restart llama-server-router on new GGUF