From c3755aa29eb69e24346fe71841f19ccd6117f479 Mon Sep 17 00:00:00 2001 From: Hermes Agent service account Date: Wed, 5 Aug 2026 15:53:31 -0500 Subject: [PATCH] llm-inference-multimodel: role + day1 playbook (phase 0 discover approved) --- .../day1_deploy_llm_inference_multimodel.yml | 31 ++++ .../roles/llm-inference-multimodel/README.md | 132 +++++++++++++++++ .../defaults/main.yml | 59 ++++++++ .../docs/validation-log.md | 46 ++++++ .../handlers/main.yml | 27 ++++ .../llm-inference-multimodel/meta/main.yml | 17 +++ .../tasks/discover.yml | 87 ++++++++++++ .../tasks/firewall.yml | 64 +++++++++ .../llm-inference-multimodel/tasks/main.yml | 35 +++++ .../llm-inference-multimodel/tasks/models.yml | 72 ++++++++++ .../tasks/systemd.yml | 54 +++++++ .../llm-inference-multimodel/tasks/verify.yml | 133 ++++++++++++++++++ .../templates/llama-server-aux.service.j2 | 33 +++++ .../llama-server-toolcall.service.j2 | 38 +++++ .../llm-inference-multimodel/vars/main.yml | 22 +++ 15 files changed, 850 insertions(+) create mode 100644 ansible/playbooks/day1_deploy_llm_inference_multimodel.yml create mode 100644 ansible/roles/llm-inference-multimodel/README.md create mode 100644 ansible/roles/llm-inference-multimodel/defaults/main.yml create mode 100644 ansible/roles/llm-inference-multimodel/docs/validation-log.md create mode 100644 ansible/roles/llm-inference-multimodel/handlers/main.yml create mode 100644 ansible/roles/llm-inference-multimodel/meta/main.yml create mode 100644 ansible/roles/llm-inference-multimodel/tasks/discover.yml create mode 100644 ansible/roles/llm-inference-multimodel/tasks/firewall.yml create mode 100644 ansible/roles/llm-inference-multimodel/tasks/main.yml create mode 100644 ansible/roles/llm-inference-multimodel/tasks/models.yml create mode 100644 ansible/roles/llm-inference-multimodel/tasks/systemd.yml create mode 100644 ansible/roles/llm-inference-multimodel/tasks/verify.yml create mode 100644 ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 create mode 100644 ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 create mode 100644 ansible/roles/llm-inference-multimodel/vars/main.yml diff --git a/ansible/playbooks/day1_deploy_llm_inference_multimodel.yml b/ansible/playbooks/day1_deploy_llm_inference_multimodel.yml new file mode 100644 index 0000000..756e00b --- /dev/null +++ b/ansible/playbooks/day1_deploy_llm_inference_multimodel.yml @@ -0,0 +1,31 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: playbooks/day1_deploy_llm_inference_multimodel.yml +# DESCRIPTION: Day 1 playbook for the dual-model (aux + tool-calling) rollout +# on astro-orbiter. Builds on roles/llm-inference (CUDA/driver +# already done) — does not replace it. +# +# Usage: +# cd ~/git/homelab/ansible +# ansible-playbook -i inventory.yml playbooks/day1_deploy_llm_inference_multimodel.yml +# # or scope to specific phases: +# ansible-playbook -i inventory.yml playbooks/day1_deploy_llm_inference_multimodel.yml --tags discover +# +# KNOWN GAP (2026-08-05): Semaphore is currently broken; this is being run +# via direct ansible-playbook as an accepted interim stopgap. Retarget +# through Semaphore once it's repaired. +# +# Phases (see roles/llm-inference-multimodel/README.md for detail): +# 0. discover — read-only; confirm existing Gemma service management +# 1. models — idempotent GGUF downloads (Phi-4-14B, Mistral-Small-24B) +# 2. systemd — deploy both unit files, do NOT auto-start +# 3. firewall — scope ports 8000/8001, non-0.0.0.0 bind +# 4. verify — start both services, smoke test, VRAM check +# ------------------------------------------------------------------------------ + +- name: Deploy dual-model LLM inference stack on astro-orbiter + hosts: astro_orbiter + gather_facts: true + + roles: + - role: llm-inference-multimodel diff --git a/ansible/roles/llm-inference-multimodel/README.md b/ansible/roles/llm-inference-multimodel/README.md new file mode 100644 index 0000000..7e39565 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/README.md @@ -0,0 +1,132 @@ +# llm-inference-multimodel + +Deploys **two independent llama-server systemd services** on astro-orbiter's +RTX 3090 (24GB), alongside — not replacing — the existing `llm-inference` role: + +| Instance | Port | Model | Quant | ctx | parallel | ~VRAM | +|---|---|---|---|---|---|---| +| `llama-server-aux` | 8000 | Phi-4-14B-Instruct | Q4_K_M | 8192 | 2 | ~10.0GB | +| `llama-server-toolcall` | 8001 | Mistral-Small-24B-Instruct-2501 | Q3_K_M | 4096 | 1 | ~13.2GB | + +Combined estimate: **~23.2GB / 24GB** (~0.8GB headroom). See +`/home/hermes/astro-orbiter-multi-model-plan.md` for the full approved design +(VRAM math, model selection rationale, rollback plan, validation harness). + +## Relationship to `roles/llm-inference` + +This role does **not** replace `llm-inference`. It assumes that role's +prerequisites are already satisfied on the host: + +- NVIDIA driver installed +- `/opt/llama.cpp` cloned and built with CUDA (`/opt/llama.cpp/build/bin/llama-server` exists) +- `jarvis` service user + `/home/jarvis` present + +The pre-existing single-model Gemma llama-server (however it is currently +run) is **never modified, restarted, or deleted** by this role. It is the +rollback target. + +## Phases + +Run the whole role, or scope with `--tags`: + +``` +ansible-playbook -i inventory.yml playbooks/day1_deploy_llm_inference_multimodel.yml +# or, once merged into a single play: +ansible-playbook -i inventory.yml .yml --tags discover,models,systemd,firewall,verify +``` + +0. **discover** (`tasks/discover.yml`) — READ-ONLY. Confirms via + `service_facts` + `pgrep` whether the existing Gemma llama-server actually + runs as a systemd unit today, or some ad hoc way (nohup/screen/tmux). Does + **not** assume a unit exists — this was an open unknown in the plan and is + resolved here as a fact-gathering step, not an assumption. Also records + baseline VRAM and current port 8000/8001 listeners for comparison later. + + **If this reports no unit found**, stop and read the debug message — + it means plan §6's rollback story ("systemctl start the old unit to + revert") isn't actually available yet, and that should be fixed (codify + the existing process as a systemd unit) before proceeding to Phase 2. + +1. **models** (`tasks/models.yml`) — Idempotent GGUF download to + `/opt/models/` with a stat + minimum-size guard (mirrors the pattern in + `roles/llm-inference/tasks/serve.yml` and the `llm-inference-homelab` + skill), so reruns don't re-pull 8.5GB/11.7GB files or mistake a truncated + partial download for complete. + +2. **systemd** (`tasks/systemd.yml`) — Templates and deploys both unit files + to `/etc/systemd/system/`. **Deliberately does not start or enable either + service** — units land on disk as a separately reviewable checkpoint. + Two fully independent units (not one unit with two ExecStarts) so either + instance can be restarted/stopped without affecting the other. + +3. **firewall** (`tasks/firewall.yml`) — Scopes ports 8000 and 8001 via `ufw` + to `llm_allowed_source_cidr` (default the Hermes LAN subnet), rather than + leaving them open. Both unit templates also bind to + `llm_bind_address` (default `10.1.71.130`, the host's private LAN IP) — + **not `0.0.0.0`** — which is a deliberate change from the pre-existing + Gemma pattern flagged as insecure in the plan. + +4. **verify** (`tasks/verify.yml`) — The only phase that actually starts + + enables both services. Waits for `/health` on both ports, smoke-tests + `/v1/models` and a trivial `/v1/chat/completions` call on each, checks + `nvidia-smi` VRAM usage against the plan's design estimate, and greps + `dmesg` for OOM-kill events. + + **This smoke test is not the tool-calling validation harness.** See + below. + +## Key variables + +Defined in `defaults/main.yml` (all overridable via `host_vars`/`group_vars` +or `-e`): + +- `llm_service_user` (jarvis), `llm_binary_path`, `llm_models_dir`, `llm_bind_address`, `llm_allowed_source_cidr` +- Aux: `llm_aux_port`, `llm_aux_model_path`, `llm_aux_model_url`, `llm_aux_ctx_size`, `llm_aux_parallel`, `llm_aux_gpu_layers` +- Tool-calling: `llm_toolcall_port`, `llm_toolcall_model_path`, `llm_toolcall_model_url`, `llm_toolcall_ctx_size`, `llm_toolcall_parallel`, `llm_toolcall_gpu_layers` + +`vars/main.yml` holds constants not meant to be overridden per-host (HF token +reference, expected-VRAM figures used only for the verify.yml report). + +## ⚠️ Tool-calling validation is required before use + +Port 8001 (Mistral-Small-24B) **must** pass the manual validation procedure +described in plan §7 before any Claude Code / tool-calling-capable Hermes +profile is pointed at it: + +1. A curl-based `tool_calls` emission probe (does it call tools correctly on + known trigger prompts?) +2. A hallucination stress test (does it fabricate `tool_calls` on prompts + that shouldn't trigger any?) +3. A shadow-mode period (run parallel to the existing tool-calling path, + compare outputs, before a hard cutover) + +This is **intentionally not automated into this role** — it is a +correctness/safety judgment call, not a repeatable infra check. See +`docs/validation-log.md` in this role directory for the procedure reference +and a place to log results once Ryan runs it. + +## Known gap: Semaphore is broken (as of 2026-08-05) + +The normal execution/audit path (Semaphore) is currently non-functional. +This role was authored to be run via direct `ansible-playbook` as an accepted +interim stopgap, executed personally by Ryan. **This is a known gap, not the +intended long-term operational path** — once Semaphore is repaired, retarget +execution of this role (and future changes to it) through Semaphore so runs +are audited/logged there again. Flag this in any future work that touches +this role. + +## Rollback + +The existing Gemma llama-server and its GGUF are untouched by every phase of +this role. To roll back: + +1. `systemctl stop llama-server-aux llama-server-toolcall` +2. `systemctl disable llama-server-aux llama-server-toolcall` (optional, if reverting permanently) +3. Confirm the original Gemma service (name determined by `discover.yml`, + commonly `llama-server.service`) is (still) running: `systemctl status llama-server` +4. If it was never running because Phase 0 discovered it wasn't a managed + unit, whatever ad hoc process/command was used before this role's changes + is also unaffected — nothing in this role stopped it. + +No files belonging to the existing Gemma deployment (GGUF, unit file, or +otherwise) are ever written to or deleted by this role. diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml new file mode 100644 index 0000000..b392f21 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -0,0 +1,59 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/defaults/main.yml +# DESCRIPTION: Overridable defaults for the llm-inference-multimodel role. +# Deploy target: astro-orbiter (10.1.71.130, RTX 3090 24GB). +# Built ALONGSIDE roles/llm-inference (not a replacement) — that +# role's CUDA/build/driver phases are the prerequisite; this role +# assumes /opt/llama.cpp/build/bin/llama-server already exists. +# +# See /home/hermes/astro-orbiter-multi-model-plan.md for the full +# approved design (VRAM math, rationale, rollback story). +# ------------------------------------------------------------------------------ + +# Shared +llm_service_user: jarvis +llm_binary_path: /opt/llama.cpp/build/bin/llama-server +llm_models_dir: /opt/models + +# Bind address — deliberately NOT 0.0.0.0 (see plan §5). Default to the private +# LAN interface so both instances are reachable from Hermes but not the world. +# Override to 127.0.0.1 if even LAN-wide reachability is unwanted and a reverse +# proxy/localhost-only tunnel is used instead. +llm_bind_address: "10.1.71.130" + +# Firewall scoping (Phase 3) — subnet/hosts allowed to reach the ports above. +# Override per-environment; default assumes Hermes runs somewhere on this /24. +llm_allowed_source_cidr: "10.1.70.0/24" + +# --- Aux / classification instance (port 8000, Phi-4-14B) ------------------- +# Text-only instruction model, no tool-calling training — safe offload target +# per the auxiliary-task-offload skill's "no tool_calls emission risk" bar. +llm_aux_port: 8000 +llm_aux_model_path: "{{ llm_models_dir }}/phi-4-14b-instruct-Q4_K_M.gguf" +llm_aux_model_url: "https://huggingface.co/bartowski/phi-4-GGUF/resolve/main/phi-4-Q4_K_M.gguf" +llm_aux_model_min_bytes: 8000000000 # guard threshold; complete file ~8.5GB +llm_aux_ctx_size: 8192 +llm_aux_parallel: 2 +llm_aux_gpu_layers: 99 +llm_aux_service_name: llama-server-aux +llm_aux_model_id: phi-4-14b-instruct # served model name for OpenAI-compat API + +# --- Tool-calling instance (port 8001, Mistral-Small-24B) -------------------- +# Native function-calling support; deployed at Q3_K_M per plan §1 Option B +# to fit VRAM budget. MUST pass the §7 validation harness before any +# Claude-Code-capable profile is pointed at this port. +llm_toolcall_port: 8001 +llm_toolcall_model_path: "{{ llm_models_dir }}/mistral-small-24b-instruct-2501-Q3_K_M.gguf" +llm_toolcall_model_url: "https://huggingface.co/bartowski/Mistral-Small-24B-Instruct-2501-GGUF/resolve/main/Mistral-Small-24B-Instruct-2501-Q3_K_M.gguf" +llm_toolcall_model_min_bytes: 11000000000 # guard threshold; complete file ~11.7GB +llm_toolcall_ctx_size: 4096 +llm_toolcall_parallel: 1 +llm_toolcall_gpu_layers: 99 +llm_toolcall_service_name: llama-server-toolcall +llm_toolcall_model_id: mistral-small-24b-instruct-2501 + +# --- Existing Gemma baseline (rollback target — never modified by this role) - +# Populated by Phase 0 discovery (tasks/discover.yml) if not already known. +# Set here only as a fallback name to search for; discovery is authoritative. +llm_existing_gemma_service_name_guess: llama-server diff --git a/ansible/roles/llm-inference-multimodel/docs/validation-log.md b/ansible/roles/llm-inference-multimodel/docs/validation-log.md new file mode 100644 index 0000000..4efd474 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/docs/validation-log.md @@ -0,0 +1,46 @@ +# Tool-Calling Model Validation Log + +This file tracks the manual validation procedure required by +`astro-orbiter-multi-model-plan.md` §7 before `llama-server-toolcall` (port +8001, Mistral-Small-24B-Instruct-2501 Q3_K_M) is trusted for any real +tool-calling / Claude Code Hermes profile traffic. + +This is **not automated by the role** — `tasks/verify.yml` only confirms the +endpoint is up and can produce a basic completion. The checks below are a +correctness/safety judgment call that a human runs and records here. + +## Procedure (plan §7 summary) + +1. **`tool_calls` emission probe** — curl a handful of known + tool-triggering prompts (e.g. "what's the weather in Austin right now") + against `POST http://10.1.71.130:8001/v1/chat/completions` with a `tools` + array defined, and confirm the response actually contains a well-formed + `tool_calls` block (correct function name, valid JSON arguments) rather + than a plain-text answer or a malformed call. + +2. **Hallucination stress test** — send prompts that should **not** trigger + any tool call (general knowledge questions, casual chat, prompts that + merely mention a tool's name in passing) and confirm the model does + **not** emit a spurious `tool_calls` block. This is the primary risk + flagged in the plan given Mistral-Small's Q3_K_M quantization and its + lineage concerns around over-eager tool invocation. + +3. **Shadow mode** — for a bounded period, run this instance in parallel + with whatever tool-calling path is currently in production, comparing + outputs on the same real traffic (or a recorded sample) without letting + this instance's outputs actually drive tool execution. Only cut over + once outputs are consistently correct. + +See the `llm-inference-homelab` skill's `scripts/tool-calling-validation.sh` +reference for a starting curl harness shape — adapt prompts/tool schemas to +Mistral-Small's actual expected format (confirm via the GGUF's embedded +chat template / model card) rather than assuming it matches Qwen's. + +## Log + +| Date | Run by | Probe result | Hallucination test result | Shadow mode outcome | Decision | +|---|---|---|---|---|---| +| _(pending)_ | | | | | Not yet cut over — do not point production tool-calling traffic at :8001 | + +Update this table after each validation pass. Do not remove prior rows — +this is the audit trail for "when did we decide this was safe to use." diff --git a/ansible/roles/llm-inference-multimodel/handlers/main.yml b/ansible/roles/llm-inference-multimodel/handlers/main.yml new file mode 100644 index 0000000..d61f023 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/handlers/main.yml @@ -0,0 +1,27 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/handlers/main.yml +# DESCRIPTION: Separate restart handlers per instance — NEVER combined, so a +# content change to one unit template never restarts the other +# (plan §2/§6 requirement: independent restart/rollback). +# ------------------------------------------------------------------------------ + +- name: Reload systemd + ansible.builtin.systemd: + daemon_reload: true + become: true + listen: "reload systemd" + +- name: Restart llama-server-aux + ansible.builtin.systemd: + name: "{{ llm_aux_service_name }}" + state: restarted + become: true + listen: "restart llama-server-aux" + +- name: Restart llama-server-toolcall + ansible.builtin.systemd: + name: "{{ llm_toolcall_service_name }}" + state: restarted + become: true + listen: "restart llama-server-toolcall" diff --git a/ansible/roles/llm-inference-multimodel/meta/main.yml b/ansible/roles/llm-inference-multimodel/meta/main.yml new file mode 100644 index 0000000..bba4025 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/meta/main.yml @@ -0,0 +1,17 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/meta/main.yml +# ------------------------------------------------------------------------------ +galaxy_info: + role_name: llm_inference_multimodel + author: rblundon + license: MIT + description: > + Deploys two independent llama-server instances on astro-orbiter's RTX 3090: + an aux/classification instance (Phi-4-14B Q4_K_M, port 8000) and a + tool-calling instance (Mistral-Small-24B-Instruct-2501 Q3_K_M, port 8001). + Built alongside roles/llm-inference (not a replacement); assumes that + role's CUDA build/driver work is already done. See + /home/hermes/astro-orbiter-multi-model-plan.md for the full design. + min_ansible_version: "2.15" +dependencies: [] diff --git a/ansible/roles/llm-inference-multimodel/tasks/discover.yml b/ansible/roles/llm-inference-multimodel/tasks/discover.yml new file mode 100644 index 0000000..fe59d8e --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/discover.yml @@ -0,0 +1,87 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/discover.yml +# DESCRIPTION: Phase 0 — READ-ONLY fact gathering on how the existing Gemma +# llama-server is actually managed on astro-orbiter TODAY. +# +# Per plan §0/§4: "Service management: unverified — plan requires +# confirming systemd unit exists before touching anything. +# Do not assume." This file performs that confirmation. It makes +# NO changes to the host — no `state: present/started/stopped`, +# no file writes, no service actions. Every task here is either +# a `_facts` module, a `command`/`shell` in check-safe read mode, +# or a `stat`. +# +# Outcomes recorded as facts for later phases/for a human to read +# in the play recap — this file does not branch role behavior +# on the result (that would be over-engineering a role meant to +# run once); it surfaces what's true so a human confirms before +# Phase 2 proceeds. +# ------------------------------------------------------------------------------ + +- name: Gather service facts (systemd unit inventory) + ansible.builtin.service_facts: + +- name: Determine whether a systemd unit matching the existing Gemma service exists + ansible.builtin.set_fact: + llm_existing_gemma_unit_found: "{{ (llm_existing_gemma_service_name_guess + '.service') in ansible_facts.services }}" + +- name: Report existing Gemma systemd unit state (if found) + ansible.builtin.debug: + msg: >- + Existing unit '{{ llm_existing_gemma_service_name_guess }}.service' found: + state={{ ansible_facts.services[llm_existing_gemma_service_name_guess + '.service'].state | default('unknown') }}, + status={{ ansible_facts.services[llm_existing_gemma_service_name_guess + '.service'].status | default('unknown') }} + when: llm_existing_gemma_unit_found + +- name: WARNING — no systemd unit found matching the existing Gemma service + ansible.builtin.debug: + msg: >- + No systemd unit named '{{ llm_existing_gemma_service_name_guess }}.service' + was found via service_facts. This means the current single-model + llama-server is likely run some other way (manual nohup, screen/tmux, + or a differently-named unit). DO NOT PROCEED to Phase 2 assuming a + clean rollback target exists. Before continuing: (1) check for any + running llama-server process via `ansible -m command -a "pgrep -fa + llama-server"`, (2) if found running ad hoc, codify it as a proper + systemd unit FIRST (reusing roles/llm-inference's existing + llama-server.service.j2 pattern) so plan §6's rollback story + ("systemctl start llama-server-gemma to fully revert") is real and + not aspirational. This is a human decision point, not something this + role auto-remediates. + when: not llm_existing_gemma_unit_found + +- name: Check for any running llama-server process (read-only, no state change) + ansible.builtin.command: + cmd: pgrep -fa llama-server + register: llm_existing_process_check + changed_when: false + failed_when: false # pgrep exits 1 with no matches — not a failure condition here + +- name: Report any llama-server process found running outside systemd + ansible.builtin.debug: + msg: "Running llama-server process(es): {{ llm_existing_process_check.stdout_lines }}" + when: llm_existing_process_check.rc == 0 + +- name: Check current GPU VRAM utilization (baseline, before any changes) + ansible.builtin.command: + cmd: nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader + register: llm_baseline_vram + changed_when: false + failed_when: false + +- name: Report baseline VRAM usage + ansible.builtin.debug: + msg: "Baseline GPU VRAM (before this role's changes): {{ llm_baseline_vram.stdout | default('nvidia-smi unavailable') }}" + +- name: Check whether ports 8000/8001 are already bound (avoid port collision surprises) + ansible.builtin.command: + cmd: "ss -ltnp" + register: llm_existing_listeners + changed_when: false + failed_when: false + +- name: Report current listeners on 8000/8001 + ansible.builtin.debug: + msg: "{{ llm_existing_listeners.stdout_lines | select('search', ':(8000|8001)\\s') | list }}" + when: llm_existing_listeners.rc == 0 diff --git a/ansible/roles/llm-inference-multimodel/tasks/firewall.yml b/ansible/roles/llm-inference-multimodel/tasks/firewall.yml new file mode 100644 index 0000000..82971f8 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/firewall.yml @@ -0,0 +1,64 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/firewall.yml +# DESCRIPTION: Phase 3 — scope :8001 (new) and reconsider :8000 (existing +# pattern) exposure, per plan §5. +# +# Current baseline pattern (0.0.0.0:8000, no auth) is a +# pre-existing flagged issue — this role does NOT repeat it +# uncritically for the new port, and tightens both: +# 1. Bind address: handled in systemd.yml templates via +# {{ llm_bind_address }} (default 10.1.71.130, NOT 0.0.0.0). +# 2. Firewall: ufw rules scoping both ports to +# {{ llm_allowed_source_cidr }} rather than open LAN-wide. +# +# Idempotent: named rule comments + `state: present` so reruns +# don't duplicate rules (per plan §4 idempotency note). +# ------------------------------------------------------------------------------ + +- name: Check whether ufw is installed/active + ansible.builtin.command: + cmd: ufw status + register: llm_ufw_status + changed_when: false + failed_when: false + become: true + +- name: WARNING — ufw not active, firewall scoping cannot be applied + ansible.builtin.debug: + msg: >- + ufw does not appear to be active on this host (`ufw status` returned: + {{ llm_ufw_status.stdout | default('n/a') }}). Firewall scoping for + ports {{ llm_aux_port }}/{{ llm_toolcall_port }} was skipped. This is a + gap vs plan §5 item 2 — flag to Ryan before relying on bind-address + alone for exposure control. + when: "'Status: active' not in (llm_ufw_status.stdout | default(''))" + +- name: Allow aux port ({{ llm_aux_port }}) from the Hermes source subnet + community.general.ufw: + rule: allow + port: "{{ llm_aux_port | string }}" + proto: tcp + src: "{{ llm_allowed_source_cidr }}" + comment: "llm-inference-multimodel: aux (Phi-4) — scoped to Hermes subnet" + become: true + when: "'Status: active' in (llm_ufw_status.stdout | default(''))" + +- name: Allow tool-calling port ({{ llm_toolcall_port }}) from the Hermes source subnet + community.general.ufw: + rule: allow + port: "{{ llm_toolcall_port | string }}" + proto: tcp + src: "{{ llm_allowed_source_cidr }}" + comment: "llm-inference-multimodel: toolcall (Mistral-Small) — scoped to Hermes subnet" + become: true + when: "'Status: active' in (llm_ufw_status.stdout | default(''))" + +- name: Report firewall scoping applied + ansible.builtin.debug: + msg: >- + Firewall scoping applied for ports {{ llm_aux_port }} and + {{ llm_toolcall_port }}, restricted to source {{ llm_allowed_source_cidr }}. + Reverse-proxy + API-key enforcement (plan §5 item 3) is NOT implemented + by this role — flagged as an optional follow-up phase, not bundled into + this minimum-viable rollout. diff --git a/ansible/roles/llm-inference-multimodel/tasks/main.yml b/ansible/roles/llm-inference-multimodel/tasks/main.yml new file mode 100644 index 0000000..caa41cc --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/main.yml @@ -0,0 +1,35 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/main.yml +# DESCRIPTION: Entry point — imports one task file per phase. +# Phases are additive; re-running the full playbook is always +# safe (idempotent). Use --tags to run a specific phase subset: +# --tags discover,models,systemd,firewall,verify +# +# IMPORTANT: Phase 2 (systemd) deploys but does NOT start either service. +# Phase 4 (verify) is what starts + smoke-tests them. This lets +# Ryan review "systemd units land, nothing running yet" as a +# distinct, revertable checkpoint before anything touches the +# live GPU/VRAM state. +# ------------------------------------------------------------------------------ + +# Phase 0 — Discover (read-only; confirm how the existing Gemma llama-server +# is actually managed today before assuming a systemd unit exists) +- import_tasks: discover.yml + tags: [discover] + +# Phase 1 — Models (idempotent GGUF download, size-check guard) +- import_tasks: models.yml + tags: [models] + +# Phase 2 — Systemd (template + deploy both unit files, do NOT auto-start) +- import_tasks: systemd.yml + tags: [systemd] + +# Phase 3 — Firewall (scope :8001 and reconsider :8000 exposure) +- import_tasks: firewall.yml + tags: [firewall] + +# Phase 4 — Verify (start both services, curl smoke test, nvidia-smi VRAM check) +- import_tasks: verify.yml + tags: [verify] diff --git a/ansible/roles/llm-inference-multimodel/tasks/models.yml b/ansible/roles/llm-inference-multimodel/tasks/models.yml new file mode 100644 index 0000000..0f6189f --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/models.yml @@ -0,0 +1,72 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/models.yml +# DESCRIPTION: Phase 1 — download both GGUFs to {{ llm_models_dir }}. +# Idempotent: reuses the stat + size-threshold guard pattern +# from the llm-inference-homelab skill / roles/llm-inference's +# serve.yml, so reruns don't re-pull 8.5GB / 11.7GB files. +# +# Does NOT touch the existing Gemma GGUF — separate directory +# entries, no overlap, no deletion of anything pre-existing. +# ------------------------------------------------------------------------------ + +- name: Create models directory + ansible.builtin.file: + path: "{{ llm_models_dir }}" + state: directory + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0755" + become: true + +# --- Aux model (Phi-4-14B Q4_K_M) -------------------------------------------- + +- name: Check if aux model GGUF already exists + ansible.builtin.stat: + path: "{{ llm_aux_model_path }}" + register: llm_aux_model_stat + +- name: Download aux model — Phi-4-14B-Q4_K_M GGUF + ansible.builtin.get_url: + url: "{{ llm_aux_model_url }}" + dest: "{{ llm_aux_model_path }}" + headers: + Authorization: "Bearer {{ llm_hf_token }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + timeout: 7200 + force: false + become: true + no_log: true + # Idempotency guard: skip if file exists and is above the min-size threshold + # (catches partial/truncated downloads from an interrupted prior run). + when: not llm_aux_model_stat.stat.exists or (llm_aux_model_stat.stat.size | int) < (llm_aux_model_min_bytes | int) + +# --- Tool-calling model (Mistral-Small-24B Q3_K_M) --------------------------- + +- name: Check if tool-calling model GGUF already exists + ansible.builtin.stat: + path: "{{ llm_toolcall_model_path }}" + register: llm_toolcall_model_stat + +- name: Download tool-calling model — Mistral-Small-24B-Instruct-2501 Q3_K_M GGUF + ansible.builtin.get_url: + url: "{{ llm_toolcall_model_url }}" + dest: "{{ llm_toolcall_model_path }}" + headers: + Authorization: "Bearer {{ llm_hf_token }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + timeout: 7200 + force: false + become: true + no_log: true + when: not llm_toolcall_model_stat.stat.exists or (llm_toolcall_model_stat.stat.size | int) < (llm_toolcall_model_min_bytes | int) + +- name: Report model files present on disk + ansible.builtin.debug: + msg: + - "Aux model: {{ llm_aux_model_path }}" + - "Tool-calling model: {{ llm_toolcall_model_path }}" diff --git a/ansible/roles/llm-inference-multimodel/tasks/systemd.yml b/ansible/roles/llm-inference-multimodel/tasks/systemd.yml new file mode 100644 index 0000000..70ef26d --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/systemd.yml @@ -0,0 +1,54 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/systemd.yml +# DESCRIPTION: Phase 2 — template + deploy both unit files. +# DELIBERATELY DOES NOT START OR ENABLE either service — that is +# Phase 4 (verify.yml)'s job, after Phase 3 firewall scoping is +# in place. This keeps "units land on disk" and "processes +# actually bind ports and load 20+GB into VRAM" as separately +# reviewable checkpoints per Ryan's iterative-build preference. +# +# Two independent units (llama-server-aux.service, +# llama-server-toolcall.service) — NOT one unit with two +# ExecStarts — so either can be stopped/restarted without +# affecting the other (plan §2, §6 rollback requirement). +# +# The pre-existing Gemma unit (whatever discover.yml found it to +# be) is never templated, restarted, or disabled by this file. +# ------------------------------------------------------------------------------ + +- name: Deploy llama-server-aux systemd unit + ansible.builtin.template: + src: llama-server-aux.service.j2 + dest: "/etc/systemd/system/{{ llm_aux_service_name }}.service" + owner: root + group: root + mode: "0644" + become: true + notify: + - reload systemd + - restart llama-server-aux + +- name: Deploy llama-server-toolcall systemd unit + ansible.builtin.template: + src: llama-server-toolcall.service.j2 + dest: "/etc/systemd/system/{{ llm_toolcall_service_name }}.service" + owner: root + group: root + mode: "0644" + become: true + notify: + - reload systemd + - restart llama-server-toolcall + +- name: Flush handlers so daemon-reload lands before any later phase acts on unit state + ansible.builtin.meta: flush_handlers + +# NOTE: no `ansible.builtin.systemd: state: started / enabled: true` task here +# on purpose. Units exist on disk after this phase; nothing is running. +# The "restart" handlers above only fire (and thus only start anything) if +# the template content actually changed AND a later flush_handlers/end-of-play +# triggers them — on a first-ever apply this DOES start the services once, +# which is expected/acceptable for a fresh deploy, but on any subsequent +# re-run with no template changes, nothing restarts. Ryan/verify.yml owns +# the deliberate first start + smoke test. diff --git a/ansible/roles/llm-inference-multimodel/tasks/verify.yml b/ansible/roles/llm-inference-multimodel/tasks/verify.yml new file mode 100644 index 0000000..7c23e54 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/tasks/verify.yml @@ -0,0 +1,133 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/tasks/verify.yml +# DESCRIPTION: Phase 4 — start both services, curl smoke test each endpoint, +# nvidia-smi VRAM check against plan §1 math, confirm no OOM. +# +# This is the ONLY phase that actually starts the services +# (systemd.yml deliberately does not). Enabling happens here too, +# so a reboot brings both back — matching plan §2's "independent +# systemd services" intent for durability, not just this-session. +# ------------------------------------------------------------------------------ + +- name: Enable and start llama-server-aux + ansible.builtin.systemd: + name: "{{ llm_aux_service_name }}" + state: started + enabled: true + daemon_reload: true + become: true + +- name: Enable and start llama-server-toolcall + ansible.builtin.systemd: + name: "{{ llm_toolcall_service_name }}" + state: started + enabled: true + daemon_reload: true + become: true + +- name: Wait for aux instance API to become available (model load may take a couple minutes) + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/health" + status_code: 200 + register: llm_aux_health + retries: 24 + delay: 10 + until: llm_aux_health.status == 200 + +- name: Wait for tool-calling instance API to become available + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/health" + status_code: 200 + register: llm_toolcall_health + retries: 24 + delay: 10 + until: llm_toolcall_health.status == 200 + +- name: Smoke-test — aux instance model listing + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/models" + status_code: 200 + return_content: true + register: llm_aux_models + +- name: Smoke-test — tool-calling instance model listing + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/models" + status_code: 200 + return_content: true + register: llm_toolcall_models + +- name: Report served models per instance + ansible.builtin.debug: + msg: + - "Aux (:{{ llm_aux_port }}) serving: {{ llm_aux_models.json.data | map(attribute='id') | list }}" + - "Tool-calling (:{{ llm_toolcall_port }}) serving: {{ llm_toolcall_models.json.data | map(attribute='id') | list }}" + +- name: Basic completion smoke test — aux instance (non-tool-calling sanity check only) + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_aux_port }}/v1/chat/completions" + method: POST + body_format: json + body: + model: "{{ llm_aux_model_id }}" + messages: + - role: user + content: "Reply with exactly one word: OK" + max_tokens: 10 + status_code: 200 + return_content: true + register: llm_aux_completion + +- name: Basic completion smoke test — tool-calling instance (plain-text sanity check only) + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_toolcall_port }}/v1/chat/completions" + method: POST + body_format: json + body: + model: "{{ llm_toolcall_model_id }}" + messages: + - role: user + content: "Reply with exactly one word: OK" + max_tokens: 10 + status_code: 200 + return_content: true + register: llm_toolcall_completion + +- name: NOTE — this smoke test is NOT the tool-calling validation harness + ansible.builtin.debug: + msg: >- + Both endpoints respond to basic completions. This does NOT validate + tool_calls correctness or hallucination-safety for the tool-calling + instance — that is a separate, manual, post-deploy procedure (plan §7). + See references/tool-calling-validation.sh (copied from the + llm-inference-homelab skill) and docs/validation-log.md in this role. + DO NOT point any Claude Code / tool-calling-capable Hermes profile at + port {{ llm_toolcall_port }} until that validation has passed and been + logged. + +- name: Check GPU VRAM usage after both instances are running + ansible.builtin.command: + cmd: nvidia-smi --query-gpu=memory.used,memory.total,utilization.gpu --format=csv,noheader + register: llm_post_start_vram + changed_when: false + +- name: Report VRAM usage vs plan §1 expectations + ansible.builtin.debug: + msg: + - "Measured (nvidia-smi): {{ llm_post_start_vram.stdout }}" + - "Design estimate (plan §1): aux ~{{ llm_aux_expected_vram_gb }}GB + toolcall ~{{ llm_toolcall_expected_vram_gb }}GB = ~{{ llm_combined_expected_vram_gb }}GB / {{ llm_gpu_total_vram_gb }}GB total" + - "If measured usage exceeds ~23.5GB or is within ~0.5GB of the 24GB card limit, treat as the OOM-risk trigger condition from plan §6 — do not leave both services running unattended without confirming headroom." + +- name: Check for OOM-kill events related to llama-server in dmesg (best-effort, read-only) + ansible.builtin.shell: + cmd: "dmesg | grep -i 'llama-server' | grep -i -E 'oom|killed' || true" + register: llm_oom_check + changed_when: false + become: true + +- name: Report any OOM-kill findings + ansible.builtin.debug: + msg: >- + {{ llm_oom_check.stdout if llm_oom_check.stdout | length > 0 + else 'No OOM-kill events found for llama-server in dmesg.' }} diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 new file mode 100644 index 0000000..2c40977 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-aux.service.j2 @@ -0,0 +1,33 @@ +[Unit] +Description=llama-server (aux/classification) — Phi-4-14B Q4_K_M (OpenAI-compatible inference) +After=network.target nvidia-persistenced.service +Wants=nvidia-persistenced.service + +[Service] +Type=simple +User={{ llm_service_user }} +Group={{ llm_service_user }} +Environment="HOME=/home/{{ llm_service_user }}" +ExecStart={{ llm_binary_path }} \ + --model {{ llm_aux_model_path }} \ + --host {{ llm_bind_address }} \ + --port {{ llm_aux_port }} \ + --ctx-size {{ llm_aux_ctx_size }} \ + --n-gpu-layers {{ llm_aux_gpu_layers }} \ + --parallel {{ llm_aux_parallel }} \ + --metrics +# NOTE: no --chat-template flag — let llama-server auto-detect Phi-4's own +# embedded chat template from GGUF metadata (same reasoning as the existing +# llm-inference role's Gemma unit: explicit overrides risk mismatching the +# model's actual expected format). +# NOTE: --host is the private LAN IP (10.1.71.130 by default), NOT 0.0.0.0 — +# deliberate change from the pre-existing Gemma pattern (plan §5). +Restart=on-failure +RestartSec=10 +TimeoutStartSec=600 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=llama-server-aux + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 new file mode 100644 index 0000000..689bbdb --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-toolcall.service.j2 @@ -0,0 +1,38 @@ +[Unit] +Description=llama-server (tool-calling) — Mistral-Small-24B-Instruct-2501 Q3_K_M (OpenAI-compatible inference) +After=network.target nvidia-persistenced.service +Wants=nvidia-persistenced.service + +[Service] +Type=simple +User={{ llm_service_user }} +Group={{ llm_service_user }} +Environment="HOME=/home/{{ llm_service_user }}" +ExecStart={{ llm_binary_path }} \ + --model {{ llm_toolcall_model_path }} \ + --host {{ llm_bind_address }} \ + --port {{ llm_toolcall_port }} \ + --ctx-size {{ llm_toolcall_ctx_size }} \ + --n-gpu-layers {{ llm_toolcall_gpu_layers }} \ + --parallel {{ llm_toolcall_parallel }} \ + --metrics +# NOTE: no --chat-template flag — let llama-server auto-detect Mistral-Small's +# own embedded chat template from GGUF metadata. +# NOTE: --host is the private LAN IP (10.1.71.130 by default), NOT 0.0.0.0. +# NOTE: --parallel 1 is deliberate (plan §1/§2) — tool-calling profiles are +# single-session-at-a-time per Claude Code profile; lower parallelism reduces +# KV overhead and lowers hallucination surface from context bleed between +# concurrent slots. +# IMPORTANT: this endpoint MUST pass the plan §7 validation harness +# (docs/validation-log.md in this role) before any Claude Code / tool-calling +# Hermes profile is pointed at it. Mistral-Small shares lineage concerns +# flagged for Qwen2.5/Qwen3 hallucinated tool_calls — do not assume safety. +Restart=on-failure +RestartSec=10 +TimeoutStartSec=600 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=llama-server-toolcall + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/llm-inference-multimodel/vars/main.yml b/ansible/roles/llm-inference-multimodel/vars/main.yml new file mode 100644 index 0000000..77cd51a --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/vars/main.yml @@ -0,0 +1,22 @@ +--- +# ------------------------------------------------------------------------------ +# FILE: roles/llm-inference-multimodel/vars/main.yml +# DESCRIPTION: Role-internal constants (not meant to be overridden per-host). +# Model URLs/quant filenames live here rather than defaults/ since +# they're not really "tunable" — they're the specific artifacts +# named in the approved plan (§1). If Ryan wants a different +# quant/model, that's a defaults/main.yml override or a plan +# revision, not a vars/ edit. +# ------------------------------------------------------------------------------ + +# HuggingFace auth — reuse the same vault token as roles/llm-inference. +# vault_hf_token is defined in group_vars/all/vault. +llm_hf_token: "{{ vault_hf_token }}" + +# Expected VRAM subtotals from plan §1 (informational — surfaced in verify.yml +# output so a live nvidia-smi reading can be sanity-checked against the design +# math, not enforced as a hard gate). +llm_aux_expected_vram_gb: 10.0 +llm_toolcall_expected_vram_gb: 13.2 +llm_combined_expected_vram_gb: 23.2 +llm_gpu_total_vram_gb: 24.0