diff --git a/ansible/roles/llm-inference-multimodel/README.md b/ansible/roles/llm-inference-multimodel/README.md index 7e39565..c63a706 100644 --- a/ansible/roles/llm-inference-multimodel/README.md +++ b/ansible/roles/llm-inference-multimodel/README.md @@ -105,15 +105,44 @@ correctness/safety judgment call, not a repeatable infra check. See `docs/validation-log.md` in this role directory for the procedure reference and a place to log results once Ryan runs it. -## Known gap: Semaphore is broken (as of 2026-08-05) +## Known gap: Semaphore execution path bypassed for this role (2026-08-05) -The normal execution/audit path (Semaphore) is currently non-functional. -This role was authored to be run via direct `ansible-playbook` as an accepted -interim stopgap, executed personally by Ryan. **This is a known gap, not the -intended long-term operational path** — once Semaphore is repaired, retarget -execution of this role (and future changes to it) through Semaphore so runs -are audited/logged there again. Flag this in any future work that touches -this role. +The normal execution/audit path (Semaphore) was believed non-functional at authoring time, so this role was run via direct `ansible-playbook` instead, executed personally by Ryan. + +**Confirmed 2026-08-05 (JARVIS, via Semaphore API — token `vault_semaphore_api_token` in the homelab Ansible vault):** this was a misdiagnosis, not an outage. Semaphore's service, Postgres backend, and API (`/api/ping` returns `pong`) are all healthy on figment (10.1.71.37 — note the documented host `city-hall`/10.1.71.38 is stale; DNS for `imagineering.local.mk-labs.cloud` actually resolves through Traefik on lightning-lane to figment:3000). Queried `/api/project/1/templates` directly: only 6 templates exist project-wide (day0 baseline/root-LV checks, day1 Semaphore self-deploy, Traefik route updates) — **none for this role, nor for the original single-model `llm-inference` role**. Root cause confirmed: no Semaphore project template was ever created for LLM inference deployment, which presents identically to "Semaphore is broken" if you don't check the template list. + +**This is still a known gap** — direct `ansible-playbook` execution bypasses the audit trail Semaphore normally provides. Create a project template for this role's playbook and retarget execution through Semaphore so runs are audited/logged there. Flag this in any future work that touches this role. + +## Rollback + +## Qwen2.5-14B shadow deployment (port 8002) — 2026-08-06 + +Added a third instance definition (`llama-server-qwen`) per +`/home/hermes/reports/local-llm-64k-context-recommendation.md`, intended to +eventually replace the `llama-server-toolcall` (8001) slot once validated — +runs alongside 8000/8001 during the shadow-test window, does not stop or +replace either. + +**VRAM GATE — service NOT started as of this commit.** Live `nvidia-smi` check +on 2026-08-06 showed Phi-4 (8000, ~10.4GB) + Mistral (8001, ~6.2GB) already +consuming ~16.6GB / 24GB, leaving only ~7.5GB free. Qwen2.5-14B-Instruct +Q5_K_M weights alone are ~10-12GB — **does not fit concurrently** with both +existing instances at full GPU offload. The unit is deployed to disk +(`llm_qwen_service_enabled: false` default in `defaults/main.yml`) but will +not start until this is resolved. Options for the shadow-test window, +none applied yet — pick one and flip `llm_qwen_service_enabled: true`: + +1. Temporarily stop `llama-server-toolcall` (8001) for the duration of the + shadow test — it's the model being superseded anyway, so this is low-risk + and reversible (`systemctl start llama-server-toolcall` restores it). +2. Reduce Qwen's `--n-gpu-layers` (partial CPU offload) to fit the ~7.5GB + remaining headroom — will materially hurt throughput, not recommended as + first choice. +3. Reduce `--ctx-size` below 65536 — undermines the entire point of this + exercise (Hermes's 64K floor), not recommended. + +Recommended: option 1, coordinated with Ryan/JARVIS since it does touch a +live service, even though 8001 was already flagged for retirement. ## Rollback diff --git a/ansible/roles/llm-inference-multimodel/defaults/main.yml b/ansible/roles/llm-inference-multimodel/defaults/main.yml index fa80014..377659c 100644 --- a/ansible/roles/llm-inference-multimodel/defaults/main.yml +++ b/ansible/roles/llm-inference-multimodel/defaults/main.yml @@ -62,6 +62,29 @@ llm_toolcall_chat_template_file: "{{ llm_models_dir }}/templates/mistral-small-t llm_toolcall_service_name: llama-server-toolcall llm_toolcall_model_id: mistral-small-24b-instruct-2501 +# --- Shadow instance (port 8002, Qwen2.5-14B-Instruct) ----------------------- +# Shadow-deployment candidate to eventually replace the toolcall (8001) slot +# per /home/hermes/reports/local-llm-64k-context-recommendation.md. Runs +# alongside 8000/8001 during validation — does NOT replace either. +# GATE: VRAM co-residency NOT confirmed as of 2026-08-06 — see role README / +# deployment report. Do not enable/start this service until that is resolved +# (either free VRAM by stopping llama-server-toolcall for the shadow window, +# or reduce ctx-size). llm_qwen_service_enabled defaults to false so a normal +# playbook run will template the unit but NOT start it. +llm_qwen_service_enabled: false +llm_qwen_port: 8002 +llm_qwen_model_path: "{{ llm_models_dir }}/Qwen2.5-14B-Instruct-Q5_K_M.gguf" +llm_qwen_model_url: "https://huggingface.co/bartowski/Qwen2.5-14B-Instruct-GGUF/resolve/main/Qwen2.5-14B-Instruct-Q5_K_M.gguf" +llm_qwen_model_min_bytes: 9500000000 # guard threshold; complete file ~10.5GB +llm_qwen_ctx_size: 65536 +llm_qwen_parallel: 1 +llm_qwen_gpu_layers: 99 +llm_qwen_batch_size: 2048 +llm_qwen_ubatch_size: 512 +llm_qwen_service_name: llama-server-qwen +llm_qwen_model_id: qwen2.5-14b-instruct +llm_qwen_expected_vram_gb: 16.5 + # --- Existing Gemma baseline (rollback target — never modified by this role) - # Populated by Phase 0 discovery (tasks/discover.yml) if not already known. # Set here only as a fallback name to search for; discovery is authoritative. diff --git a/ansible/roles/llm-inference-multimodel/tasks/models.yml b/ansible/roles/llm-inference-multimodel/tasks/models.yml index c9fca48..5fe018a 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/models.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/models.yml @@ -97,3 +97,28 @@ group: "{{ llm_service_user }}" mode: "0644" become: true + +# --- Shadow model (Qwen2.5-14B-Instruct Q5_K_M, port 8002) -------------------- +# Downloaded unconditionally (so the ~10.5GB file is staged ahead of any VRAM +# decision) — only the *service start* is gated by llm_qwen_service_enabled +# (see verify.yml / VRAM gate note in defaults/main.yml). + +- name: Check if Qwen shadow model GGUF already exists + ansible.builtin.stat: + path: "{{ llm_qwen_model_path }}" + register: llm_qwen_model_stat + +- name: Download Qwen2.5-14B-Instruct-Q5_K_M GGUF (bartowski quant) + ansible.builtin.get_url: + url: "{{ llm_qwen_model_url }}" + dest: "{{ llm_qwen_model_path }}" + headers: + Authorization: "Bearer {{ llm_hf_token }}" + owner: "{{ llm_service_user }}" + group: "{{ llm_service_user }}" + mode: "0644" + timeout: 7200 + force: false + become: true + no_log: true + when: not llm_qwen_model_stat.stat.exists or (llm_qwen_model_stat.stat.size | int) < (llm_qwen_model_min_bytes | int) diff --git a/ansible/roles/llm-inference-multimodel/tasks/systemd.yml b/ansible/roles/llm-inference-multimodel/tasks/systemd.yml index ba95cdf..fe4c585 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/systemd.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/systemd.yml @@ -41,6 +41,18 @@ notify: - reload systemd +- name: Deploy llama-server-qwen systemd unit (shadow, port 8002) + ansible.builtin.template: + src: llama-server-qwen.service.j2 + dest: "/etc/systemd/system/{{ llm_qwen_service_name }}.service" + owner: root + group: root + mode: "0644" + become: true + register: llm_qwen_unit_deployed + notify: + - reload systemd + - name: Flush handlers so daemon-reload lands before any later phase acts on unit state ansible.builtin.meta: flush_handlers diff --git a/ansible/roles/llm-inference-multimodel/tasks/verify.yml b/ansible/roles/llm-inference-multimodel/tasks/verify.yml index f77cb3b..da73d11 100644 --- a/ansible/roles/llm-inference-multimodel/tasks/verify.yml +++ b/ansible/roles/llm-inference-multimodel/tasks/verify.yml @@ -166,6 +166,51 @@ - "Design estimate (plan §1): aux ~{{ llm_aux_expected_vram_gb }}GB + toolcall ~{{ llm_toolcall_expected_vram_gb }}GB = ~{{ llm_combined_expected_vram_gb }}GB / {{ llm_gpu_total_vram_gb }}GB total" - "If measured usage exceeds ~23.5GB or is within ~0.5GB of the 24GB card limit, treat as the OOM-risk trigger condition from plan §6 — do not leave both services running unattended without confirming headroom." +- name: Enable llama-server-qwen and start/restart (GATED — only if llm_qwen_service_enabled) + ansible.builtin.systemd: + name: "{{ llm_qwen_service_name }}" + state: "{{ 'restarted' if (llm_qwen_unit_deployed.changed | default(false)) else 'started' }}" + enabled: true + daemon_reload: true + become: true + when: llm_qwen_service_enabled | default(false) + +- name: NOTE if Qwen shadow unit was skipped due to VRAM gate + ansible.builtin.debug: + msg: >- + llama-server-qwen unit deployed to disk but NOT started + (llm_qwen_service_enabled=false). See VRAM gate note in + defaults/main.yml / deployment report — Phi-4(8000)+Mistral(8001) + already use ~16.6GB/24GB, leaving ~7.5GB free, insufficient for + Qwen2.5-14B's ~10-12GB weight footprint concurrently. Resolve before + setting llm_qwen_service_enabled: true. + when: not (llm_qwen_service_enabled | default(false)) + +- name: Wait for Qwen shadow instance API to become available (only if enabled) + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_qwen_port }}/health" + status_code: 200 + register: llm_qwen_health + retries: 24 + delay: 10 + until: llm_qwen_health.status == 200 + when: llm_qwen_service_enabled | default(false) + +- name: Smoke-test — Qwen shadow instance model listing + n_ctx verification (only if enabled) + ansible.builtin.uri: + url: "http://{{ llm_bind_address }}:{{ llm_qwen_port }}/v1/models" + status_code: 200 + return_content: true + register: llm_qwen_models + when: llm_qwen_service_enabled | default(false) + +- name: Report Qwen shadow instance served model + verified n_ctx (only if enabled) + ansible.builtin.debug: + msg: + - "Qwen shadow (:{{ llm_qwen_port }}) serving: {{ llm_qwen_models.json.data | map(attribute='id') | list }}" + - "Verified n_ctx (must be >= 64000, not just requested): {{ llm_qwen_models.json.data | map(attribute='meta') | map(attribute='n_ctx') | list }}" + when: llm_qwen_service_enabled | default(false) + - name: Check for OOM-kill events related to llama-server in dmesg (best-effort, read-only) ansible.builtin.shell: cmd: "dmesg | grep -i 'llama-server' | grep -i -E 'oom|killed' || true" diff --git a/ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2 b/ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2 new file mode 100644 index 0000000..8f23346 --- /dev/null +++ b/ansible/roles/llm-inference-multimodel/templates/llama-server-qwen.service.j2 @@ -0,0 +1,39 @@ +[Unit] +Description=llama-server (shadow) — Qwen2.5-14B-Instruct Q5_K_M (OpenAI-compatible inference, 64K ctx) +After=network.target nvidia-persistenced.service +Wants=nvidia-persistenced.service + +[Service] +Type=simple +User={{ llm_service_user }} +Group={{ llm_service_user }} +Environment="HOME=/home/{{ llm_service_user }}" +ExecStart={{ llm_binary_path }} \ + --model {{ llm_qwen_model_path }} \ + --host 0.0.0.0 \ + --port {{ llm_qwen_port }} \ + --n-gpu-layers {{ llm_qwen_gpu_layers }} \ + --ctx-size {{ llm_qwen_ctx_size }} \ + --flash-attn \ + --cache-type-k q8_0 --cache-type-v q8_0 \ + --batch-size {{ llm_qwen_batch_size }} --ubatch-size {{ llm_qwen_ubatch_size }} \ + --jinja \ + --parallel {{ llm_qwen_parallel }} \ + --metrics +# Shadow-deployment candidate per local-llm-64k-context-recommendation.md. +# NOT yet cleared for production Hermes profile routing — must pass +# scripts/tool-calling-validation.sh AND have verified n_ctx >= 64000 from +# /v1/models before any repoint decision. +# VRAM GATE: as of 2026-08-06, Phi-4(8000)+Mistral(8001) already consume +# ~16.6GB/24GB (7.5GB free). This model's weights alone are ~10-12GB — does +# NOT fit concurrently without freeing VRAM. Do not enable this unit until +# that is resolved (see role README "Qwen shadow deployment — VRAM gate"). +Restart=on-failure +RestartSec=10 +TimeoutStartSec=600 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=llama-server-qwen + +[Install] +WantedBy=multi-user.target