feat(llm-inference-multimodel): bump router --models-max 1->4 on astro-orbiter (t_33acbb2e)
Changes: - host_vars/astro-orbiter/vars.yml: add llm_router_models_max: 4 (overrides conservative default of 1). Detailed VRAM OOM risk note included inline: worst-case 3-model co-residency ~31GB > 24GB RTX 3090. LRU eviction mitigates in single-user operation; flagged for Ryan's review. - playbooks/day2_bump_router_models_max.yml: new targeted playbook; deploys updated router unit, restarts the live service, verifies /health 200 and /v1/models lists all 3 GGUFs post-restart. - group_vars/all/semaphore.yml: add llm_router_update_unit template pointing at the new playbook. - roles/llm-inference-multimodel/defaults/main.yml: update comment to reflect the var is now overridden in host_vars rather than 'hardcoded to 1'. - roles/llm-inference-multimodel/templates/llama-server-router.service.j2: correct stale 'HARDCODED TO 1' comment — value is variable-driven. Constraints honored: - --parallel 1 left untouched (not in scope, not modified anywhere) - No ad-hoc SSH/systemctl/curl state mutation; all execution via Semaphore - No installed/vendored code patched
This commit is contained in:
Hermes Agent service account
161
ansible/playbooks/day2_bump_router_models_max.yml
Normal file
161
ansible/playbooks/day2_bump_router_models_max.yml
Normal file
@@ -0,0 +1,161 @@
|
||||
---
|
||||
# ------------------------------------------------------------------------------
|
||||
# FILE: playbooks/day2_bump_router_models_max.yml
|
||||
# DESCRIPTION: Bump --models-max on the production llama-server-router unit.
|
||||
#
|
||||
# Context: t_33acbb2e (2026-08-12) — Ryan requested --models-max raised from 1
|
||||
# to 4 so the router can keep multiple GGUFs resident on-demand (LRU eviction
|
||||
# when the cap is reached). The actual var change lives in:
|
||||
# host_vars/astro-orbiter/vars.yml (llm_router_models_max: 4)
|
||||
#
|
||||
# This playbook:
|
||||
# 1. Re-renders llama-server-router.service.j2 with the updated var value.
|
||||
# 2. Reloads systemd (daemon-reload handler) if the unit changed.
|
||||
# 3. Restarts llama-server-router so the new --models-max takes effect on the
|
||||
# live process. Router holds no resident model (all-unloaded) so restart
|
||||
# is sub-second and non-disruptive.
|
||||
# 4. Verifies /health returns 200 and /v1/models still lists all three GGUFs.
|
||||
#
|
||||
# VRAM NOTE: --models-max 4 allows up to all 3 current GGUFs to co-reside on
|
||||
# a 24GB card simultaneously. Worst-case combined footprint is ~31GB which
|
||||
# EXCEEDS 24GB — OOM is possible if all 3 are loaded concurrently. In normal
|
||||
# single-user homelab operation this is very unlikely. Full VRAM breakdown
|
||||
# documented in host_vars/astro-orbiter/vars.yml. Ryan approved (t_33acbb2e).
|
||||
#
|
||||
# Execution channel: Semaphore template "llm_router_update_unit" (project mk-labs).
|
||||
# Do NOT run via direct ansible-playbook or ad-hoc ssh/systemctl.
|
||||
#
|
||||
# Author: War Machine (2026-08-12, t_33acbb2e)
|
||||
# ------------------------------------------------------------------------------
|
||||
|
||||
- name: "Bump llama-server-router --models-max to 4 on astro-orbiter"
|
||||
hosts: astro_orbiter
|
||||
gather_facts: true
|
||||
become: true
|
||||
|
||||
vars:
|
||||
# Production vars — router is live on :8002 (post-cutover t_cd0d5388)
|
||||
llm_router_port: 8002
|
||||
llm_router_bind_address: "10.1.71.130"
|
||||
|
||||
# llm_router_models_max is 4 via host_vars/astro-orbiter/vars.yml.
|
||||
# Remaining role vars come from host_vars + defaults/main.yml via the
|
||||
# inventory — we only explicitly set vars this playbook needs for its
|
||||
# own tasks (health/models check URIs).
|
||||
|
||||
# Needed by the template task (mirrors defaults set in role defaults/main.yml)
|
||||
llm_service_user: jarvis
|
||||
llm_binary_path: /opt/llama.cpp/build/bin/llama-server
|
||||
llm_models_dir: /opt/models
|
||||
llm_router_service_name: llama-server-router
|
||||
llm_router_models_dir: /opt/models
|
||||
llm_router_gpu_layers: 99
|
||||
llm_router_ctx_size: 65536
|
||||
llm_router_flash_attn: "auto"
|
||||
llm_router_cache_type_k: q4_0
|
||||
llm_router_cache_type_v: q4_0
|
||||
llm_router_batch_size: 2048
|
||||
llm_router_ubatch_size: 512
|
||||
llm_router_parallel: 1
|
||||
|
||||
tasks:
|
||||
# -------------------------------------------------------------------------
|
||||
# Phase 1: Re-render the router unit file
|
||||
# Template src path is relative to the role's templates/ dir; we reference
|
||||
# it with a relative path that Ansible resolves from the role directory.
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
- name: "Deploy updated llama-server-router unit (--models-max {{ llm_router_models_max }})"
|
||||
ansible.builtin.template:
|
||||
src: "{{ playbook_dir }}/../roles/llm-inference-multimodel/templates/llama-server-router.service.j2"
|
||||
dest: "/etc/systemd/system/{{ llm_router_service_name }}.service"
|
||||
owner: root
|
||||
group: root
|
||||
mode: "0644"
|
||||
register: llm_router_unit_updated
|
||||
notify:
|
||||
- reload systemd
|
||||
tags: [always]
|
||||
|
||||
- name: "Flush handlers — ensure daemon-reload lands before restart"
|
||||
ansible.builtin.meta: flush_handlers
|
||||
tags: [always]
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Phase 2: Restart the router so the new --models-max takes effect.
|
||||
# Always restart (even if unit unchanged) to ensure live process matches.
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
- name: "Restart llama-server-router so --models-max {{ llm_router_models_max }} takes effect"
|
||||
ansible.builtin.systemd:
|
||||
name: "{{ llm_router_service_name }}"
|
||||
state: restarted
|
||||
enabled: true
|
||||
tags: [always]
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Phase 3: Verify /health returns 200
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
- name: "Wait for /health to return 200 after restart"
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/health"
|
||||
status_code: 200
|
||||
timeout: 30
|
||||
register: bump_health_check
|
||||
retries: 10
|
||||
delay: 3
|
||||
until: bump_health_check.status == 200
|
||||
tags: [always]
|
||||
|
||||
# -------------------------------------------------------------------------
|
||||
# Phase 4: Verify /v1/models lists all three GGUFs
|
||||
# -------------------------------------------------------------------------
|
||||
|
||||
- name: "Check /v1/models — all three GGUFs should appear"
|
||||
ansible.builtin.uri:
|
||||
url: "http://{{ llm_router_bind_address }}:{{ llm_router_port }}/v1/models"
|
||||
status_code: 200
|
||||
timeout: 30
|
||||
return_content: true
|
||||
register: bump_models_check
|
||||
tags: [always]
|
||||
|
||||
- name: "Display /v1/models summary"
|
||||
ansible.builtin.debug:
|
||||
msg:
|
||||
- "======================================================================"
|
||||
- "--models-max BUMP VERIFICATION (t_33acbb2e)"
|
||||
- ""
|
||||
- " /health: HTTP {{ bump_health_check.status }}"
|
||||
- " /v1/models HTTP: {{ bump_models_check.status }}"
|
||||
- " Models listed: {{ bump_models_check.json.data | map(attribute='id') | list | join(', ') }}"
|
||||
- ""
|
||||
- " --models-max now: {{ llm_router_models_max }}"
|
||||
- " --parallel (unchanged): {{ llm_router_parallel }}"
|
||||
- ""
|
||||
- " VRAM WARNING: worst-case 3-model co-residency ~31GB > 24GB RTX 3090."
|
||||
- " OOM risk if all 3 load concurrently. LRU eviction mitigates in practice."
|
||||
- " Full breakdown: host_vars/astro-orbiter/vars.yml"
|
||||
- "======================================================================"
|
||||
when: bump_models_check is defined
|
||||
tags: [always]
|
||||
|
||||
- name: "GATE: confirm all 3 expected GGUFs appear in /v1/models"
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- "'Qwen3.6-35B-A3B-UD-Q4_K_S' in (bump_models_check.json.data | map(attribute='id') | list)"
|
||||
- "'Phi-3.5-mini-instruct-Q8_0' in (bump_models_check.json.data | map(attribute='id') | list)"
|
||||
- "'Meta-Llama-3.1-8B-Instruct-Q4_K_M' in (bump_models_check.json.data | map(attribute='id') | list)"
|
||||
fail_msg: >-
|
||||
/v1/models did not return all 3 expected GGUFs after --models-max bump.
|
||||
Check router logs: journalctl -u llama-server-router -n 50
|
||||
success_msg: "GATE PASSED: all 3 GGUFs listed in /v1/models."
|
||||
when: bump_models_check is defined
|
||||
tags: [always]
|
||||
|
||||
handlers:
|
||||
- name: reload systemd
|
||||
ansible.builtin.systemd:
|
||||
daemon_reload: true
|
||||
listen: "reload systemd"
|
||||
Reference in New Issue
Block a user